duckdb 0.10.1-dev9.0 → 0.10.1

package/src/duckdb/src/storage/table/column_segment.cpp

@@ -244,12 +244,15 @@ void ColumnSegment::CommitDropSegment() {
 // Filter Selection
 //===--------------------------------------------------------------------===//
 template <class T, class OP, bool HAS_NULL>
-static idx_t TemplatedFilterSelection(T *vec, T predicate, SelectionVector &sel, idx_t approved_tuple_count,
-                                      ValidityMask &mask, SelectionVector &result_sel) {
+static idx_t TemplatedFilterSelection(UnifiedVectorFormat &vdata, T predicate, SelectionVector &sel,
+                                      idx_t approved_tuple_count, SelectionVector &result_sel) {
+	auto &mask = vdata.validity;
+	auto vec = UnifiedVectorFormat::GetData<T>(vdata);
 	idx_t result_count = 0;
 	for (idx_t i = 0; i < approved_tuple_count; i++) {
 		auto idx = sel.get_index(i);
-		if ((!HAS_NULL || mask.RowIsValid(idx)) && OP::Operation(vec[idx], predicate)) {
+		auto vector_idx = vdata.sel->get_index(idx);
+		if ((!HAS_NULL || mask.RowIsValid(vector_idx)) && OP::Operation(vec[vector_idx], predicate)) {
 			result_sel.set_index(result_count++, idx);
 		}
 	}
@@ -257,68 +260,69 @@ static idx_t TemplatedFilterSelection(T *vec, T predicate, SelectionVector &sel,
 }
 
 template <class T>
-static void FilterSelectionSwitch(T *vec, T predicate, SelectionVector &sel, idx_t &approved_tuple_count,
-                                  ExpressionType comparison_type, ValidityMask &mask) {
+static void FilterSelectionSwitch(UnifiedVectorFormat &vdata, T predicate, SelectionVector &sel,
+                                  idx_t &approved_tuple_count, ExpressionType comparison_type) {
 	SelectionVector new_sel(approved_tuple_count);
+	auto &mask = vdata.validity;
 	// the inplace loops take the result as the last parameter
 	switch (comparison_type) {
 	case ExpressionType::COMPARE_EQUAL: {
 		if (mask.AllValid()) {
 			approved_tuple_count =
-			    TemplatedFilterSelection<T, Equals, false>(vec, predicate, sel, approved_tuple_count, mask, new_sel);
+			    TemplatedFilterSelection<T, Equals, false>(vdata, predicate, sel, approved_tuple_count, new_sel);
 		} else {
 			approved_tuple_count =
-			    TemplatedFilterSelection<T, Equals, true>(vec, predicate, sel, approved_tuple_count, mask, new_sel);
+			    TemplatedFilterSelection<T, Equals, true>(vdata, predicate, sel, approved_tuple_count, new_sel);
 		}
 		break;
 	}
 	case ExpressionType::COMPARE_NOTEQUAL: {
 		if (mask.AllValid()) {
 			approved_tuple_count =
-			    TemplatedFilterSelection<T, NotEquals, false>(vec, predicate, sel, approved_tuple_count, mask, new_sel);
+			    TemplatedFilterSelection<T, NotEquals, false>(vdata, predicate, sel, approved_tuple_count, new_sel);
 		} else {
 			approved_tuple_count =
-			    TemplatedFilterSelection<T, NotEquals, true>(vec, predicate, sel, approved_tuple_count, mask, new_sel);
+			    TemplatedFilterSelection<T, NotEquals, true>(vdata, predicate, sel, approved_tuple_count, new_sel);
 		}
 		break;
 	}
 	case ExpressionType::COMPARE_LESSTHAN: {
 		if (mask.AllValid()) {
 			approved_tuple_count =
-			    TemplatedFilterSelection<T, LessThan, false>(vec, predicate, sel, approved_tuple_count, mask, new_sel);
+			    TemplatedFilterSelection<T, LessThan, false>(vdata, predicate, sel, approved_tuple_count, new_sel);
 		} else {
 			approved_tuple_count =
-			    TemplatedFilterSelection<T, LessThan, true>(vec, predicate, sel, approved_tuple_count, mask, new_sel);
+			    TemplatedFilterSelection<T, LessThan, true>(vdata, predicate, sel, approved_tuple_count, new_sel);
 		}
 		break;
 	}
 	case ExpressionType::COMPARE_GREATERTHAN: {
 		if (mask.AllValid()) {
-			approved_tuple_count = TemplatedFilterSelection<T, GreaterThan, false>(vec, predicate, sel,
-			                                                                       approved_tuple_count, mask, new_sel);
+			approved_tuple_count =
+			    TemplatedFilterSelection<T, GreaterThan, false>(vdata, predicate, sel, approved_tuple_count, new_sel);
 		} else {
-			approved_tuple_count = TemplatedFilterSelection<T, GreaterThan, true>(vec, predicate, sel,
-			                                                                      approved_tuple_count, mask, new_sel);
+			approved_tuple_count =
+			    TemplatedFilterSelection<T, GreaterThan, true>(vdata, predicate, sel, approved_tuple_count, new_sel);
 		}
 		break;
 	}
 	case ExpressionType::COMPARE_LESSTHANOREQUALTO: {
 		if (mask.AllValid()) {
-			approved_tuple_count = TemplatedFilterSelection<T, LessThanEquals, false>(
-			    vec, predicate, sel, approved_tuple_count, mask, new_sel);
+			approved_tuple_count = TemplatedFilterSelection<T, LessThanEquals, false>(vdata, predicate, sel,
+			                                                                          approved_tuple_count, new_sel);
 		} else {
-			approved_tuple_count = TemplatedFilterSelection<T, LessThanEquals, true>(
-			    vec, predicate, sel, approved_tuple_count, mask, new_sel);
+			approved_tuple_count =
+			    TemplatedFilterSelection<T, LessThanEquals, true>(vdata, predicate, sel, approved_tuple_count, new_sel);
 		}
 		break;
 	}
 	case ExpressionType::COMPARE_GREATERTHANOREQUALTO: {
 		if (mask.AllValid()) {
-			approved_tuple_count = TemplatedFilterSelection<T, GreaterThanEquals, false>(
-			    vec, predicate, sel, approved_tuple_count, mask, new_sel);
+			approved_tuple_count = TemplatedFilterSelection<T, GreaterThanEquals, false>(vdata, predicate, sel,
+			                                                                             approved_tuple_count, new_sel);
 		} else {
-			approved_tuple_count = TemplatedFilterSelection<T, GreaterThanEquals, true>(
-			    vec, predicate, sel, approved_tuple_count, mask, new_sel);
+			approved_tuple_count = TemplatedFilterSelection<T, GreaterThanEquals, true>(vdata, predicate, sel,
+			                                                                            approved_tuple_count, new_sel);
 		}
 		break;
 	}
@@ -329,7 +333,8 @@ static void FilterSelectionSwitch(T *vec, T predicate, SelectionVector &sel, idx
 }
 
 template <bool IS_NULL>
-static idx_t TemplatedNullSelection(SelectionVector &sel, idx_t &approved_tuple_count, ValidityMask &mask) {
+static idx_t TemplatedNullSelection(UnifiedVectorFormat &vdata, SelectionVector &sel, idx_t &approved_tuple_count) {
+	auto &mask = vdata.validity;
 	if (mask.AllValid()) {
 		// no NULL values
 		if (IS_NULL) {
@@ -343,7 +348,8 @@ static idx_t TemplatedNullSelection(SelectionVector &sel, idx_t &approved_tuple_
 		idx_t result_count = 0;
 		for (idx_t i = 0; i < approved_tuple_count; i++) {
 			auto idx = sel.get_index(i);
-			if (mask.RowIsValid(idx) != IS_NULL) {
+			auto vector_idx = vdata.sel->get_index(idx);
+			if (mask.RowIsValid(vector_idx) != IS_NULL) {
 				result_sel.set_index(result_count++, idx);
 			}
 		}
@@ -353,8 +359,8 @@ static idx_t TemplatedNullSelection(SelectionVector &sel, idx_t &approved_tuple_
 	}
 }
 
-idx_t ColumnSegment::FilterSelection(SelectionVector &sel, Vector &result, const TableFilter &filter,
-                                     idx_t &approved_tuple_count, ValidityMask &mask) {
+idx_t ColumnSegment::FilterSelection(SelectionVector &sel, Vector &vector, UnifiedVectorFormat &vdata,
+                                     const TableFilter &filter, idx_t scan_count, idx_t &approved_tuple_count) {
 	switch (filter.filter_type) {
 	case TableFilterType::CONJUNCTION_OR: {
 		// similar to the CONJUNCTION_AND, but we need to take care of the SelectionVectors (OR all of them)
@@ -365,7 +371,7 @@ idx_t ColumnSegment::FilterSelection(SelectionVector &sel, Vector &result, const
 			SelectionVector temp_sel;
 			temp_sel.Initialize(sel);
 			idx_t temp_tuple_count = approved_tuple_count;
-			idx_t temp_count = FilterSelection(temp_sel, result, *child_filter, temp_tuple_count, mask);
+			idx_t temp_count = FilterSelection(temp_sel, vector, vdata, *child_filter, scan_count, temp_tuple_count);
 			// tuples passed, move them into the actual result vector
 			for (idx_t i = 0; i < temp_count; i++) {
 				auto new_idx = temp_sel.get_index(i);
@@ -388,127 +394,111 @@ idx_t ColumnSegment::FilterSelection(SelectionVector &sel, Vector &result, const
 	case TableFilterType::CONJUNCTION_AND: {
 		auto &conjunction_and = filter.Cast<ConjunctionAndFilter>();
 		for (auto &child_filter : conjunction_and.child_filters) {
-			FilterSelection(sel, result, *child_filter, approved_tuple_count, mask);
+			FilterSelection(sel, vector, vdata, *child_filter, scan_count, approved_tuple_count);
 		}
 		return approved_tuple_count;
 	}
 	case TableFilterType::CONSTANT_COMPARISON: {
 		auto &constant_filter = filter.Cast<ConstantFilter>();
 		// the inplace loops take the result as the last parameter
-		switch (result.GetType().InternalType()) {
+		switch (vector.GetType().InternalType()) {
 		case PhysicalType::UINT8: {
-			auto result_flat = FlatVector::GetData<uint8_t>(result);
 			auto predicate = UTinyIntValue::Get(constant_filter.constant);
-			FilterSelectionSwitch<uint8_t>(result_flat, predicate, sel, approved_tuple_count,
-			                               constant_filter.comparison_type, mask);
+			FilterSelectionSwitch<uint8_t>(vdata, predicate, sel, approved_tuple_count,
+			                               constant_filter.comparison_type);
 			break;
 		}
 		case PhysicalType::UINT16: {
-			auto result_flat = FlatVector::GetData<uint16_t>(result);
 			auto predicate = USmallIntValue::Get(constant_filter.constant);
-			FilterSelectionSwitch<uint16_t>(result_flat, predicate, sel, approved_tuple_count,
-			                                constant_filter.comparison_type, mask);
+			FilterSelectionSwitch<uint16_t>(vdata, predicate, sel, approved_tuple_count,
+			                                constant_filter.comparison_type);
 			break;
 		}
 		case PhysicalType::UINT32: {
-			auto result_flat = FlatVector::GetData<uint32_t>(result);
 			auto predicate = UIntegerValue::Get(constant_filter.constant);
-			FilterSelectionSwitch<uint32_t>(result_flat, predicate, sel, approved_tuple_count,
-			                                constant_filter.comparison_type, mask);
+			FilterSelectionSwitch<uint32_t>(vdata, predicate, sel, approved_tuple_count,
+			                                constant_filter.comparison_type);
 			break;
 		}
 		case PhysicalType::UINT64: {
-			auto result_flat = FlatVector::GetData<uint64_t>(result);
 			auto predicate = UBigIntValue::Get(constant_filter.constant);
-			FilterSelectionSwitch<uint64_t>(result_flat, predicate, sel, approved_tuple_count,
-			                                constant_filter.comparison_type, mask);
+			FilterSelectionSwitch<uint64_t>(vdata, predicate, sel, approved_tuple_count,
+			                                constant_filter.comparison_type);
 			break;
 		}
 		case PhysicalType::INT8: {
-			auto result_flat = FlatVector::GetData<int8_t>(result);
 			auto predicate = TinyIntValue::Get(constant_filter.constant);
-			FilterSelectionSwitch<int8_t>(result_flat, predicate, sel, approved_tuple_count,
-			                              constant_filter.comparison_type, mask);
+			FilterSelectionSwitch<int8_t>(vdata, predicate, sel, approved_tuple_count, constant_filter.comparison_type);
 			break;
 		}
 		case PhysicalType::INT16: {
-			auto result_flat = FlatVector::GetData<int16_t>(result);
 			auto predicate = SmallIntValue::Get(constant_filter.constant);
-			FilterSelectionSwitch<int16_t>(result_flat, predicate, sel, approved_tuple_count,
-			                               constant_filter.comparison_type, mask);
+			FilterSelectionSwitch<int16_t>(vdata, predicate, sel, approved_tuple_count,
+			                               constant_filter.comparison_type);
 			break;
 		}
 		case PhysicalType::INT32: {
-			auto result_flat = FlatVector::GetData<int32_t>(result);
 			auto predicate = IntegerValue::Get(constant_filter.constant);
-			FilterSelectionSwitch<int32_t>(result_flat, predicate, sel, approved_tuple_count,
-			                               constant_filter.comparison_type, mask);
+			FilterSelectionSwitch<int32_t>(vdata, predicate, sel, approved_tuple_count,
+			                               constant_filter.comparison_type);
 			break;
 		}
 		case PhysicalType::INT64: {
-			auto result_flat = FlatVector::GetData<int64_t>(result);
 			auto predicate = BigIntValue::Get(constant_filter.constant);
-			FilterSelectionSwitch<int64_t>(result_flat, predicate, sel, approved_tuple_count,
-			                               constant_filter.comparison_type, mask);
+			FilterSelectionSwitch<int64_t>(vdata, predicate, sel, approved_tuple_count,
+			                               constant_filter.comparison_type);
 			break;
 		}
 		case PhysicalType::INT128: {
-			auto result_flat = FlatVector::GetData<hugeint_t>(result);
 			auto predicate = HugeIntValue::Get(constant_filter.constant);
-			FilterSelectionSwitch<hugeint_t>(result_flat, predicate, sel, approved_tuple_count,
-			                                 constant_filter.comparison_type, mask);
+			FilterSelectionSwitch<hugeint_t>(vdata, predicate, sel, approved_tuple_count,
+			                                 constant_filter.comparison_type);
 			break;
 		}
 		case PhysicalType::UINT128: {
-			auto result_flat = FlatVector::GetData<uhugeint_t>(result);
 			auto predicate = UhugeIntValue::Get(constant_filter.constant);
-			FilterSelectionSwitch<uhugeint_t>(result_flat, predicate, sel, approved_tuple_count,
-			                                  constant_filter.comparison_type, mask);
+			FilterSelectionSwitch<uhugeint_t>(vdata, predicate, sel, approved_tuple_count,
+			                                  constant_filter.comparison_type);
 			break;
 		}
 		case PhysicalType::FLOAT: {
-			auto result_flat = FlatVector::GetData<float>(result);
 			auto predicate = FloatValue::Get(constant_filter.constant);
-			FilterSelectionSwitch<float>(result_flat, predicate, sel, approved_tuple_count,
-			                             constant_filter.comparison_type, mask);
+			FilterSelectionSwitch<float>(vdata, predicate, sel, approved_tuple_count, constant_filter.comparison_type);
 			break;
 		}
 		case PhysicalType::DOUBLE: {
-			auto result_flat = FlatVector::GetData<double>(result);
 			auto predicate = DoubleValue::Get(constant_filter.constant);
-			FilterSelectionSwitch<double>(result_flat, predicate, sel, approved_tuple_count,
-			                              constant_filter.comparison_type, mask);
+			FilterSelectionSwitch<double>(vdata, predicate, sel, approved_tuple_count, constant_filter.comparison_type);
 			break;
 		}
 		case PhysicalType::VARCHAR: {
-			auto result_flat = FlatVector::GetData<string_t>(result);
 			auto predicate = string_t(StringValue::Get(constant_filter.constant));
-			FilterSelectionSwitch<string_t>(result_flat, predicate, sel, approved_tuple_count,
-			                                constant_filter.comparison_type, mask);
+			FilterSelectionSwitch<string_t>(vdata, predicate, sel, approved_tuple_count,
+			                                constant_filter.comparison_type);
 			break;
 		}
 		case PhysicalType::BOOL: {
-			auto result_flat = FlatVector::GetData<bool>(result);
 			auto predicate = BooleanValue::Get(constant_filter.constant);
-			FilterSelectionSwitch<bool>(result_flat, predicate, sel, approved_tuple_count,
-			                            constant_filter.comparison_type, mask);
+			FilterSelectionSwitch<bool>(vdata, predicate, sel, approved_tuple_count, constant_filter.comparison_type);
 			break;
 		}
 		default:
-			throw InvalidTypeException(result.GetType(), "Invalid type for filter pushed down to table comparison");
+			throw InvalidTypeException(vector.GetType(), "Invalid type for filter pushed down to table comparison");
 		}
 		return approved_tuple_count;
 	}
 	case TableFilterType::IS_NULL:
-		return TemplatedNullSelection<true>(sel, approved_tuple_count, mask);
+		return TemplatedNullSelection<true>(vdata, sel, approved_tuple_count);
 	case TableFilterType::IS_NOT_NULL:
-		return TemplatedNullSelection<false>(sel, approved_tuple_count, mask);
+		return TemplatedNullSelection<false>(vdata, sel, approved_tuple_count);
 	case TableFilterType::STRUCT_EXTRACT: {
 		auto &struct_filter = filter.Cast<StructFilter>();
 		// Apply the filter on the child vector
-		auto &child_vec = StructVector::GetEntries(result)[struct_filter.child_idx];
-		auto &child_mask = FlatVector::Validity(*child_vec);
-		return FilterSelection(sel, *child_vec, *struct_filter.child_filter, approved_tuple_count, child_mask);
+		auto &child_vec = StructVector::GetEntries(vector)[struct_filter.child_idx];
+		UnifiedVectorFormat child_data;
+		child_vec->ToUnifiedFormat(scan_count, child_data);
+		return FilterSelection(sel, *child_vec, child_data, *struct_filter.child_filter, scan_count,
+		                       approved_tuple_count);
 	}
 	default:
 		throw InternalException("FIXME: unsupported type for filter selection");

package/src/duckdb/src/storage/table/list_column_data.cpp

@@ -303,6 +303,7 @@ void ListColumnData::FetchRow(TransactionData transaction, ColumnFetchState &sta
 }
 
 void ListColumnData::CommitDropColumn() {
+	ColumnData::CommitDropColumn();
 	validity.CommitDropColumn();
 	child_column->CommitDropColumn();
 }

package/src/duckdb/src/storage/table/row_group.cpp

@@ -26,12 +26,12 @@
 namespace duckdb {
 
 RowGroup::RowGroup(RowGroupCollection &collection, idx_t start, idx_t count)
-    : SegmentBase<RowGroup>(start, count), collection(collection) {
+    : SegmentBase<RowGroup>(start, count), collection(collection), allocation_size(0) {
 	Verify();
 }
 
 RowGroup::RowGroup(RowGroupCollection &collection, RowGroupPointer &&pointer)
-    : SegmentBase<RowGroup>(pointer.row_start, pointer.tuple_count), collection(collection) {
+    : SegmentBase<RowGroup>(pointer.row_start, pointer.tuple_count), collection(collection), allocation_size(0) {
 	// deserialize the columns
 	if (pointer.data_pointers.size() != collection.GetTypes().size()) {
 		throw IOException("Row group column count is unaligned with table column count. Corrupt file?");
@@ -184,12 +184,17 @@ bool RowGroup::InitializeScanWithOffset(CollectionScanState &state, idx_t vector
 	state.vector_index = vector_offset;
 	state.max_row_group_row =
 	    this->start > state.max_row ? 0 : MinValue<idx_t>(this->count, state.max_row - this->start);
+	auto row_number = start + vector_offset * STANDARD_VECTOR_SIZE;
+	if (state.max_row_group_row == 0) {
+		// exceeded row groups to scan
+		return false;
+	}
 	D_ASSERT(state.column_scans);
 	for (idx_t i = 0; i < column_ids.size(); i++) {
 		const auto &column = column_ids[i];
 		if (column != COLUMN_IDENTIFIER_ROW_ID) {
 			auto &column_data = GetColumn(column);
-			column_data.InitializeScanWithOffset(state.column_scans[i], start + vector_offset * STANDARD_VECTOR_SIZE);
+			column_data.InitializeScanWithOffset(state.column_scans[i], row_number);
 			state.column_scans[i].scan_options = &state.GetOptions();
 		} else {
 			state.column_scans[i].current = nullptr;
@@ -660,16 +665,19 @@ void RowGroup::FetchRow(TransactionData transaction, ColumnFetchState &state, co
                         row_t row_id, DataChunk &result, idx_t result_idx) {
 	for (idx_t col_idx = 0; col_idx < column_ids.size(); col_idx++) {
 		auto column = column_ids[col_idx];
+		auto &result_vector = result.data[col_idx];
+		D_ASSERT(result_vector.GetVectorType() == VectorType::FLAT_VECTOR);
+		D_ASSERT(!FlatVector::IsNull(result_vector, result_idx));
 		if (column == COLUMN_IDENTIFIER_ROW_ID) {
 			// row id column: fill in the row ids
-			D_ASSERT(result.data[col_idx].GetType().InternalType() == PhysicalType::INT64);
-			result.data[col_idx].SetVectorType(VectorType::FLAT_VECTOR);
-			auto data = FlatVector::GetData<row_t>(result.data[col_idx]);
+			D_ASSERT(result_vector.GetType().InternalType() == PhysicalType::INT64);
+			result_vector.SetVectorType(VectorType::FLAT_VECTOR);
+			auto data = FlatVector::GetData<row_t>(result_vector);
 			data[result_idx] = row_id;
 		} else {
 			// regular column: fetch data from the base column
 			auto &col_data = GetColumn(column);
-			col_data.FetchRow(transaction, state, row_id, result.data[col_idx], result_idx);
+			col_data.FetchRow(transaction, state, row_id, result_vector, result_idx);
 		}
 	}
 }
@@ -717,7 +725,9 @@ void RowGroup::Append(RowGroupAppendState &state, DataChunk &chunk, idx_t append
 	D_ASSERT(chunk.ColumnCount() == GetColumnCount());
 	for (idx_t i = 0; i < GetColumnCount(); i++) {
 		auto &col_data = GetColumn(i);
+		auto prev_allocation_size = col_data.GetAllocationSize();
 		col_data.Append(state.states[i], chunk.data[i], append_count);
+		allocation_size += col_data.GetAllocationSize() - prev_allocation_size;
 	}
 	state.offset_in_row_group += append_count;
 }

package/src/duckdb/src/storage/table/row_group_collection.cpp

@@ -54,7 +54,7 @@ unique_ptr<RowGroup> RowGroupSegmentTree::LoadSegment() {
 RowGroupCollection::RowGroupCollection(shared_ptr<DataTableInfo> info_p, BlockManager &block_manager,
                                        vector<LogicalType> types_p, idx_t row_start_p, idx_t total_rows_p)
     : block_manager(block_manager), total_rows(total_rows_p), info(std::move(info_p)), types(std::move(types_p)),
-      row_start(row_start_p) {
+      row_start(row_start_p), allocation_size(0) {
 	row_groups = make_shared<RowGroupSegmentTree>(*this);
 }
 
@@ -217,6 +217,8 @@ bool RowGroupCollection::NextParallelScan(ClientContext &context, ParallelCollec
 		}
 		return true;
 	}
+	lock_guard<mutex> l(state.lock);
+	scan_state.batch_index = state.batch_index;
 	return false;
 }
 
@@ -288,11 +290,10 @@ void RowGroupCollection::Fetch(TransactionData transaction, DataChunk &result, c
 // Append
 //===--------------------------------------------------------------------===//
 TableAppendState::TableAppendState()
-    : row_group_append_state(*this), total_append_count(0), start_row_group(nullptr), transaction(0, 0), remaining(0) {
+    : row_group_append_state(*this), total_append_count(0), start_row_group(nullptr), transaction(0, 0) {
 }
 
 TableAppendState::~TableAppendState() {
-	D_ASSERT(Exception::UncaughtException() || remaining == 0);
 }
 
 bool RowGroupCollection::IsEmpty() const {
@@ -304,7 +305,7 @@ bool RowGroupCollection::IsEmpty(SegmentLock &l) const {
 	return row_groups->IsEmpty(l);
 }
 
-void RowGroupCollection::InitializeAppend(TransactionData transaction, TableAppendState &state, idx_t append_count) {
+void RowGroupCollection::InitializeAppend(TransactionData transaction, TableAppendState &state) {
 	state.row_start = total_rows;
 	state.current_row = state.row_start;
 	state.total_append_count = 0;
@@ -318,17 +319,12 @@ void RowGroupCollection::InitializeAppend(TransactionData transaction, TableAppe
 	state.start_row_group = row_groups->GetLastSegment(l);
 	D_ASSERT(this->row_start + total_rows == state.start_row_group->start + state.start_row_group->count);
 	state.start_row_group->InitializeAppend(state.row_group_append_state);
-	state.remaining = append_count;
 	state.transaction = transaction;
-	if (state.remaining > 0) {
-		state.start_row_group->AppendVersionInfo(transaction, state.remaining);
-		total_rows += state.remaining;
-	}
 }
 
 void RowGroupCollection::InitializeAppend(TableAppendState &state) {
 	TransactionData tdata(0, 0);
-	InitializeAppend(tdata, state, 0);
+	InitializeAppend(tdata, state);
 }
 
 bool RowGroupCollection::Append(DataChunk &chunk, TableAppendState &state) {
@@ -336,16 +332,18 @@ bool RowGroupCollection::Append(DataChunk &chunk, TableAppendState &state) {
 	chunk.Verify();
 
 	bool new_row_group = false;
-	idx_t append_count = chunk.size();
+	idx_t total_append_count = chunk.size();
 	idx_t remaining = chunk.size();
-	state.total_append_count += append_count;
+	state.total_append_count += total_append_count;
 	while (true) {
 		auto current_row_group = state.row_group_append_state.row_group;
 		// check how much we can fit into the current row_group
 		idx_t append_count =
 		    MinValue<idx_t>(remaining, Storage::ROW_GROUP_SIZE - state.row_group_append_state.offset_in_row_group);
 		if (append_count > 0) {
+			auto previous_allocation_size = current_row_group->GetAllocationSize();
 			current_row_group->Append(state.row_group_append_state, chunk, append_count);
+			allocation_size += current_row_group->GetAllocationSize() - previous_allocation_size;
 			// merge the stats
 			auto stats_lock = stats.GetLock();
 			for (idx_t i = 0; i < types.size(); i++) {
@@ -353,9 +351,6 @@ bool RowGroupCollection::Append(DataChunk &chunk, TableAppendState &state) {
 			}
 		}
 		remaining -= append_count;
-		if (state.remaining > 0) {
-			state.remaining -= append_count;
-		}
 		if (remaining > 0) {
 			// we expect max 1 iteration of this loop (i.e. a single chunk should never overflow more than one
 			// row_group)
@@ -373,15 +368,12 @@ bool RowGroupCollection::Append(DataChunk &chunk, TableAppendState &state) {
 			// set up the append state for this row_group
 			auto last_row_group = row_groups->GetLastSegment(l);
 			last_row_group->InitializeAppend(state.row_group_append_state);
-			if (state.remaining > 0) {
-				last_row_group->AppendVersionInfo(state.transaction, state.remaining);
-			}
 			continue;
 		} else {
 			break;
 		}
 	}
-	state.current_row += append_count;
+	state.current_row += row_t(total_append_count);
 	auto stats_lock = stats.GetLock();
 	for (idx_t col_idx = 0; col_idx < types.size(); col_idx++) {
 		stats.GetStats(col_idx).UpdateDistinctStatistics(chunk.data[col_idx], chunk.size());
@@ -427,14 +419,20 @@ void RowGroupCollection::CommitAppend(transaction_t commit_id, idx_t row_start,
 }
 
 void RowGroupCollection::RevertAppendInternal(idx_t start_row) {
-	if (total_rows <= start_row) {
-		return;
-	}
 	total_rows = start_row;
 
 	auto l = row_groups->Lock();
-	// find the segment index that the current row belongs to
-	idx_t segment_index = row_groups->GetSegmentIndex(l, start_row);
+	idx_t segment_count = row_groups->GetSegmentCount(l);
+	if (segment_count == 0) {
+		// we have no segments to revert
+		return;
+	}
+	idx_t segment_index;
+	// find the segment index that the start row belongs to
+	if (!row_groups->TryGetSegmentIndex(l, start_row, segment_index)) {
+		// revert from the last segment
+		segment_index = segment_count - 1;
+	}
 	auto &segment = *row_groups->GetSegmentByIndex(l, segment_index);
 
 	// remove any segments AFTER this segment: they should be deleted entirely
@@ -776,11 +774,14 @@ public:
 		scan_state.Initialize(column_ids);
 		scan_state.table_state.Initialize(types);
 		scan_state.table_state.max_row = idx_t(-1);
-		idx_t next_idx = segment_idx + merge_count;
-		for (idx_t c_idx = segment_idx; c_idx < next_idx; c_idx++) {
+		idx_t merged_groups = 0;
+		idx_t total_row_groups = vacuum_state.row_group_counts.size();
+		for (idx_t c_idx = segment_idx; merged_groups < merge_count && c_idx < total_row_groups; c_idx++) {
 			if (vacuum_state.row_group_counts[c_idx] == 0) {
 				continue;
 			}
+			merged_groups++;
+
 			auto &current_row_group = *checkpoint_state.segments[c_idx].node;
 
 			current_row_group.InitializeScan(scan_state.table_state);
@@ -920,7 +921,7 @@ bool RowGroupCollection::ScheduleVacuumTasks(CollectionCheckpointState &checkpoi
 	                                         merge_rows, state.row_start);
 	checkpoint_state.ScheduleTask(std::move(vacuum_task));
 	// skip vacuuming by the row groups we have merged
-	state.next_vacuum_idx = segment_idx + merge_count;
+	state.next_vacuum_idx = next_idx;
 	state.row_start += merge_rows;
 	return true;
 }

package/src/duckdb/src/storage/table/ub_duckdb_storage_table.cpp

@@ -0,0 +1,17 @@
+// Unity Build generated by CMake
+#include </Users/carlo/duckdb/src/storage/table/chunk_info.cpp>
+#include </Users/carlo/duckdb/src/storage/table/column_checkpoint_state.cpp>
+#include </Users/carlo/duckdb/src/storage/table/column_data_checkpointer.cpp>
+#include </Users/carlo/duckdb/src/storage/table/column_data.cpp>
+#include </Users/carlo/duckdb/src/storage/table/column_segment.cpp>
+#include </Users/carlo/duckdb/src/storage/table/list_column_data.cpp>
+#include </Users/carlo/duckdb/src/storage/table/update_segment.cpp>
+#include </Users/carlo/duckdb/src/storage/table/persistent_table_data.cpp>
+#include </Users/carlo/duckdb/src/storage/table/row_group.cpp>
+#include </Users/carlo/duckdb/src/storage/table/row_group_collection.cpp>
+#include </Users/carlo/duckdb/src/storage/table/row_version_manager.cpp>
+#include </Users/carlo/duckdb/src/storage/table/scan_state.cpp>
+#include </Users/carlo/duckdb/src/storage/table/standard_column_data.cpp>
+#include </Users/carlo/duckdb/src/storage/table/struct_column_data.cpp>
+#include </Users/carlo/duckdb/src/storage/table/table_statistics.cpp>
+#include </Users/carlo/duckdb/src/storage/table/validity_column_data.cpp>

package/src/duckdb/src/storage/table/update_segment.cpp

@@ -574,12 +574,12 @@ void UpdateSegment::InitializeUpdateInfo(UpdateInfo &info, row_t *ids, const Sel
 	info.next = nullptr;
 
 	// set up the tuple ids
-	info.N = count;
+	info.N = UnsafeNumericCast<sel_t>(count);
 	for (idx_t i = 0; i < count; i++) {
 		auto idx = sel.get_index(i);
 		auto id = ids[idx];
 		D_ASSERT(idx_t(id) >= vector_offset && idx_t(id) < vector_offset + STANDARD_VECTOR_SIZE);
-		info.tuples[i] = id - vector_offset;
+		info.tuples[i] = NumericCast<sel_t>(id - vector_offset);
 	};
 }
 
@@ -804,7 +804,7 @@ static void MergeUpdateLoopInternal(UpdateInfo *base_info, V *base_table_data, U
 			result_values[result_offset] = UpdateSelectElement::Operation<T>(
 			    base_info->segment, OP::template Extract<T, V>(base_table_data, update_id));
 		}
-		result_ids[result_offset++] = update_id;
+		result_ids[result_offset++] = UnsafeNumericCast<sel_t>(update_id);
 	}
 	// write any remaining entries from the old updates
 	while (update_info_offset < update_info->N) {
@@ -813,7 +813,7 @@ static void MergeUpdateLoopInternal(UpdateInfo *base_info, V *base_table_data, U
 		update_info_offset++;
 	}
 	// now copy them back
-	update_info->N = result_offset;
+	update_info->N = UnsafeNumericCast<sel_t>(result_offset);
 	memcpy(update_info_data, result_values, result_offset * sizeof(T));
 	memcpy(update_info->tuples, result_ids, result_offset * sizeof(sel_t));
 
@@ -821,12 +821,12 @@ static void MergeUpdateLoopInternal(UpdateInfo *base_info, V *base_table_data, U
 	result_offset = 0;
 	auto pick_new = [&](idx_t id, idx_t aidx, idx_t count) {
 		result_values[result_offset] = OP::template Extract<T, V>(update_vector_data, aidx);
-		result_ids[result_offset] = id;
+		result_ids[result_offset] = UnsafeNumericCast<sel_t>(id);
 		result_offset++;
 	};
 	auto pick_old = [&](idx_t id, idx_t bidx, idx_t count) {
 		result_values[result_offset] = base_info_data[bidx];
-		result_ids[result_offset] = id;
+		result_ids[result_offset] = UnsafeNumericCast<sel_t>(id);
 		result_offset++;
 	};
 	// now we perform a merge of the new ids with the old ids
@@ -835,7 +835,7 @@ static void MergeUpdateLoopInternal(UpdateInfo *base_info, V *base_table_data, U
 	};
 	MergeLoop(ids, base_info->tuples, count, base_info->N, base_id, merge, pick_new, pick_old, sel);
 
-	base_info->N = result_offset;
+	base_info->N = UnsafeNumericCast<sel_t>(result_offset);
 	memcpy(base_info_data, result_values, result_offset * sizeof(T));
 	memcpy(base_info->tuples, result_ids, result_offset * sizeof(sel_t));
 }