duckdb 1.0.1-dev22.0 → 1.0.1-dev27.0

package/src/duckdb/src/common/types/row/tuple_data_scatter_gather.cpp

@@ -1,5 +1,6 @@
 #include "duckdb/common/enum_util.hpp"
 #include "duckdb/common/fast_mem.hpp"
+#include "duckdb/common/type_visitor.hpp"
 #include "duckdb/common/types/null_value.hpp"
 #include "duckdb/common/types/row/tuple_data_collection.hpp"
 #include "duckdb/common/uhugeint.hpp"
@@ -19,8 +20,8 @@ constexpr idx_t TupleDataWithinListFixedSize<string_t>() {
 }
 
 template <class T>
-static inline void TupleDataValueStore(const T &source, const data_ptr_t &row_location, const idx_t offset_in_row,
-                                       data_ptr_t &heap_location) {
+static void TupleDataValueStore(const T &source, const data_ptr_t &row_location, const idx_t offset_in_row,
+                                data_ptr_t &) {
 	Store<T>(source, row_location + offset_in_row);
 }
 
@@ -33,7 +34,7 @@ inline void TupleDataValueStore(const string_t &source, const data_ptr_t &row_lo
 	if (source.IsInlined()) {
 		Store<string_t>(source, row_location + offset_in_row);
 	} else {
-		memcpy(heap_location, source.GetData(), source.GetSize());
+		FastMemcpy(heap_location, source.GetData(), source.GetSize());
 		Store<string_t>(string_t(const_char_ptr_cast(heap_location), UnsafeNumericCast<uint32_t>(source.GetSize())),
 		                row_location + offset_in_row);
 		heap_location += source.GetSize();
@@ -41,8 +42,7 @@ inline void TupleDataValueStore(const string_t &source, const data_ptr_t &row_lo
 }
 
 template <class T>
-static inline void TupleDataWithinListValueStore(const T &source, const data_ptr_t &location,
-                                                 data_ptr_t &heap_location) {
+static void TupleDataWithinListValueStore(const T &source, const data_ptr_t &location, data_ptr_t &) {
 	Store<T>(source, location);
 }
 
@@ -52,13 +52,13 @@ inline void TupleDataWithinListValueStore(const string_t &source, const data_ptr
 #ifdef DEBUG
 	source.VerifyCharacters();
 #endif
-	Store<uint32_t>(NumericCast<uint32_t>(source.GetSize()), location);
-	memcpy(heap_location, source.GetData(), source.GetSize());
+	Store<uint32_t>(UnsafeNumericCast<uint32_t>(source.GetSize()), location);
+	FastMemcpy(heap_location, source.GetData(), source.GetSize());
 	heap_location += source.GetSize();
 }
 
 template <class T>
-inline void TupleDataValueVerify(const LogicalType &type, const T &value) {
+void TupleDataValueVerify(const LogicalType &, const T &) {
 #ifdef DEBUG
 	// NOP
 #endif
@@ -74,7 +74,7 @@ inline void TupleDataValueVerify(const LogicalType &type, const string_t &value)
 }
 
 template <class T>
-static inline T TupleDataWithinListValueLoad(const data_ptr_t &location, data_ptr_t &heap_location) {
+static T TupleDataWithinListValueLoad(const data_ptr_t &location, data_ptr_t &) {
 	return Load<T>(location);
 }
 
@@ -86,7 +86,7 @@ inline string_t TupleDataWithinListValueLoad(const data_ptr_t &location, data_pt
 	return result;
 }
 
-static inline void ResetCombinedListData(vector<TupleDataVectorFormat> &vector_data) {
+static void ResetCombinedListData(vector<TupleDataVectorFormat> &vector_data) {
 #ifdef DEBUG
 	for (auto &vd : vector_data) {
 		vd.combined_list_data = nullptr;
@@ -100,17 +100,16 @@ void TupleDataCollection::ComputeHeapSizes(TupleDataChunkState &chunk_state, con
 	ResetCombinedListData(chunk_state.vector_data);
 
 	auto heap_sizes = FlatVector::GetData<idx_t>(chunk_state.heap_sizes);
-	std::fill_n(heap_sizes, new_chunk.size(), 0);
+	std::fill_n(heap_sizes, append_count, 0);
 
 	for (idx_t col_idx = 0; col_idx < new_chunk.ColumnCount(); col_idx++) {
 		auto &source_v = new_chunk.data[col_idx];
 		auto &source_format = chunk_state.vector_data[col_idx];
-		TupleDataCollection::ComputeHeapSizes(chunk_state.heap_sizes, source_v, source_format, append_sel,
-		                                      append_count);
+		ComputeHeapSizes(chunk_state.heap_sizes, source_v, source_format, append_sel, append_count);
 	}
 }
 
-static inline idx_t StringHeapSize(const string_t &val) {
+static idx_t StringHeapSize(const string_t &val) {
 	return val.IsInlined() ? 0 : val.GetSize();
 }
 
@@ -150,8 +149,7 @@ void TupleDataCollection::ComputeHeapSizes(Vector &heap_sizes_v, const Vector &s
 		for (idx_t struct_col_idx = 0; struct_col_idx < struct_sources.size(); struct_col_idx++) {
 			const auto &struct_source = struct_sources[struct_col_idx];
 			auto &struct_format = source_format.children[struct_col_idx];
-			TupleDataCollection::ComputeHeapSizes(heap_sizes_v, *struct_source, struct_format, append_sel,
-			                                      append_count);
+			ComputeHeapSizes(heap_sizes_v, *struct_source, struct_format, append_sel, append_count);
 		}
 		break;
 	}
@@ -168,8 +166,8 @@ void TupleDataCollection::ComputeHeapSizes(Vector &heap_sizes_v, const Vector &s
 		D_ASSERT(source_format.children.size() == 1);
 		auto &child_source_v = ListVector::GetEntry(source_v);
 		auto &child_format = source_format.children[0];
-		TupleDataCollection::WithinCollectionComputeHeapSizes(heap_sizes_v, child_source_v, child_format, append_sel,
-		                                                      append_count, source_vector_data);
+		WithinCollectionComputeHeapSizes(heap_sizes_v, child_source_v, child_format, append_sel, append_count,
+		                                 source_vector_data);
 		break;
 	}
 	case PhysicalType::ARRAY: {
@@ -185,8 +183,8 @@ void TupleDataCollection::ComputeHeapSizes(Vector &heap_sizes_v, const Vector &s
 		D_ASSERT(source_format.children.size() == 1);
 		auto &child_source_v = ArrayVector::GetEntry(source_v);
 		auto &child_format = source_format.children[0];
-		TupleDataCollection::WithinCollectionComputeHeapSizes(heap_sizes_v, child_source_v, child_format, append_sel,
-		                                                      append_count, source_vector_data);
+		WithinCollectionComputeHeapSizes(heap_sizes_v, child_source_v, child_format, append_sel, append_count,
+		                                 source_vector_data);
 		break;
 	}
 	default:
@@ -200,26 +198,26 @@ void TupleDataCollection::WithinCollectionComputeHeapSizes(Vector &heap_sizes_v,
                                                            const UnifiedVectorFormat &list_data) {
 	auto type = source_v.GetType().InternalType();
 	if (TypeIsConstantSize(type)) {
-		TupleDataCollection::ComputeFixedWithinCollectionHeapSizes(heap_sizes_v, source_v, source_format, append_sel,
-		                                                           append_count, list_data);
+		ComputeFixedWithinCollectionHeapSizes(heap_sizes_v, source_v, source_format, append_sel, append_count,
+		                                      list_data);
 		return;
 	}
 	switch (type) {
 	case PhysicalType::VARCHAR:
-		TupleDataCollection::StringWithinCollectionComputeHeapSizes(heap_sizes_v, source_v, source_format, append_sel,
-		                                                            append_count, list_data);
+		StringWithinCollectionComputeHeapSizes(heap_sizes_v, source_v, source_format, append_sel, append_count,
+		                                       list_data);
 		break;
 	case PhysicalType::STRUCT:
-		TupleDataCollection::StructWithinCollectionComputeHeapSizes(heap_sizes_v, source_v, source_format, append_sel,
-		                                                            append_count, list_data);
+		StructWithinCollectionComputeHeapSizes(heap_sizes_v, source_v, source_format, append_sel, append_count,
+		                                       list_data);
 		break;
 	case PhysicalType::LIST:
-		TupleDataCollection::CollectionWithinCollectionComputeHeapSizes(heap_sizes_v, source_v, source_format,
-		                                                                append_sel, append_count, list_data);
+		CollectionWithinCollectionComputeHeapSizes(heap_sizes_v, source_v, source_format, append_sel, append_count,
+		                                           list_data);
 		break;
 	case PhysicalType::ARRAY:
-		TupleDataCollection::CollectionWithinCollectionComputeHeapSizes(heap_sizes_v, source_v, source_format,
-		                                                                append_sel, append_count, list_data);
+		CollectionWithinCollectionComputeHeapSizes(heap_sizes_v, source_v, source_format, append_sel, append_count,
+		                                           list_data);
 		break;
 	default:
 		throw NotImplementedException("WithinListHeapComputeSizes for %s", EnumUtil::ToString(source_v.GetType().id()));
@@ -227,7 +225,7 @@ void TupleDataCollection::WithinCollectionComputeHeapSizes(Vector &heap_sizes_v,
 }
 
 void TupleDataCollection::ComputeFixedWithinCollectionHeapSizes(Vector &heap_sizes_v, const Vector &source_v,
-                                                                TupleDataVectorFormat &source_format,
+                                                                TupleDataVectorFormat &,
                                                                 const SelectionVector &append_sel,
                                                                 const idx_t append_count,
                                                                 const UnifiedVectorFormat &list_data) {
@@ -260,7 +258,7 @@ void TupleDataCollection::ComputeFixedWithinCollectionHeapSizes(Vector &heap_siz
 	}
 }
 
-void TupleDataCollection::StringWithinCollectionComputeHeapSizes(Vector &heap_sizes_v, const Vector &source_v,
+void TupleDataCollection::StringWithinCollectionComputeHeapSizes(Vector &heap_sizes_v, const Vector &,
                                                                  TupleDataVectorFormat &source_format,
                                                                  const SelectionVector &append_sel,
                                                                  const idx_t append_count,
@@ -343,8 +341,8 @@ void TupleDataCollection::StructWithinCollectionComputeHeapSizes(Vector &heap_si
 		auto &struct_source = *struct_sources[struct_col_idx];
 
 		auto &struct_format = source_format.children[struct_col_idx];
-		TupleDataCollection::WithinCollectionComputeHeapSizes(heap_sizes_v, struct_source, struct_format, append_sel,
-		                                                      append_count, list_data);
+		WithinCollectionComputeHeapSizes(heap_sizes_v, struct_source, struct_format, append_sel, append_count,
+		                                 list_data);
 	}
 }
 
@@ -507,8 +505,56 @@ void TupleDataCollection::CollectionWithinCollectionComputeHeapSizes(Vector &hea
 	combined_child_list_data.validity.Initialize(combined_validity);
 
 	// Recurse
-	TupleDataCollection::WithinCollectionComputeHeapSizes(heap_sizes_v, child_source, child_format, append_sel,
-	                                                      append_count, combined_child_list_data);
+	WithinCollectionComputeHeapSizes(heap_sizes_v, child_source, child_format, append_sel, append_count,
+	                                 combined_child_list_data);
+}
+
+template <class T>
+static void TemplatedInitializeValidityMask(const data_ptr_t row_locations[], const idx_t append_count) {
+	for (idx_t i = 0; i < append_count; i++) {
+		Store<T>(T(-1), row_locations[i]);
+	}
+}
+
+template <idx_t validity_bytes>
+static void TemplatedInitializeValidityMask(const data_ptr_t row_locations[], const idx_t append_count) {
+	for (idx_t i = 0; i < append_count; i++) {
+		memset(row_locations[i], ~0, validity_bytes);
+	}
+}
+
+static void InitializeValidityMask(const data_ptr_t row_locations[], const idx_t append_count,
+                                   const idx_t validity_bytes) {
+	switch (validity_bytes) {
+	case 1:
+		TemplatedInitializeValidityMask<uint8_t>(row_locations, append_count);
+		break;
+	case 2:
+		TemplatedInitializeValidityMask<uint16_t>(row_locations, append_count);
+		break;
+	case 3:
+		TemplatedInitializeValidityMask<3>(row_locations, append_count);
+		break;
+	case 4:
+		TemplatedInitializeValidityMask<uint32_t>(row_locations, append_count);
+		break;
+	case 5:
+		TemplatedInitializeValidityMask<5>(row_locations, append_count);
+		break;
+	case 6:
+		TemplatedInitializeValidityMask<6>(row_locations, append_count);
+		break;
+	case 7:
+		TemplatedInitializeValidityMask<7>(row_locations, append_count);
+		break;
+	case 8:
+		TemplatedInitializeValidityMask<uint64_t>(row_locations, append_count);
+		break;
+	default:
+		for (idx_t i = 0; i < append_count; i++) {
+			FastMemset(row_locations[i], ~0, validity_bytes);
+		}
+	}
 }
 
 void TupleDataCollection::Scatter(TupleDataChunkState &chunk_state, const DataChunk &new_chunk,
@@ -516,24 +562,25 @@ void TupleDataCollection::Scatter(TupleDataChunkState &chunk_state, const DataCh
 #ifdef DEBUG
 	Vector heap_locations_copy(LogicalType::POINTER);
 	if (!layout.AllConstant()) {
-		VectorOperations::Copy(chunk_state.heap_locations, heap_locations_copy, append_count, 0, 0);
+		const auto heap_locations = FlatVector::GetData<data_ptr_t>(chunk_state.heap_locations);
+		const auto copied_heap_locations = FlatVector::GetData<data_ptr_t>(heap_locations_copy);
+		for (idx_t i = 0; i < append_count; i++) {
+			copied_heap_locations[i] = heap_locations[i];
+		}
 	}
 #endif
 
 	const auto row_locations = FlatVector::GetData<data_ptr_t>(chunk_state.row_locations);
 
 	// Set the validity mask for each row before inserting data
-	const auto validity_bytes = ValidityBytes::SizeInBytes(layout.ColumnCount());
-	for (idx_t i = 0; i < append_count; i++) {
-		FastMemset(row_locations[i], ~0, validity_bytes);
-	}
+	InitializeValidityMask(row_locations, append_count, ValidityBytes::SizeInBytes(layout.ColumnCount()));
 
 	if (!layout.AllConstant()) {
 		// Set the heap size for each row
 		const auto heap_size_offset = layout.GetHeapSizeOffset();
 		const auto heap_sizes = FlatVector::GetData<idx_t>(chunk_state.heap_sizes);
 		for (idx_t i = 0; i < append_count; i++) {
-			Store<uint32_t>(NumericCast<uint32_t>(heap_sizes[i]), row_locations[i] + heap_size_offset);
+			Store<uint32_t>(UnsafeNumericCast<uint32_t>(heap_sizes[i]), row_locations[i] + heap_size_offset);
 		}
 	}
 
@@ -549,7 +596,9 @@ void TupleDataCollection::Scatter(TupleDataChunkState &chunk_state, const DataCh
 		const auto heap_sizes = FlatVector::GetData<idx_t>(chunk_state.heap_sizes);
 		const auto offset_heap_locations = FlatVector::GetData<data_ptr_t>(chunk_state.heap_locations);
 		for (idx_t i = 0; i < append_count; i++) {
-			D_ASSERT(offset_heap_locations[i] == original_heap_locations[i] + heap_sizes[i]);
+			if (heap_sizes[i] != 0) {
+				D_ASSERT(offset_heap_locations[i] == original_heap_locations[i] + heap_sizes[i]);
+			}
 		}
 	}
 #endif
@@ -564,11 +613,11 @@ void TupleDataCollection::Scatter(TupleDataChunkState &chunk_state, const Vector
 }
 
 template <class T>
-static void TupleDataTemplatedScatter(const Vector &source, const TupleDataVectorFormat &source_format,
+static void TupleDataTemplatedScatter(const Vector &, const TupleDataVectorFormat &source_format,
                                       const SelectionVector &append_sel, const idx_t append_count,
                                       const TupleDataLayout &layout, const Vector &row_locations,
-                                      Vector &heap_locations, const idx_t col_idx, const UnifiedVectorFormat &dummy_arg,
-                                      const vector<TupleDataScatterFunction> &child_functions) {
+                                      Vector &heap_locations, const idx_t col_idx, const UnifiedVectorFormat &,
+                                      const vector<TupleDataScatterFunction> &) {
 	// Source
 	const auto &source_data = source_format.unified;
 	const auto &source_sel = *source_data.sel;
@@ -576,8 +625,8 @@ static void TupleDataTemplatedScatter(const Vector &source, const TupleDataVecto
 	const auto &validity = source_data.validity;
 
 	// Target
-	auto target_locations = FlatVector::GetData<data_ptr_t>(row_locations);
-	auto target_heap_locations = FlatVector::GetData<data_ptr_t>(heap_locations);
+	const auto target_locations = FlatVector::GetData<data_ptr_t>(row_locations);
+	const auto target_heap_locations = FlatVector::GetData<data_ptr_t>(heap_locations);
 
 	// Precompute mask indexes
 	idx_t entry_idx;
@@ -614,7 +663,7 @@ static void TupleDataStructScatter(const Vector &source, const TupleDataVectorFo
 	const auto &validity = source_data.validity;
 
 	// Target
-	auto target_locations = FlatVector::GetData<data_ptr_t>(row_locations);
+	const auto target_locations = FlatVector::GetData<data_ptr_t>(row_locations);
 
 	// Precompute mask indexes
 	idx_t entry_idx;
@@ -644,10 +693,8 @@ static void TupleDataStructScatter(const Vector &source, const TupleDataVectorFo
 	D_ASSERT(struct_layout.ColumnCount() == struct_sources.size());
 
 	// Set the validity of the entries within the STRUCTs
-	const auto validity_bytes = ValidityBytes::SizeInBytes(struct_layout.ColumnCount());
-	for (idx_t i = 0; i < append_count; i++) {
-		memset(struct_target_locations[i], ~0, validity_bytes);
-	}
+	InitializeValidityMask(struct_target_locations, append_count,
+	                       ValidityBytes::SizeInBytes(struct_layout.ColumnCount()));
 
 	// Recurse through the struct children
 	for (idx_t struct_col_idx = 0; struct_col_idx < struct_layout.ColumnCount(); struct_col_idx++) {
@@ -666,7 +713,7 @@ static void TupleDataStructScatter(const Vector &source, const TupleDataVectorFo
 static void TupleDataListScatter(const Vector &source, const TupleDataVectorFormat &source_format,
                                  const SelectionVector &append_sel, const idx_t append_count,
                                  const TupleDataLayout &layout, const Vector &row_locations, Vector &heap_locations,
-                                 const idx_t col_idx, const UnifiedVectorFormat &dummy_arg,
+                                 const idx_t col_idx, const UnifiedVectorFormat &,
                                  const vector<TupleDataScatterFunction> &child_functions) {
 	// Source
 	const auto &source_data = source_format.unified;
@@ -675,8 +722,8 @@ static void TupleDataListScatter(const Vector &source, const TupleDataVectorForm
 	const auto &validity = source_data.validity;
 
 	// Target
-	auto target_locations = FlatVector::GetData<data_ptr_t>(row_locations);
-	auto target_heap_locations = FlatVector::GetData<data_ptr_t>(heap_locations);
+	const auto target_locations = FlatVector::GetData<data_ptr_t>(row_locations);
+	const auto target_heap_locations = FlatVector::GetData<data_ptr_t>(heap_locations);
 
 	// Precompute mask indexes
 	idx_t entry_idx;
@@ -714,7 +761,7 @@ static void TupleDataListScatter(const Vector &source, const TupleDataVectorForm
 static void TupleDataArrayScatter(const Vector &source, const TupleDataVectorFormat &source_format,
                                   const SelectionVector &append_sel, const idx_t append_count,
                                   const TupleDataLayout &layout, const Vector &row_locations, Vector &heap_locations,
-                                  const idx_t col_idx, const UnifiedVectorFormat &dummy_arg,
+                                  const idx_t col_idx, const UnifiedVectorFormat &,
                                   const vector<TupleDataScatterFunction> &child_functions) {
 	// Source
 	// The Array vector has fake list_entry_t's set by this point, so this is fine
@@ -724,8 +771,8 @@ static void TupleDataArrayScatter(const Vector &source, const TupleDataVectorFor
 	const auto &validity = source_data.validity;
 
 	// Target
-	auto target_locations = FlatVector::GetData<data_ptr_t>(row_locations);
-	auto target_heap_locations = FlatVector::GetData<data_ptr_t>(heap_locations);
+	const auto target_locations = FlatVector::GetData<data_ptr_t>(row_locations);
+	const auto target_heap_locations = FlatVector::GetData<data_ptr_t>(heap_locations);
 
 	// Precompute mask indexes
 	idx_t entry_idx;
@@ -761,12 +808,11 @@ static void TupleDataArrayScatter(const Vector &source, const TupleDataVectorFor
 // Collection Scatter
 //------------------------------------------------------------------------------
 template <class T>
-static void TupleDataTemplatedWithinCollectionScatter(const Vector &source, const TupleDataVectorFormat &source_format,
+static void TupleDataTemplatedWithinCollectionScatter(const Vector &, const TupleDataVectorFormat &source_format,
                                                       const SelectionVector &append_sel, const idx_t append_count,
-                                                      const TupleDataLayout &layout, const Vector &row_locations,
-                                                      Vector &heap_locations, const idx_t col_idx,
-                                                      const UnifiedVectorFormat &list_data,
-                                                      const vector<TupleDataScatterFunction> &child_functions) {
+                                                      const TupleDataLayout &, const Vector &, Vector &heap_locations,
+                                                      const idx_t, const UnifiedVectorFormat &list_data,
+                                                      const vector<TupleDataScatterFunction> &) {
 	// Parent list data
 	const auto &list_sel = *list_data.sel;
 	const auto list_entries = UnifiedVectorFormat::GetData<list_entry_t>(list_data);
@@ -779,7 +825,7 @@ static void TupleDataTemplatedWithinCollectionScatter(const Vector &source, cons
 	const auto &source_validity = source_data.validity;
 
 	// Target
-	auto target_heap_locations = FlatVector::GetData<data_ptr_t>(heap_locations);
+	const auto target_heap_locations = FlatVector::GetData<data_ptr_t>(heap_locations);
 
 	for (idx_t i = 0; i < append_count; i++) {
 		const auto list_idx = list_sel.get_index(append_sel.get_index(i));
@@ -822,7 +868,7 @@ static void TupleDataTemplatedWithinCollectionScatter(const Vector &source, cons
 static void TupleDataStructWithinCollectionScatter(const Vector &source, const TupleDataVectorFormat &source_format,
                                                    const SelectionVector &append_sel, const idx_t append_count,
                                                    const TupleDataLayout &layout, const Vector &row_locations,
-                                                   Vector &heap_locations, const idx_t col_idx,
+                                                   Vector &heap_locations, const idx_t,
                                                    const UnifiedVectorFormat &list_data,
                                                    const vector<TupleDataScatterFunction> &child_functions) {
 	// Parent list data
@@ -836,7 +882,7 @@ static void TupleDataStructWithinCollectionScatter(const Vector &source, const T
 	const auto &source_validity = source_data.validity;
 
 	// Target
-	auto target_heap_locations = FlatVector::GetData<data_ptr_t>(heap_locations);
+	const auto target_heap_locations = FlatVector::GetData<data_ptr_t>(heap_locations);
 
 	// Initialize the validity of the STRUCTs
 	for (idx_t i = 0; i < append_count; i++) {
@@ -900,7 +946,7 @@ static void TupleDataCollectionWithinCollectionScatter(const Vector &child_list,
 	const auto &child_list_validity = child_list_data.validity;
 
 	// Target
-	auto target_heap_locations = FlatVector::GetData<data_ptr_t>(heap_locations);
+	const auto target_heap_locations = FlatVector::GetData<data_ptr_t>(heap_locations);
 
 	for (idx_t i = 0; i < append_count; i++) {
 		const auto list_idx = list_sel.get_index(append_sel.get_index(i));
@@ -1054,6 +1100,7 @@ void TupleDataCollection::Gather(Vector &row_locations, const SelectionVector &s
 void TupleDataCollection::Gather(Vector &row_locations, const SelectionVector &scan_sel, const idx_t scan_count,
                                  const column_t column_id, Vector &result, const SelectionVector &target_sel,
                                  optional_ptr<Vector> cached_cast_vector) const {
+	D_ASSERT(!cached_cast_vector || FlatVector::Validity(*cached_cast_vector).AllValid()); // ResetCachedCastVectors
 	const auto &gather_function = gather_functions[column_id];
 	gather_function.function(layout, row_locations, column_id, scan_sel, scan_count, result, target_sel,
 	                         cached_cast_vector, gather_function.child_functions);
@@ -1063,10 +1110,10 @@ void TupleDataCollection::Gather(Vector &row_locations, const SelectionVector &s
 template <class T>
 static void TupleDataTemplatedGather(const TupleDataLayout &layout, Vector &row_locations, const idx_t col_idx,
                                      const SelectionVector &scan_sel, const idx_t scan_count, Vector &target,
-                                     const SelectionVector &target_sel, optional_ptr<Vector> dummy_vector,
-                                     const vector<TupleDataGatherFunction> &child_functions) {
+                                     const SelectionVector &target_sel, optional_ptr<Vector>,
+                                     const vector<TupleDataGatherFunction> &) {
 	// Source
-	auto source_locations = FlatVector::GetData<data_ptr_t>(row_locations);
+	const auto source_locations = FlatVector::GetData<data_ptr_t>(row_locations);
 
 	// Target
 	auto target_data = FlatVector::GetData<T>(target);
@@ -1081,13 +1128,16 @@ static void TupleDataTemplatedGather(const TupleDataLayout &layout, Vector &row_
 	for (idx_t i = 0; i < scan_count; i++) {
 		const auto &source_row = source_locations[scan_sel.get_index(i)];
 		const auto target_idx = target_sel.get_index(i);
+		target_data[target_idx] = Load<T>(source_row + offset_in_row);
 		ValidityBytes row_mask(source_row);
-		if (row_mask.RowIsValid(row_mask.GetValidityEntryUnsafe(entry_idx), idx_in_entry)) {
-			target_data[target_idx] = Load<T>(source_row + offset_in_row);
-			TupleDataValueVerify<T>(target.GetType(), target_data[target_idx]);
-		} else {
+		if (!row_mask.RowIsValid(row_mask.GetValidityEntryUnsafe(entry_idx), idx_in_entry)) {
 			target_validity.SetInvalid(target_idx);
 		}
+#ifdef DEBUG
+		else {
+			TupleDataValueVerify<T>(target.GetType(), target_data[target_idx]);
+		}
+#endif
 	}
 }
 
@@ -1096,7 +1146,7 @@ static void TupleDataStructGather(const TupleDataLayout &layout, Vector &row_loc
                                   const SelectionVector &target_sel, optional_ptr<Vector> dummy_vector,
                                   const vector<TupleDataGatherFunction> &child_functions) {
 	// Source
-	auto source_locations = FlatVector::GetData<data_ptr_t>(row_locations);
+	const auto source_locations = FlatVector::GetData<data_ptr_t>(row_locations);
 
 	// Target
 	auto &target_validity = FlatVector::Validity(target);
@@ -1145,13 +1195,13 @@ static void TupleDataStructGather(const TupleDataLayout &layout, Vector &row_loc
 //------------------------------------------------------------------------------
 static void TupleDataListGather(const TupleDataLayout &layout, Vector &row_locations, const idx_t col_idx,
                                 const SelectionVector &scan_sel, const idx_t scan_count, Vector &target,
-                                const SelectionVector &target_sel, optional_ptr<Vector> dummy_vector,
+                                const SelectionVector &target_sel, optional_ptr<Vector>,
                                 const vector<TupleDataGatherFunction> &child_functions) {
 	// Source
-	auto source_locations = FlatVector::GetData<data_ptr_t>(row_locations);
+	const auto source_locations = FlatVector::GetData<data_ptr_t>(row_locations);
 
 	// Target
-	auto target_list_entries = FlatVector::GetData<list_entry_t>(target);
+	const auto target_list_entries = FlatVector::GetData<list_entry_t>(target);
 	auto &target_list_validity = FlatVector::Validity(target);
 
 	// Precompute mask indexes
@@ -1161,7 +1211,7 @@ static void TupleDataListGather(const TupleDataLayout &layout, Vector &row_locat
 
 	// Load pointers to the data from the row
 	Vector heap_locations(LogicalType::POINTER);
-	auto source_heap_locations = FlatVector::GetData<data_ptr_t>(heap_locations);
+	const auto source_heap_locations = FlatVector::GetData<data_ptr_t>(heap_locations);
 
 	const auto offset_in_row = layout.GetOffsets()[col_idx];
 	uint64_t target_list_offset = 0;
@@ -1202,21 +1252,20 @@ static void TupleDataListGather(const TupleDataLayout &layout, Vector &row_locat
 // Collection Gather
 //------------------------------------------------------------------------------
 template <class T>
-static void TupleDataTemplatedWithinCollectionGather(const TupleDataLayout &layout, Vector &heap_locations,
-                                                     const idx_t list_size_before, const SelectionVector &scan_sel,
-                                                     const idx_t scan_count, Vector &target,
-                                                     const SelectionVector &target_sel,
-                                                     optional_ptr<Vector> list_vector,
-                                                     const vector<TupleDataGatherFunction> &child_functions) {
+static void
+TupleDataTemplatedWithinCollectionGather(const TupleDataLayout &, Vector &heap_locations, const idx_t list_size_before,
+                                         const SelectionVector &, const idx_t scan_count, Vector &target,
+                                         const SelectionVector &target_sel, optional_ptr<Vector> list_vector,
+                                         const vector<TupleDataGatherFunction> &) {
 	// List parent
 	const auto list_entries = FlatVector::GetData<list_entry_t>(*list_vector);
 	const auto &list_validity = FlatVector::Validity(*list_vector);
 
 	// Source
-	auto source_heap_locations = FlatVector::GetData<data_ptr_t>(heap_locations);
+	const auto source_heap_locations = FlatVector::GetData<data_ptr_t>(heap_locations);
 
 	// Target
-	auto target_data = FlatVector::GetData<T>(target);
+	const auto target_data = FlatVector::GetData<T>(target);
 	auto &target_validity = FlatVector::Validity(target);
 
 	uint64_t target_offset = list_size_before;
@@ -1265,7 +1314,7 @@ static void TupleDataStructWithinCollectionGather(const TupleDataLayout &layout,
 	const auto &list_validity = FlatVector::Validity(*list_vector);
 
 	// Source
-	auto source_heap_locations = FlatVector::GetData<data_ptr_t>(heap_locations);
+	const auto source_heap_locations = FlatVector::GetData<data_ptr_t>(heap_locations);
 
 	// Target
 	auto &target_validity = FlatVector::Validity(target);
@@ -1317,17 +1366,17 @@ static void TupleDataCollectionWithinCollectionGather(const TupleDataLayout &lay
 	const auto &list_validity = FlatVector::Validity(*list_vector);
 
 	// Source
-	auto source_heap_locations = FlatVector::GetData<data_ptr_t>(heap_locations);
+	const auto source_heap_locations = FlatVector::GetData<data_ptr_t>(heap_locations);
 
 	// Target
-	auto target_list_entries = FlatVector::GetData<list_entry_t>(target);
+	const auto target_list_entries = FlatVector::GetData<list_entry_t>(target);
 	auto &target_validity = FlatVector::Validity(target);
 	const auto child_list_size_before = ListVector::GetListSize(target);
 
 	// We need to create a vector that has the combined list sizes (hugeint_t has same size as list_entry_t)
 	Vector combined_list_vector(LogicalType::HUGEINT);
 	FlatVector::SetValidity(combined_list_vector, list_validity); // Has same validity as list parent
-	auto combined_list_entries = FlatVector::GetData<list_entry_t>(combined_list_vector);
+	const auto combined_list_entries = FlatVector::GetData<list_entry_t>(combined_list_vector);
 
 	uint64_t target_offset = list_size_before;
 	uint64_t target_child_offset = child_list_size_before;
@@ -1337,8 +1386,13 @@ static void TupleDataCollectionWithinCollectionGather(const TupleDataLayout &lay
 			continue;
 		}
 
+		// Set the offset of the combined list entry
+		auto &combined_list_entry = combined_list_entries[target_idx];
+		combined_list_entry.offset = target_child_offset;
+
 		const auto &list_length = list_entries[target_idx].length;
 		if (list_length == 0) {
+			combined_list_entry.length = 0;
 			continue;
 		}
 
@@ -1351,10 +1405,6 @@ static void TupleDataCollectionWithinCollectionGather(const TupleDataLayout &lay
 		const auto source_data_location = source_heap_location;
 		source_heap_location += list_length * sizeof(uint64_t);
 
-		// Set the offset of the combined list entry
-		auto &combined_list_entry = combined_list_entries[target_sel.get_index(i)];
-		combined_list_entry.offset = target_child_offset;
-
 		// Load the child validity and data belonging to this list entry
 		for (idx_t child_i = 0; child_i < list_length; child_i++) {
 			if (source_mask.RowIsValidUnsafe(child_i)) {
@@ -1392,7 +1442,6 @@ static void TupleDataCastToArrayListGather(const TupleDataLayout &layout, Vector
                                            const SelectionVector &scan_sel, const idx_t scan_count, Vector &target,
                                            const SelectionVector &target_sel, optional_ptr<Vector> cached_cast_vector,
                                            const vector<TupleDataGatherFunction> &child_functions) {
-
 	if (cached_cast_vector) {
 		// Reuse the cached cast vector
 		TupleDataListGather(layout, row_locations, col_idx, scan_sel, scan_count, *cached_cast_vector, target_sel,
@@ -1508,23 +1557,24 @@ TupleDataGatherFunction TupleDataCollection::GetGatherFunction(const LogicalType
 		return TupleDataGetGatherFunctionInternal(type, false);
 	}
 
-	if (type.Contains(LogicalTypeId::ARRAY)) {
+	if (TypeVisitor::Contains(type, LogicalTypeId::ARRAY)) {
 		// Special case: we cant handle arrays yet, so we need to replace them with lists when gathering
-		auto new_type = ArrayType::ConvertToList(type);
+		const auto new_type = ArrayType::ConvertToList(type);
 		TupleDataGatherFunction result;
 		// Theres only two cases: Either the array is within a struct, or it is within a list (or has now become a list)
-		if (new_type.InternalType() == PhysicalType::LIST) {
+		switch (new_type.InternalType()) {
+		case PhysicalType::LIST:
 			result.function = TupleDataCastToArrayListGather;
 			result.child_functions.push_back(
 			    TupleDataGetGatherFunctionInternal(ListType::GetChildType(new_type), true));
 			return result;
-		} else if (new_type.InternalType() == PhysicalType::STRUCT) {
+		case PhysicalType::STRUCT:
 			result.function = TupleDataCastToArrayStructGather;
 			for (const auto &child_type : StructType::GetChildTypes(new_type)) {
 				result.child_functions.push_back(TupleDataGetGatherFunctionInternal(child_type.second, false));
 			}
 			return result;
-		} else {
+		default:
 			throw InternalException("Unsupported type for TupleDataCollection::GetGatherFunction");
 		}
 	}

package/src/duckdb/src/common/types/row/tuple_data_segment.cpp

@@ -113,9 +113,12 @@ TupleDataSegment::TupleDataSegment(shared_ptr<TupleDataAllocator> allocator_p)
 
 TupleDataSegment::~TupleDataSegment() {
 	lock_guard<mutex> guard(pinned_handles_lock);
+	if (allocator) {
+		allocator->SetDestroyBufferUponUnpin(); // Prevent blocks from being added to eviction queue
+	}
 	pinned_row_handles.clear();
 	pinned_heap_handles.clear();
-	allocator = nullptr;
+	allocator.reset();
 }
 
 void SwapTupleDataSegment(TupleDataSegment &a, TupleDataSegment &b) {