duckdb 1.3.3-dev4.0 → 1.3.3

package/src/duckdb/src/common/types/row/tuple_data_scatter_gather.cpp

@@ -4,7 +4,6 @@
 #include "duckdb/common/types/null_value.hpp"
 #include "duckdb/common/types/row/tuple_data_collection.hpp"
 #include "duckdb/common/uhugeint.hpp"
-#include "duckdb/common/sorting/sort_key.hpp"
 
 namespace duckdb {
 
@@ -21,27 +20,23 @@ constexpr idx_t TupleDataWithinListFixedSize<string_t>() {
 }
 
 template <class T>
-static void TupleDataValueStore(const T &source, data_t *__restrict const &row_location, const idx_t &offset_in_row,
+static void TupleDataValueStore(const T &source, const data_ptr_t &row_location, const idx_t offset_in_row,
                                 data_ptr_t &) {
 	Store<T>(source, row_location + offset_in_row);
 }
 
 template <>
-inline void TupleDataValueStore(const string_t &source, data_t *__restrict const &row_location,
-                                const idx_t &offset_in_row, data_ptr_t &heap_location) {
-#ifdef D_ASSERT_IS_ENABLED
+inline void TupleDataValueStore(const string_t &source, const data_ptr_t &row_location, const idx_t offset_in_row,
+                                data_ptr_t &heap_location) {
+#ifdef DEBUG
 	source.VerifyCharacters();
 #endif
 	if (source.IsInlined()) {
 		Store<string_t>(source, row_location + offset_in_row);
 	} else {
-		// Copy non-inlined part
-		FastMemcpy(heap_location, source.GetPointer(), source.GetSize());
-		// Copy first 8 bytes of string_t
-		memcpy(row_location + offset_in_row, &source, string_t::HEADER_SIZE);
-		// Copy new heap pointer into the correct offset
-		Store<data_ptr_t>(heap_location, row_location + offset_in_row + string_t::HEADER_SIZE);
-		// Increment heap pointer
+		FastMemcpy(heap_location, source.GetData(), source.GetSize());
+		Store<string_t>(string_t(const_char_ptr_cast(heap_location), UnsafeNumericCast<uint32_t>(source.GetSize())),
+		                row_location + offset_in_row);
 		heap_location += source.GetSize();
 	}
 }
@@ -54,7 +49,7 @@ static void TupleDataWithinListValueStore(const T &source, const data_ptr_t &loc
 template <>
 inline void TupleDataWithinListValueStore(const string_t &source, const data_ptr_t &location,
                                           data_ptr_t &heap_location) {
-#ifdef D_ASSERT_IS_ENABLED
+#ifdef DEBUG
 	source.VerifyCharacters();
 #endif
 	Store<uint32_t>(UnsafeNumericCast<uint32_t>(source.GetSize()), location);
@@ -64,14 +59,14 @@ inline void TupleDataWithinListValueStore(const string_t &source, const data_ptr
 
 template <class T>
 void TupleDataValueVerify(const LogicalType &, const T &) {
-#ifdef D_ASSERT_IS_ENABLED
+#ifdef DEBUG
 	// NOP
 #endif
 }
 
 template <>
 inline void TupleDataValueVerify(const LogicalType &type, const string_t &value) {
-#ifdef D_ASSERT_IS_ENABLED
+#ifdef DEBUG
 	if (type.id() == LogicalTypeId::VARCHAR) {
 		value.Verify();
 	}
@@ -92,7 +87,7 @@ inline string_t TupleDataWithinListValueLoad(const data_ptr_t &location, data_pt
 }
 
 static void ResetCombinedListData(vector<TupleDataVectorFormat> &vector_data) {
-#ifdef D_ASSERT_IS_ENABLED
+#ifdef DEBUG
 	for (auto &vd : vector_data) {
 		vd.combined_list_data = nullptr;
 		ResetCombinedListData(vd.children);
@@ -115,32 +110,7 @@ void TupleDataCollection::ComputeHeapSizes(TupleDataChunkState &chunk_state, con
 }
 
 static idx_t StringHeapSize(const string_t &val) {
-	return !val.IsInlined() * val.GetSize();
-}
-
-#ifndef DUCKDB_SMALLER_BINARY
-template <bool ALL_VALID, bool HAS_APPEND_SEL, bool HAS_SOURCE_SEL>
-#endif
-void ComputeStringHeapSizesInternal(idx_t *const heap_sizes, const UnifiedVectorFormat &source_vector_data,
-                                    const SelectionVector &append_sel, const idx_t append_count) {
-	const auto source_data = UnifiedVectorFormat::GetData<string_t>(source_vector_data);
-	const auto &source_sel = *source_vector_data.sel;
-	const auto &source_validity = source_vector_data.validity;
-
-#ifdef DUCKDB_SMALLER_BINARY
-	const auto ALL_VALID = source_validity.AllValid();
-	const auto HAS_APPEND_SEL = append_sel.IsSet();
-	const auto HAS_SOURCE_SEL = source_sel.IsSet();
-#endif
-
-	// Fully branchless loop
-	const auto null_string_size = StringHeapSize(NullValue<string_t>());
-	for (idx_t i = 0; i < append_count; i++) {
-		const auto append_idx = HAS_APPEND_SEL ? append_sel.get_index_unsafe(i) : i;
-		const auto source_idx = HAS_SOURCE_SEL ? source_sel.get_index_unsafe(append_idx) : append_idx;
-		const auto valid = ALL_VALID || source_validity.RowIsValidUnsafe(source_idx);
-		heap_sizes[i] += valid * StringHeapSize(source_data[source_idx]) + !valid * null_string_size;
-	}
+	return val.IsInlined() ? 0 : val.GetSize();
 }
 
 void TupleDataCollection::ComputeHeapSizes(Vector &heap_sizes_v, const Vector &source_v,
@@ -152,7 +122,7 @@ void TupleDataCollection::ComputeHeapSizes(Vector &heap_sizes_v, const Vector &s
 		return;
 	}
 
-	const auto heap_sizes = FlatVector::GetData<idx_t>(heap_sizes_v);
+	auto heap_sizes = FlatVector::GetData<idx_t>(heap_sizes_v);
 
 	// Source
 	const auto &source_vector_data = source_format.unified;
@@ -162,47 +132,15 @@ void TupleDataCollection::ComputeHeapSizes(Vector &heap_sizes_v, const Vector &s
 	switch (type) {
 	case PhysicalType::VARCHAR: {
 		// Only non-inlined strings are stored in the heap
-#ifdef DUCKDB_SMALLER_BINARY
-		ComputeStringHeapSizesInternal(heap_sizes, source_vector_data, append_sel, append_count);
-#else
-		if (source_validity.AllValid()) {
-			if (append_sel.IsSet()) {
-				if (source_sel.IsSet()) {
-					ComputeStringHeapSizesInternal<true, true, true>(heap_sizes, source_vector_data, append_sel,
-					                                                 append_count);
-				} else {
-					ComputeStringHeapSizesInternal<true, true, false>(heap_sizes, source_vector_data, append_sel,
-					                                                  append_count);
-				}
-			} else {
-				if (source_sel.IsSet()) {
-					ComputeStringHeapSizesInternal<true, false, true>(heap_sizes, source_vector_data, append_sel,
-					                                                  append_count);
-				} else {
-					ComputeStringHeapSizesInternal<true, false, false>(heap_sizes, source_vector_data, append_sel,
-					                                                   append_count);
-				}
-			}
-		} else {
-			if (append_sel.IsSet()) {
-				if (source_sel.IsSet()) {
-					ComputeStringHeapSizesInternal<false, true, true>(heap_sizes, source_vector_data, append_sel,
-					                                                  append_count);
-				} else {
-					ComputeStringHeapSizesInternal<false, true, false>(heap_sizes, source_vector_data, append_sel,
-					                                                   append_count);
-				}
+		const auto source_data = UnifiedVectorFormat::GetData<string_t>(source_vector_data);
+		for (idx_t i = 0; i < append_count; i++) {
+			const auto source_idx = source_sel.get_index(append_sel.get_index(i));
+			if (source_validity.RowIsValid(source_idx)) {
+				heap_sizes[i] += StringHeapSize(source_data[source_idx]);
 			} else {
-				if (source_sel.IsSet()) {
-					ComputeStringHeapSizesInternal<false, false, true>(heap_sizes, source_vector_data, append_sel,
-					                                                   append_count);
-				} else {
-					ComputeStringHeapSizesInternal<false, false, false>(heap_sizes, source_vector_data, append_sel,
-					                                                    append_count);
-				}
+				heap_sizes[i] += StringHeapSize(NullValue<string_t>());
 			}
 		}
-#endif
 		break;
 	}
 	case PhysicalType::STRUCT: {
@@ -254,32 +192,6 @@ void TupleDataCollection::ComputeHeapSizes(Vector &heap_sizes_v, const Vector &s
 	}
 }
 
-void TupleDataCollection::SortKeyComputeHeapSizes(TupleDataChunkState &chunk_state, const DataChunk &new_chunk,
-                                                  const SelectionVector &append_sel, const idx_t append_count,
-                                                  const SortKeyType sort_key_type) {
-	D_ASSERT(sort_key_type != SortKeyType::INVALID);
-	D_ASSERT(!SortKeyUtils::IsConstantSize(sort_key_type));
-	D_ASSERT(new_chunk.ColumnCount() == 1);
-	if (new_chunk.data[0].GetType().id() != LogicalTypeId::BLOB) {
-		return;
-	}
-
-	const auto heap_sizes = FlatVector::GetData<idx_t>(chunk_state.heap_sizes);
-
-	const auto &source_vector_data = chunk_state.vector_data[0].unified;
-	const auto &source_sel = *source_vector_data.sel;
-	const auto &source_validity = source_vector_data.validity;
-	const auto source_data = UnifiedVectorFormat::GetData<string_t>(source_vector_data);
-
-	const auto inlined_size = SortKeyUtils::GetInlineLength(sort_key_type);
-	for (idx_t i = 0; i < append_count; i++) {
-		const auto source_idx = source_sel.get_index(append_sel.get_index(i));
-		const auto &string_size = source_data[source_idx].GetSize();
-		// If valid, and string cannot be inlined, we need to allocate heap space
-		heap_sizes[i] = (source_validity.RowIsValid(source_idx) & (string_size > inlined_size)) * string_size;
-	}
-}
-
 void TupleDataCollection::WithinCollectionComputeHeapSizes(Vector &heap_sizes_v, const Vector &source_v,
                                                            TupleDataVectorFormat &source_format,
                                                            const SelectionVector &append_sel, const idx_t append_count,
@@ -319,7 +231,7 @@ void TupleDataCollection::ComputeFixedWithinCollectionHeapSizes(Vector &heap_siz
                                                                 const UnifiedVectorFormat &list_data) {
 	// Parent list data
 	const auto list_sel = *list_data.sel;
-	const auto list_entries = UnifiedVectorFormat::GetDataUnsafe<list_entry_t>(list_data);
+	const auto list_entries = UnifiedVectorFormat::GetData<list_entry_t>(list_data);
 	const auto &list_validity = list_data.validity;
 
 	// Target
@@ -353,7 +265,7 @@ void TupleDataCollection::StringWithinCollectionComputeHeapSizes(Vector &heap_si
                                                                  const UnifiedVectorFormat &list_data) {
 	// Parent list data
 	const auto list_sel = *list_data.sel;
-	const auto list_entries = UnifiedVectorFormat::GetDataUnsafe<list_entry_t>(list_data);
+	const auto list_entries = UnifiedVectorFormat::GetData<list_entry_t>(list_data);
 	const auto &list_validity = list_data.validity;
 
 	// Source
@@ -401,7 +313,7 @@ void TupleDataCollection::StructWithinCollectionComputeHeapSizes(Vector &heap_si
                                                                  const UnifiedVectorFormat &list_data) {
 	// Parent list data
 	const auto list_sel = *list_data.sel;
-	const auto list_entries = UnifiedVectorFormat::GetDataUnsafe<list_entry_t>(list_data);
+	const auto list_entries = UnifiedVectorFormat::GetData<list_entry_t>(list_data);
 	const auto &list_validity = list_data.validity;
 
 	// Target
@@ -449,7 +361,7 @@ static void ApplySliceRecursive(const Vector &source_v, TupleDataVectorFormat &s
 		for (idx_t struct_col_idx = 0; struct_col_idx < struct_sources.size(); struct_col_idx++) {
 			auto &struct_source = *struct_sources[struct_col_idx];
 			auto &struct_format = source_format.children[struct_col_idx];
-#ifdef D_ASSERT_IS_ENABLED
+#ifdef DEBUG
 			D_ASSERT(!struct_format.combined_list_data);
 #endif
 			if (!struct_format.combined_list_data) {
@@ -467,13 +379,13 @@ void TupleDataCollection::CollectionWithinCollectionComputeHeapSizes(Vector &hea
                                                                      const UnifiedVectorFormat &list_data) {
 	// Parent list data
 	const auto list_sel = *list_data.sel;
-	const auto list_entries = UnifiedVectorFormat::GetDataUnsafe<list_entry_t>(list_data);
+	const auto list_entries = UnifiedVectorFormat::GetData<list_entry_t>(list_data);
 	const auto &list_validity = list_data.validity;
 
 	// Source
 	const auto &child_list_data = source_format.unified;
 	const auto child_list_sel = *child_list_data.sel;
-	const auto child_list_entries = UnifiedVectorFormat::GetDataUnsafe<list_entry_t>(child_list_data);
+	const auto child_list_entries = UnifiedVectorFormat::GetData<list_entry_t>(child_list_data);
 	const auto &child_list_validity = child_list_data.validity;
 
 	// Target
@@ -515,7 +427,7 @@ void TupleDataCollection::CollectionWithinCollectionComputeHeapSizes(Vector &hea
 
 	D_ASSERT(source_format.children.size() == 1);
 	auto &child_format = source_format.children[0];
-#ifdef D_ASSERT_IS_ENABLED
+#ifdef DEBUG
 	// In debug mode this should be deleted by ResetCombinedListData
 	D_ASSERT(!child_format.combined_list_data);
 #endif
@@ -647,7 +559,7 @@ static void InitializeValidityMask(const data_ptr_t row_locations[], const idx_t
 
 void TupleDataCollection::Scatter(TupleDataChunkState &chunk_state, const DataChunk &new_chunk,
                                   const SelectionVector &append_sel, const idx_t append_count) const {
-#ifdef D_ASSERT_IS_ENABLED
+#ifdef DEBUG
 	Vector heap_locations_copy(LogicalType::POINTER);
 	if (!layout.AllConstant()) {
 		const auto heap_locations = FlatVector::GetData<data_ptr_t>(chunk_state.heap_locations);
@@ -658,35 +570,26 @@ void TupleDataCollection::Scatter(TupleDataChunkState &chunk_state, const DataCh
 	}
 #endif
 
-	if (layout.IsSortKeyLayout()) {
-		const auto &scatter_function = scatter_functions[0];
-		scatter_function.function(new_chunk.data[0], chunk_state.vector_data[0], append_sel, append_count, layout,
-		                          chunk_state.row_locations, chunk_state.heap_locations, 0,
-		                          chunk_state.vector_data[0].unified, scatter_function.child_functions);
-	} else {
-		const auto row_locations = FlatVector::GetData<data_ptr_t>(chunk_state.row_locations);
+	const auto row_locations = FlatVector::GetData<data_ptr_t>(chunk_state.row_locations);
 
-		// Set the validity mask for each row before inserting data
-		if (!layout.AllValid()) {
-			InitializeValidityMask(row_locations, append_count, ValidityBytes::SizeInBytes(layout.ColumnCount()));
-		}
+	// Set the validity mask for each row before inserting data
+	InitializeValidityMask(row_locations, append_count, ValidityBytes::SizeInBytes(layout.ColumnCount()));
 
-		if (!layout.AllConstant()) {
-			// Set the heap size for each row
-			const auto heap_size_offset = layout.GetHeapSizeOffset();
-			const auto heap_sizes = FlatVector::GetData<idx_t>(chunk_state.heap_sizes);
-			for (idx_t i = 0; i < append_count; i++) {
-				Store<idx_t>(heap_sizes[i], row_locations[i] + heap_size_offset);
-			}
+	if (!layout.AllConstant()) {
+		// Set the heap size for each row
+		const auto heap_size_offset = layout.GetHeapSizeOffset();
+		const auto heap_sizes = FlatVector::GetData<idx_t>(chunk_state.heap_sizes);
+		for (idx_t i = 0; i < append_count; i++) {
+			Store<uint32_t>(UnsafeNumericCast<uint32_t>(heap_sizes[i]), row_locations[i] + heap_size_offset);
 		}
+	}
 
-		// Write the data
-		for (const auto &col_idx : chunk_state.column_ids) {
-			Scatter(chunk_state, new_chunk.data[col_idx], col_idx, append_sel, append_count);
-		}
+	// Write the data
+	for (const auto &col_idx : chunk_state.column_ids) {
+		Scatter(chunk_state, new_chunk.data[col_idx], col_idx, append_sel, append_count);
 	}
 
-#ifdef D_ASSERT_IS_ENABLED
+#ifdef DEBUG
 	// Verify that the size of the data written to the heap is the same as the size we computed it would be
 	if (!layout.AllConstant()) {
 		const auto original_heap_locations = FlatVector::GetData<data_ptr_t>(heap_locations_copy);
@@ -709,28 +612,18 @@ void TupleDataCollection::Scatter(TupleDataChunkState &chunk_state, const Vector
 	                          chunk_state.vector_data[column_id].unified, scatter_function.child_functions);
 }
 
-#ifdef DUCKDB_SMALLER_BINARY
 template <class T>
-#else
-template <class T, bool HAS_APPEND_SEL, bool HAS_SOURCE_SEL, bool ALL_VALID>
-#endif
-static void TupleDataTemplatedScatterInternal(const Vector &, const TupleDataVectorFormat &source_format,
-                                              const SelectionVector &append_sel, const idx_t append_count,
-                                              const TupleDataLayout &layout, const Vector &row_locations,
-                                              Vector &heap_locations, const idx_t col_idx, const UnifiedVectorFormat &,
-                                              const vector<TupleDataScatterFunction> &) {
+static void TupleDataTemplatedScatter(const Vector &, const TupleDataVectorFormat &source_format,
+                                      const SelectionVector &append_sel, const idx_t append_count,
+                                      const TupleDataLayout &layout, const Vector &row_locations,
+                                      Vector &heap_locations, const idx_t col_idx, const UnifiedVectorFormat &,
+                                      const vector<TupleDataScatterFunction> &) {
 	// Source
 	const auto &source_data = source_format.unified;
 	const auto &source_sel = *source_data.sel;
 	const auto data = UnifiedVectorFormat::GetData<T>(source_data);
 	const auto &validity = source_data.validity;
 
-#ifdef DUCKDB_SMALLER_BINARY
-	const auto HAS_APPEND_SEL = append_sel.IsSet();
-	const auto HAS_SOURCE_SEL = source_sel.IsSet();
-	const auto ALL_VALID = validity.AllValid();
-#endif
-
 	// Target
 	const auto target_locations = FlatVector::GetData<data_ptr_t>(row_locations);
 	const auto target_heap_locations = FlatVector::GetData<data_ptr_t>(heap_locations);
@@ -740,120 +633,21 @@ static void TupleDataTemplatedScatterInternal(const Vector &, const TupleDataVec
 	idx_t idx_in_entry;
 	ValidityBytes::GetEntryIndex(col_idx, entry_idx, idx_in_entry);
 
-	const auto column_count = layout.ColumnCount();
 	const auto offset_in_row = layout.GetOffsets()[col_idx];
-
-	const auto null_value = NullValue<T>();
-	for (idx_t i = 0; i < append_count; i++) {
-		const auto append_idx = HAS_APPEND_SEL ? append_sel.get_index_unsafe(i) : i;
-		const auto source_idx = HAS_SOURCE_SEL ? source_sel.get_index_unsafe(append_idx) : append_idx;
-		const auto &target_location = target_locations[i];
-		if (ALL_VALID || validity.RowIsValidUnsafe(source_idx)) {
-			TupleDataValueStore<T>(data[source_idx], target_location, offset_in_row, target_heap_locations[i]);
-		} else {
-			D_ASSERT(!layout.AllValid());
-			TupleDataValueStore<T>(null_value, target_location, offset_in_row, target_heap_locations[i]);
-			ValidityBytes(target_location, column_count).SetInvalidUnsafe(entry_idx, idx_in_entry);
-		}
-	}
-}
-
-template <class T>
-static void TupleDataTemplatedScatter(const Vector &source, const TupleDataVectorFormat &source_format,
-                                      const SelectionVector &append_sel, const idx_t append_count,
-                                      const TupleDataLayout &layout, const Vector &row_locations,
-                                      Vector &heap_locations, const idx_t col_idx, const UnifiedVectorFormat &dummy_arg,
-                                      const vector<TupleDataScatterFunction> &child_functions) {
-#ifdef DUCKDB_SMALLER_BINARY
-	TupleDataTemplatedScatterInternal<T>(source, source_format, append_sel, append_count, layout, row_locations,
-	                                     heap_locations, col_idx, dummy_arg, child_functions);
-#else
-	if (append_sel.IsSet()) {
-		if (source_format.unified.sel->IsSet()) {
-			if (source_format.unified.validity.AllValid()) {
-				TupleDataTemplatedScatterInternal<T, true, true, true>(source, source_format, append_sel, append_count,
-				                                                       layout, row_locations, heap_locations, col_idx,
-				                                                       dummy_arg, child_functions);
-			} else {
-				TupleDataTemplatedScatterInternal<T, true, true, false>(source, source_format, append_sel, append_count,
-				                                                        layout, row_locations, heap_locations, col_idx,
-				                                                        dummy_arg, child_functions);
-			}
-		} else {
-			if (source_format.unified.validity.AllValid()) {
-				TupleDataTemplatedScatterInternal<T, true, false, true>(source, source_format, append_sel, append_count,
-				                                                        layout, row_locations, heap_locations, col_idx,
-				                                                        dummy_arg, child_functions);
-			} else {
-				TupleDataTemplatedScatterInternal<T, true, false, false>(
-				    source, source_format, append_sel, append_count, layout, row_locations, heap_locations, col_idx,
-				    dummy_arg, child_functions);
-			}
-		}
-	} else {
-		if (source_format.unified.sel->IsSet()) {
-			if (source_format.unified.validity.AllValid()) {
-				TupleDataTemplatedScatterInternal<T, false, true, true>(source, source_format, append_sel, append_count,
-				                                                        layout, row_locations, heap_locations, col_idx,
-				                                                        dummy_arg, child_functions);
-			} else {
-				TupleDataTemplatedScatterInternal<T, false, true, false>(
-				    source, source_format, append_sel, append_count, layout, row_locations, heap_locations, col_idx,
-				    dummy_arg, child_functions);
-			}
-		} else {
-			if (source_format.unified.validity.AllValid()) {
-				TupleDataTemplatedScatterInternal<T, false, false, true>(
-				    source, source_format, append_sel, append_count, layout, row_locations, heap_locations, col_idx,
-				    dummy_arg, child_functions);
-			} else {
-				TupleDataTemplatedScatterInternal<T, false, false, false>(
-				    source, source_format, append_sel, append_count, layout, row_locations, heap_locations, col_idx,
-				    dummy_arg, child_functions);
-			}
-		}
-	}
-#endif
-}
-
-template <class T, SortKeyType SORT_KEY_TYPE>
-void TupleDataSortKeyScatter(const Vector &, const TupleDataVectorFormat &source_format,
-                             const SelectionVector &append_sel, const idx_t append_count, const TupleDataLayout &layout,
-                             const Vector &row_locations, Vector &heap_locations, const idx_t,
-                             const UnifiedVectorFormat &, const vector<TupleDataScatterFunction> &) {
-	D_ASSERT(layout.IsSortKeyLayout());
-	D_ASSERT(layout.GetSortKeyType() == SORT_KEY_TYPE);
-	using SORT_KEY = SortKey<SORT_KEY_TYPE>;
-
-	// Source
-	const auto &source_data = source_format.unified;
-	const auto &source_sel = *source_data.sel;
-	const auto data = UnifiedVectorFormat::GetData<T>(source_data);
-	const auto &validity = source_data.validity;
-
-	// Target
-	const auto target_locations = FlatVector::GetData<SORT_KEY *>(row_locations);
-	const auto target_heap_locations = FlatVector::GetData<data_ptr_t>(heap_locations);
-
 	if (validity.AllValid()) {
-		// Fast path
-		if (!append_sel.IsSet() && !source_sel.IsSet()) {
-			for (idx_t i = 0; i < append_count; i++) {
-				target_locations[i]->Construct(data[i], target_heap_locations[i]);
-			}
-		} else {
-			for (idx_t i = 0; i < append_count; i++) {
-				const auto source_idx = source_sel.get_index(append_sel.get_index(i));
-				target_locations[i]->Construct(data[source_idx], target_heap_locations[i]);
-			}
+		for (idx_t i = 0; i < append_count; i++) {
+			const auto source_idx = source_sel.get_index(append_sel.get_index(i));
+			TupleDataValueStore<T>(data[source_idx], target_locations[i], offset_in_row, target_heap_locations[i]);
 		}
 	} else {
 		for (idx_t i = 0; i < append_count; i++) {
 			const auto source_idx = source_sel.get_index(append_sel.get_index(i));
-			// validity.AllValid() may not be true when doing aggressive vector verification
-			// but the actual values should always all be valid
-			D_ASSERT(validity.RowIsValid(source_idx));
-			target_locations[i]->Construct(data[source_idx], target_heap_locations[i]);
+			if (validity.RowIsValid(source_idx)) {
+				TupleDataValueStore<T>(data[source_idx], target_locations[i], offset_in_row, target_heap_locations[i]);
+			} else {
+				TupleDataValueStore<T>(NullValue<T>(), target_locations[i], offset_in_row, target_heap_locations[i]);
+				ValidityBytes(target_locations[i], layout.ColumnCount()).SetInvalidUnsafe(entry_idx, idx_in_entry);
+			}
 		}
 	}
 }
@@ -878,7 +672,6 @@ static void TupleDataStructScatter(const Vector &source, const TupleDataVectorFo
 
 	// Set validity of the STRUCT in this layout
 	if (!validity.AllValid()) {
-		D_ASSERT(!layout.AllValid());
 		for (idx_t i = 0; i < append_count; i++) {
 			const auto source_idx = source_sel.get_index(append_sel.get_index(i));
 			if (!validity.RowIsValid(source_idx)) {
@@ -925,7 +718,7 @@ static void TupleDataListScatter(const Vector &source, const TupleDataVectorForm
 	// Source
 	const auto &source_data = source_format.unified;
 	const auto &source_sel = *source_data.sel;
-	const auto data = UnifiedVectorFormat::GetDataUnsafe<list_entry_t>(source_data);
+	const auto data = UnifiedVectorFormat::GetData<list_entry_t>(source_data);
 	const auto &validity = source_data.validity;
 
 	// Target
@@ -949,7 +742,6 @@ static void TupleDataListScatter(const Vector &source, const TupleDataVectorForm
 			Store<uint64_t>(data[source_idx].length, target_heap_location);
 			target_heap_location += sizeof(uint64_t);
 		} else {
-			D_ASSERT(!layout.AllValid());
 			ValidityBytes(target_locations[i], layout.ColumnCount()).SetInvalidUnsafe(entry_idx, idx_in_entry);
 		}
 	}
@@ -975,7 +767,7 @@ static void TupleDataArrayScatter(const Vector &source, const TupleDataVectorFor
 	// The Array vector has fake list_entry_t's set by this point, so this is fine
 	const auto &source_data = source_format.unified;
 	const auto &source_sel = *source_data.sel;
-	const auto data = UnifiedVectorFormat::GetDataUnsafe<list_entry_t>(source_data);
+	const auto data = UnifiedVectorFormat::GetData<list_entry_t>(source_data);
 	const auto &validity = source_data.validity;
 
 	// Target
@@ -999,7 +791,6 @@ static void TupleDataArrayScatter(const Vector &source, const TupleDataVectorFor
 			Store<uint64_t>(data[source_idx].length, target_heap_location);
 			target_heap_location += sizeof(uint64_t);
 		} else {
-			D_ASSERT(!layout.AllValid());
 			ValidityBytes(target_locations[i], layout.ColumnCount()).SetInvalidUnsafe(entry_idx, idx_in_entry);
 		}
 	}
@@ -1024,7 +815,7 @@ static void TupleDataTemplatedWithinCollectionScatter(const Vector &, const Tupl
                                                       const vector<TupleDataScatterFunction> &) {
 	// Parent list data
 	const auto &list_sel = *list_data.sel;
-	const auto list_entries = UnifiedVectorFormat::GetDataUnsafe<list_entry_t>(list_data);
+	const auto list_entries = UnifiedVectorFormat::GetData<list_entry_t>(list_data);
 	const auto &list_validity = list_data.validity;
 
 	// Source
@@ -1082,7 +873,7 @@ static void TupleDataStructWithinCollectionScatter(const Vector &source, const T
                                                    const vector<TupleDataScatterFunction> &child_functions) {
 	// Parent list data
 	const auto &list_sel = *list_data.sel;
-	const auto list_entries = UnifiedVectorFormat::GetDataUnsafe<list_entry_t>(list_data);
+	const auto list_entries = UnifiedVectorFormat::GetData<list_entry_t>(list_data);
 	const auto &list_validity = list_data.validity;
 
 	// Source
@@ -1145,13 +936,13 @@ static void TupleDataCollectionWithinCollectionScatter(const Vector &child_list,
                                                        const vector<TupleDataScatterFunction> &child_functions) {
 	// Parent list data
 	const auto &list_sel = *list_data.sel;
-	const auto list_entries = UnifiedVectorFormat::GetDataUnsafe<list_entry_t>(list_data);
+	const auto list_entries = UnifiedVectorFormat::GetData<list_entry_t>(list_data);
 	const auto &list_validity = list_data.validity;
 
 	// Source
 	const auto &child_list_data = child_list_format.unified;
 	const auto &child_list_sel = *child_list_data.sel;
-	const auto child_list_entries = UnifiedVectorFormat::GetDataUnsafe<list_entry_t>(child_list_data);
+	const auto child_list_entries = UnifiedVectorFormat::GetData<list_entry_t>(child_list_data);
 	const auto &child_list_validity = child_list_data.validity;
 
 	// Target
@@ -1281,55 +1072,6 @@ TupleDataScatterFunction TupleDataCollection::GetScatterFunction(const LogicalTy
 	return result;
 }
 
-template <class T>
-TupleDataScatterFunction GetSortKeyScatterFunctionInternal(SortKeyType sort_key_type) {
-	TupleDataScatterFunction result;
-	switch (sort_key_type) {
-	case SortKeyType::NO_PAYLOAD_FIXED_8:
-		result.function = TupleDataSortKeyScatter<T, SortKeyType::NO_PAYLOAD_FIXED_8>;
-		break;
-	case SortKeyType::NO_PAYLOAD_FIXED_16:
-		result.function = TupleDataSortKeyScatter<T, SortKeyType::NO_PAYLOAD_FIXED_16>;
-		break;
-	case SortKeyType::NO_PAYLOAD_FIXED_24:
-		result.function = TupleDataSortKeyScatter<T, SortKeyType::NO_PAYLOAD_FIXED_24>;
-		break;
-	case SortKeyType::NO_PAYLOAD_FIXED_32:
-		result.function = TupleDataSortKeyScatter<T, SortKeyType::NO_PAYLOAD_FIXED_32>;
-		break;
-	case SortKeyType::NO_PAYLOAD_VARIABLE_32:
-		result.function = TupleDataSortKeyScatter<T, SortKeyType::NO_PAYLOAD_VARIABLE_32>;
-		break;
-	case SortKeyType::PAYLOAD_FIXED_16:
-		result.function = TupleDataSortKeyScatter<T, SortKeyType::PAYLOAD_FIXED_16>;
-		break;
-	case SortKeyType::PAYLOAD_FIXED_24:
-		result.function = TupleDataSortKeyScatter<T, SortKeyType::PAYLOAD_FIXED_24>;
-		break;
-	case SortKeyType::PAYLOAD_FIXED_32:
-		result.function = TupleDataSortKeyScatter<T, SortKeyType::PAYLOAD_FIXED_32>;
-		break;
-	case SortKeyType::PAYLOAD_VARIABLE_32:
-		result.function = TupleDataSortKeyScatter<T, SortKeyType::PAYLOAD_VARIABLE_32>;
-		break;
-	default:
-		throw NotImplementedException("GetSortKeyScatterFunction for %s", EnumUtil::ToString(sort_key_type));
-	}
-	return result;
-}
-
-TupleDataScatterFunction TupleDataCollection::GetSortKeyScatterFunction(const LogicalType &type,
-                                                                        SortKeyType sort_key_type) {
-	switch (type.id()) {
-	case LogicalTypeId::BIGINT:
-		return GetSortKeyScatterFunctionInternal<int64_t>(sort_key_type);
-	case LogicalTypeId::BLOB:
-		return GetSortKeyScatterFunctionInternal<string_t>(sort_key_type);
-	default:
-		throw NotImplementedException("TupleDataCollection::GetSortKeyScatterFunction for %s", type.ToString());
-	}
-}
-
 //-------------------------------------------------------------------------------
 // Gather
 //-------------------------------------------------------------------------------
@@ -1365,20 +1107,11 @@ void TupleDataCollection::Gather(Vector &row_locations, const SelectionVector &s
 	Vector::Verify(result, target_sel, scan_count);
 }
 
-#ifdef DUCKDB_SMALLER_BINARY
 template <class T>
-#else
-template <class T, bool HAS_SCAN_SEL, bool HAS_TARGET_SEL, bool ALL_VALID>
-#endif
-static void TupleDataTemplatedGatherInternal(const TupleDataLayout &layout, Vector &row_locations, const idx_t col_idx,
-                                             const SelectionVector &scan_sel, const idx_t scan_count, Vector &target,
-                                             const SelectionVector &target_sel, optional_ptr<Vector>,
-                                             const vector<TupleDataGatherFunction> &) {
-#ifdef DUCKDB_SMALLER_BINARY
-	const bool HAS_SCAN_SEL = scan_sel.IsSet();
-	const bool HAS_TARGET_SEL = target_sel.IsSet();
-	const bool ALL_VALID = layout.AllValid();
-#endif
+static void TupleDataTemplatedGather(const TupleDataLayout &layout, Vector &row_locations, const idx_t col_idx,
+                                     const SelectionVector &scan_sel, const idx_t scan_count, Vector &target,
+                                     const SelectionVector &target_sel, optional_ptr<Vector>,
+                                     const vector<TupleDataGatherFunction> &) {
 	// Source
 	const auto source_locations = FlatVector::GetData<data_ptr_t>(row_locations);
 
@@ -1392,85 +1125,20 @@ static void TupleDataTemplatedGatherInternal(const TupleDataLayout &layout, Vect
 	ValidityBytes::GetEntryIndex(col_idx, entry_idx, idx_in_entry);
 
 	const auto offset_in_row = layout.GetOffsets()[col_idx];
-	const auto column_count = layout.ColumnCount();
 	for (idx_t i = 0; i < scan_count; i++) {
-		const auto source_idx = HAS_SCAN_SEL ? scan_sel.get_index_unsafe(i) : i;
-		const auto target_idx = HAS_TARGET_SEL ? target_sel.get_index_unsafe(i) : i;
-		const auto &source_row = source_locations[source_idx];
+		const auto &source_row = source_locations[scan_sel.get_index(i)];
+		const auto target_idx = target_sel.get_index(i);
 		target_data[target_idx] = Load<T>(source_row + offset_in_row);
-		if (!ALL_VALID &&
-		    !ValidityBytes::RowIsValid(ValidityBytes(source_row, column_count).GetValidityEntryUnsafe(entry_idx),
-		                               idx_in_entry)) {
+		ValidityBytes row_mask(source_row, layout.ColumnCount());
+		if (!row_mask.RowIsValid(row_mask.GetValidityEntryUnsafe(entry_idx), idx_in_entry)) {
 			target_validity.SetInvalid(target_idx);
-		} else {
-			TupleDataValueVerify<T>(target.GetType(), target_data[target_idx]);
-		}
-	}
-}
-
-template <class T>
-static void TupleDataTemplatedGather(const TupleDataLayout &layout, Vector &row_locations, const idx_t col_idx,
-                                     const SelectionVector &scan_sel, const idx_t scan_count, Vector &target,
-                                     const SelectionVector &target_sel, optional_ptr<Vector> list_vector,
-                                     const vector<TupleDataGatherFunction> &child_functions) {
-#ifdef DUCKDB_SMALLER_BINARY
-	TupleDataTemplatedGatherInternal<T>(layout, row_locations, col_idx, scan_sel, scan_count, target, target_sel,
-	                                    list_vector, child_functions);
-#else
-	if (scan_sel.IsSet()) {
-		if (target_sel.IsSet()) {
-			if (layout.AllValid()) {
-				TupleDataTemplatedGatherInternal<T, true, true, true>(layout, row_locations, col_idx, scan_sel,
-				                                                      scan_count, target, target_sel, list_vector,
-				                                                      child_functions);
-			} else {
-				TupleDataTemplatedGatherInternal<T, true, true, false>(layout, row_locations, col_idx, scan_sel,
-				                                                       scan_count, target, target_sel, list_vector,
-				                                                       child_functions);
-			}
-		} else {
-			if (layout.AllValid()) {
-				TupleDataTemplatedGatherInternal<T, true, false, true>(layout, row_locations, col_idx, scan_sel,
-				                                                       scan_count, target, target_sel, list_vector,
-				                                                       child_functions);
-			} else {
-				TupleDataTemplatedGatherInternal<T, true, false, false>(layout, row_locations, col_idx, scan_sel,
-				                                                        scan_count, target, target_sel, list_vector,
-				                                                        child_functions);
-			}
 		}
-	} else {
-		if (target_sel.IsSet()) {
-			if (layout.AllValid()) {
-				TupleDataTemplatedGatherInternal<T, false, true, true>(layout, row_locations, col_idx, scan_sel,
-				                                                       scan_count, target, target_sel, list_vector,
-				                                                       child_functions);
-			} else {
-				TupleDataTemplatedGatherInternal<T, false, true, false>(layout, row_locations, col_idx, scan_sel,
-				                                                        scan_count, target, target_sel, list_vector,
-				                                                        child_functions);
-			}
-		} else {
-			if (layout.AllValid()) {
-				TupleDataTemplatedGatherInternal<T, false, false, true>(layout, row_locations, col_idx, scan_sel,
-				                                                        scan_count, target, target_sel, list_vector,
-				                                                        child_functions);
-			} else {
-				TupleDataTemplatedGatherInternal<T, false, false, false>(layout, row_locations, col_idx, scan_sel,
-				                                                         scan_count, target, target_sel, list_vector,
-				                                                         child_functions);
-			}
+#ifdef DEBUG
+		else {
+			TupleDataValueVerify<T>(target.GetType(), target_data[target_idx]);
 		}
-	}
 #endif
-}
-
-template <class T, SortKeyType SORT_KEY_TYPE>
-void TupleDataSortKeyGather(const TupleDataLayout &layout, Vector &row_locations, const idx_t col_idx,
-                            const SelectionVector &scan_sel, const idx_t scan_count, Vector &target,
-                            const SelectionVector &target_sel, optional_ptr<Vector>,
-                            const vector<TupleDataGatherFunction> &) {
-	throw NotImplementedException("Unimplemented type for TupleDataSortKeyGather");
+	}
 }
 
 static void TupleDataStructGather(const TupleDataLayout &layout, Vector &row_locations, const idx_t col_idx,
@@ -1497,9 +1165,8 @@ static void TupleDataStructGather(const TupleDataLayout &layout, Vector &row_loc
 		const auto &source_row = source_locations[source_idx];
 
 		// Set the validity
-		if (!layout.AllValid() &&
-		    !ValidityBytes::RowIsValid(
-		        ValidityBytes(source_row, layout.ColumnCount()).GetValidityEntryUnsafe(entry_idx), idx_in_entry)) {
+		ValidityBytes row_mask(source_row, layout.ColumnCount());
+		if (!row_mask.RowIsValid(row_mask.GetValidityEntryUnsafe(entry_idx), idx_in_entry)) {
 			const auto target_idx = target_sel.get_index(i);
 			target_validity.SetInvalid(target_idx);
 		}
@@ -1534,7 +1201,7 @@ static void TupleDataListGather(const TupleDataLayout &layout, Vector &row_locat
 	const auto source_locations = FlatVector::GetData<data_ptr_t>(row_locations);
 
 	// Target
-	const auto target_list_entries = FlatVector::GetDataUnsafe<list_entry_t>(target);
+	const auto target_list_entries = FlatVector::GetData<list_entry_t>(target);
 	auto &target_list_validity = FlatVector::Validity(target);
 
 	// Precompute mask indexes
@@ -1551,11 +1218,10 @@ static void TupleDataListGather(const TupleDataLayout &layout, Vector &row_locat
 	uint64_t target_list_offset = list_size_before;
 	for (idx_t i = 0; i < scan_count; i++) {
 		const auto &source_row = source_locations[scan_sel.get_index(i)];
+		ValidityBytes row_mask(source_row, layout.ColumnCount());
 
 		const auto target_idx = target_sel.get_index(i);
-		if (layout.AllValid() ||
-		    ValidityBytes::RowIsValid(ValidityBytes(source_row, layout.ColumnCount()).GetValidityEntryUnsafe(entry_idx),
-		                              idx_in_entry)) {
+		if (row_mask.RowIsValid(row_mask.GetValidityEntryUnsafe(entry_idx), idx_in_entry)) {
 			auto &source_heap_location = source_heap_locations[i];
 			source_heap_location = Load<data_ptr_t>(source_row + offset_in_row);
 
@@ -1592,7 +1258,7 @@ TupleDataTemplatedWithinCollectionGather(const TupleDataLayout &, Vector &heap_l
                                          const SelectionVector &target_sel, optional_ptr<Vector> list_vector,
                                          const vector<TupleDataGatherFunction> &) {
 	// List parent
-	const auto list_entries = FlatVector::GetDataUnsafe<list_entry_t>(*list_vector);
+	const auto list_entries = FlatVector::GetData<list_entry_t>(*list_vector);
 	const auto &list_validity = FlatVector::Validity(*list_vector);
 
 	// Source
@@ -1644,7 +1310,7 @@ static void TupleDataStructWithinCollectionGather(const TupleDataLayout &layout,
                                                   const SelectionVector &target_sel, optional_ptr<Vector> list_vector,
                                                   const vector<TupleDataGatherFunction> &child_functions) {
 	// List parent
-	const auto list_entries = FlatVector::GetDataUnsafe<list_entry_t>(*list_vector);
+	const auto list_entries = FlatVector::GetData<list_entry_t>(*list_vector);
 	const auto &list_validity = FlatVector::Validity(*list_vector);
 
 	// Source
@@ -1696,21 +1362,21 @@ static void TupleDataCollectionWithinCollectionGather(const TupleDataLayout &lay
                                                       optional_ptr<Vector> list_vector,
                                                       const vector<TupleDataGatherFunction> &child_functions) {
 	// List parent
-	const auto list_entries = FlatVector::GetDataUnsafe<list_entry_t>(*list_vector);
+	const auto list_entries = FlatVector::GetData<list_entry_t>(*list_vector);
 	const auto &list_validity = FlatVector::Validity(*list_vector);
 
 	// Source
 	const auto source_heap_locations = FlatVector::GetData<data_ptr_t>(heap_locations);
 
 	// Target
-	const auto target_list_entries = FlatVector::GetDataUnsafe<list_entry_t>(target);
+	const auto target_list_entries = FlatVector::GetData<list_entry_t>(target);
 	auto &target_validity = FlatVector::Validity(target);
 	const auto child_list_size_before = ListVector::GetListSize(target);
 
 	// We need to create a vector that has the combined list sizes (hugeint_t has same size as list_entry_t)
 	Vector combined_list_vector(LogicalType::HUGEINT);
 	FlatVector::SetValidity(combined_list_vector, list_validity); // Has same validity as list parent
-	const auto combined_list_entries = FlatVector::GetDataUnsafe<list_entry_t>(combined_list_vector);
+	const auto combined_list_entries = FlatVector::GetData<list_entry_t>(combined_list_vector);
 
 	uint64_t target_offset = list_size_before;
 	uint64_t target_child_offset = child_list_size_before;
@@ -1915,53 +1581,4 @@ TupleDataGatherFunction TupleDataCollection::GetGatherFunction(const LogicalType
 	return TupleDataGetGatherFunctionInternal(type, false);
 }
 
-template <class T>
-TupleDataGatherFunction GetSortKeyGatherFunctionInternal(SortKeyType sort_key_type) {
-	TupleDataGatherFunction result;
-	switch (sort_key_type) {
-	case SortKeyType::NO_PAYLOAD_FIXED_8:
-		result.function = TupleDataSortKeyGather<T, SortKeyType::NO_PAYLOAD_FIXED_8>;
-		break;
-	case SortKeyType::NO_PAYLOAD_FIXED_16:
-		result.function = TupleDataSortKeyGather<T, SortKeyType::NO_PAYLOAD_FIXED_16>;
-		break;
-	case SortKeyType::NO_PAYLOAD_FIXED_24:
-		result.function = TupleDataSortKeyGather<T, SortKeyType::NO_PAYLOAD_FIXED_24>;
-		break;
-	case SortKeyType::NO_PAYLOAD_FIXED_32:
-		result.function = TupleDataSortKeyGather<T, SortKeyType::NO_PAYLOAD_FIXED_32>;
-		break;
-	case SortKeyType::NO_PAYLOAD_VARIABLE_32:
-		result.function = TupleDataSortKeyGather<T, SortKeyType::NO_PAYLOAD_VARIABLE_32>;
-		break;
-	case SortKeyType::PAYLOAD_FIXED_16:
-		result.function = TupleDataSortKeyGather<T, SortKeyType::PAYLOAD_FIXED_16>;
-		break;
-	case SortKeyType::PAYLOAD_FIXED_24:
-		result.function = TupleDataSortKeyGather<T, SortKeyType::PAYLOAD_FIXED_24>;
-		break;
-	case SortKeyType::PAYLOAD_FIXED_32:
-		result.function = TupleDataSortKeyGather<T, SortKeyType::PAYLOAD_FIXED_32>;
-		break;
-	case SortKeyType::PAYLOAD_VARIABLE_32:
-		result.function = TupleDataSortKeyGather<T, SortKeyType::PAYLOAD_VARIABLE_32>;
-		break;
-	default:
-		throw NotImplementedException("GetSortKeyGatherFunction for %s", EnumUtil::ToString(sort_key_type));
-	}
-	return result;
-}
-
-TupleDataGatherFunction TupleDataCollection::GetSortKeyGatherFunction(const LogicalType &type,
-                                                                      SortKeyType sort_key_type) {
-	switch (type.id()) {
-	case LogicalTypeId::BIGINT:
-		return GetSortKeyGatherFunctionInternal<int64_t>(sort_key_type);
-	case LogicalTypeId::BLOB:
-		return GetSortKeyGatherFunctionInternal<string_t>(sort_key_type);
-	default:
-		throw NotImplementedException("TupleDataCollection::GetSortKeyGatherFunction for %s", type.ToString());
-	}
-}
-
 } // namespace duckdb