duckdb 1.3.4 → 1.4.1-dev2.0

package/src/duckdb/src/common/types/row/tuple_data_scatter_gather.cpp

@@ -4,6 +4,7 @@
 #include "duckdb/common/types/null_value.hpp"
 #include "duckdb/common/types/row/tuple_data_collection.hpp"
 #include "duckdb/common/uhugeint.hpp"
+#include "duckdb/common/sorting/sort_key.hpp"
 
 namespace duckdb {
 
@@ -20,23 +21,27 @@ constexpr idx_t TupleDataWithinListFixedSize<string_t>() {
 }
 
 template <class T>
-static void TupleDataValueStore(const T &source, const data_ptr_t &row_location, const idx_t offset_in_row,
+static void TupleDataValueStore(const T &source, data_t *__restrict const &row_location, const idx_t &offset_in_row,
                                 data_ptr_t &) {
 	Store<T>(source, row_location + offset_in_row);
 }
 
 template <>
-inline void TupleDataValueStore(const string_t &source, const data_ptr_t &row_location, const idx_t offset_in_row,
-                                data_ptr_t &heap_location) {
-#ifdef DEBUG
+inline void TupleDataValueStore(const string_t &source, data_t *__restrict const &row_location,
+                                const idx_t &offset_in_row, data_ptr_t &heap_location) {
+#ifdef D_ASSERT_IS_ENABLED
 	source.VerifyCharacters();
 #endif
 	if (source.IsInlined()) {
 		Store<string_t>(source, row_location + offset_in_row);
 	} else {
-		FastMemcpy(heap_location, source.GetData(), source.GetSize());
-		Store<string_t>(string_t(const_char_ptr_cast(heap_location), UnsafeNumericCast<uint32_t>(source.GetSize())),
-		                row_location + offset_in_row);
+		// Copy non-inlined part
+		FastMemcpy(heap_location, source.GetPointer(), source.GetSize());
+		// Copy first 8 bytes of string_t
+		memcpy(row_location + offset_in_row, &source, string_t::HEADER_SIZE);
+		// Copy new heap pointer into the correct offset
+		Store<data_ptr_t>(heap_location, row_location + offset_in_row + string_t::HEADER_SIZE);
+		// Increment heap pointer
 		heap_location += source.GetSize();
 	}
 }
@@ -49,7 +54,7 @@ static void TupleDataWithinListValueStore(const T &source, const data_ptr_t &loc
 template <>
 inline void TupleDataWithinListValueStore(const string_t &source, const data_ptr_t &location,
                                           data_ptr_t &heap_location) {
-#ifdef DEBUG
+#ifdef D_ASSERT_IS_ENABLED
 	source.VerifyCharacters();
 #endif
 	Store<uint32_t>(UnsafeNumericCast<uint32_t>(source.GetSize()), location);
@@ -59,14 +64,14 @@ inline void TupleDataWithinListValueStore(const string_t &source, const data_ptr
 
 template <class T>
 void TupleDataValueVerify(const LogicalType &, const T &) {
-#ifdef DEBUG
+#ifdef D_ASSERT_IS_ENABLED
 	// NOP
 #endif
 }
 
 template <>
 inline void TupleDataValueVerify(const LogicalType &type, const string_t &value) {
-#ifdef DEBUG
+#ifdef D_ASSERT_IS_ENABLED
 	if (type.id() == LogicalTypeId::VARCHAR) {
 		value.Verify();
 	}
@@ -87,7 +92,7 @@ inline string_t TupleDataWithinListValueLoad(const data_ptr_t &location, data_pt
 }
 
 static void ResetCombinedListData(vector<TupleDataVectorFormat> &vector_data) {
-#ifdef DEBUG
+#ifdef D_ASSERT_IS_ENABLED
 	for (auto &vd : vector_data) {
 		vd.combined_list_data = nullptr;
 		ResetCombinedListData(vd.children);
@@ -110,7 +115,32 @@ void TupleDataCollection::ComputeHeapSizes(TupleDataChunkState &chunk_state, con
 }
 
 static idx_t StringHeapSize(const string_t &val) {
-	return val.IsInlined() ? 0 : val.GetSize();
+	return !val.IsInlined() * val.GetSize();
+}
+
+#ifndef DUCKDB_SMALLER_BINARY
+template <bool ALL_VALID, bool HAS_APPEND_SEL, bool HAS_SOURCE_SEL>
+#endif
+void ComputeStringHeapSizesInternal(idx_t *const heap_sizes, const UnifiedVectorFormat &source_vector_data,
+                                    const SelectionVector &append_sel, const idx_t append_count) {
+	const auto source_data = UnifiedVectorFormat::GetData<string_t>(source_vector_data);
+	const auto &source_sel = *source_vector_data.sel;
+	const auto &source_validity = source_vector_data.validity;
+
+#ifdef DUCKDB_SMALLER_BINARY
+	const auto ALL_VALID = source_validity.AllValid();
+	const auto HAS_APPEND_SEL = append_sel.IsSet();
+	const auto HAS_SOURCE_SEL = source_sel.IsSet();
+#endif
+
+	// Fully branchless loop
+	const auto null_string_size = StringHeapSize(NullValue<string_t>());
+	for (idx_t i = 0; i < append_count; i++) {
+		const auto append_idx = HAS_APPEND_SEL ? append_sel.get_index_unsafe(i) : i;
+		const auto source_idx = HAS_SOURCE_SEL ? source_sel.get_index_unsafe(append_idx) : append_idx;
+		const auto valid = ALL_VALID || source_validity.RowIsValidUnsafe(source_idx);
+		heap_sizes[i] += valid * StringHeapSize(source_data[source_idx]) + !valid * null_string_size;
+	}
 }
 
 void TupleDataCollection::ComputeHeapSizes(Vector &heap_sizes_v, const Vector &source_v,
@@ -122,7 +152,7 @@ void TupleDataCollection::ComputeHeapSizes(Vector &heap_sizes_v, const Vector &s
 		return;
 	}
 
-	auto heap_sizes = FlatVector::GetData<idx_t>(heap_sizes_v);
+	const auto heap_sizes = FlatVector::GetData<idx_t>(heap_sizes_v);
 
 	// Source
 	const auto &source_vector_data = source_format.unified;
@@ -132,15 +162,47 @@ void TupleDataCollection::ComputeHeapSizes(Vector &heap_sizes_v, const Vector &s
 	switch (type) {
 	case PhysicalType::VARCHAR: {
 		// Only non-inlined strings are stored in the heap
-		const auto source_data = UnifiedVectorFormat::GetData<string_t>(source_vector_data);
-		for (idx_t i = 0; i < append_count; i++) {
-			const auto source_idx = source_sel.get_index(append_sel.get_index(i));
-			if (source_validity.RowIsValid(source_idx)) {
-				heap_sizes[i] += StringHeapSize(source_data[source_idx]);
+#ifdef DUCKDB_SMALLER_BINARY
+		ComputeStringHeapSizesInternal(heap_sizes, source_vector_data, append_sel, append_count);
+#else
+		if (source_validity.AllValid()) {
+			if (append_sel.IsSet()) {
+				if (source_sel.IsSet()) {
+					ComputeStringHeapSizesInternal<true, true, true>(heap_sizes, source_vector_data, append_sel,
+					                                                 append_count);
+				} else {
+					ComputeStringHeapSizesInternal<true, true, false>(heap_sizes, source_vector_data, append_sel,
+					                                                  append_count);
+				}
+			} else {
+				if (source_sel.IsSet()) {
+					ComputeStringHeapSizesInternal<true, false, true>(heap_sizes, source_vector_data, append_sel,
+					                                                  append_count);
+				} else {
+					ComputeStringHeapSizesInternal<true, false, false>(heap_sizes, source_vector_data, append_sel,
+					                                                   append_count);
+				}
+			}
+		} else {
+			if (append_sel.IsSet()) {
+				if (source_sel.IsSet()) {
+					ComputeStringHeapSizesInternal<false, true, true>(heap_sizes, source_vector_data, append_sel,
+					                                                  append_count);
+				} else {
+					ComputeStringHeapSizesInternal<false, true, false>(heap_sizes, source_vector_data, append_sel,
+					                                                   append_count);
+				}
 			} else {
-				heap_sizes[i] += StringHeapSize(NullValue<string_t>());
+				if (source_sel.IsSet()) {
+					ComputeStringHeapSizesInternal<false, false, true>(heap_sizes, source_vector_data, append_sel,
+					                                                   append_count);
+				} else {
+					ComputeStringHeapSizesInternal<false, false, false>(heap_sizes, source_vector_data, append_sel,
+					                                                    append_count);
+				}
 			}
 		}
+#endif
 		break;
 	}
 	case PhysicalType::STRUCT: {
@@ -192,6 +254,32 @@ void TupleDataCollection::ComputeHeapSizes(Vector &heap_sizes_v, const Vector &s
 	}
 }
 
+void TupleDataCollection::SortKeyComputeHeapSizes(TupleDataChunkState &chunk_state, const DataChunk &new_chunk,
+                                                  const SelectionVector &append_sel, const idx_t append_count,
+                                                  const SortKeyType sort_key_type) {
+	D_ASSERT(sort_key_type != SortKeyType::INVALID);
+	D_ASSERT(!SortKeyUtils::IsConstantSize(sort_key_type));
+	D_ASSERT(new_chunk.ColumnCount() == 1);
+	if (new_chunk.data[0].GetType().id() != LogicalTypeId::BLOB) {
+		return;
+	}
+
+	const auto heap_sizes = FlatVector::GetData<idx_t>(chunk_state.heap_sizes);
+
+	const auto &source_vector_data = chunk_state.vector_data[0].unified;
+	const auto &source_sel = *source_vector_data.sel;
+	const auto &source_validity = source_vector_data.validity;
+	const auto source_data = UnifiedVectorFormat::GetData<string_t>(source_vector_data);
+
+	const auto inlined_size = SortKeyUtils::GetInlineLength(sort_key_type);
+	for (idx_t i = 0; i < append_count; i++) {
+		const auto source_idx = source_sel.get_index(append_sel.get_index(i));
+		const auto &string_size = source_data[source_idx].GetSize();
+		// If valid, and string cannot be inlined, we need to allocate heap space
+		heap_sizes[i] = (source_validity.RowIsValid(source_idx) & (string_size > inlined_size)) * string_size;
+	}
+}
+
 void TupleDataCollection::WithinCollectionComputeHeapSizes(Vector &heap_sizes_v, const Vector &source_v,
                                                            TupleDataVectorFormat &source_format,
                                                            const SelectionVector &append_sel, const idx_t append_count,
@@ -231,7 +319,7 @@ void TupleDataCollection::ComputeFixedWithinCollectionHeapSizes(Vector &heap_siz
                                                                 const UnifiedVectorFormat &list_data) {
 	// Parent list data
 	const auto list_sel = *list_data.sel;
-	const auto list_entries = UnifiedVectorFormat::GetData<list_entry_t>(list_data);
+	const auto list_entries = UnifiedVectorFormat::GetDataUnsafe<list_entry_t>(list_data);
 	const auto &list_validity = list_data.validity;
 
 	// Target
@@ -265,7 +353,7 @@ void TupleDataCollection::StringWithinCollectionComputeHeapSizes(Vector &heap_si
                                                                  const UnifiedVectorFormat &list_data) {
 	// Parent list data
 	const auto list_sel = *list_data.sel;
-	const auto list_entries = UnifiedVectorFormat::GetData<list_entry_t>(list_data);
+	const auto list_entries = UnifiedVectorFormat::GetDataUnsafe<list_entry_t>(list_data);
 	const auto &list_validity = list_data.validity;
 
 	// Source
@@ -313,7 +401,7 @@ void TupleDataCollection::StructWithinCollectionComputeHeapSizes(Vector &heap_si
                                                                  const UnifiedVectorFormat &list_data) {
 	// Parent list data
 	const auto list_sel = *list_data.sel;
-	const auto list_entries = UnifiedVectorFormat::GetData<list_entry_t>(list_data);
+	const auto list_entries = UnifiedVectorFormat::GetDataUnsafe<list_entry_t>(list_data);
 	const auto &list_validity = list_data.validity;
 
 	// Target
@@ -361,7 +449,7 @@ static void ApplySliceRecursive(const Vector &source_v, TupleDataVectorFormat &s
 		for (idx_t struct_col_idx = 0; struct_col_idx < struct_sources.size(); struct_col_idx++) {
 			auto &struct_source = *struct_sources[struct_col_idx];
 			auto &struct_format = source_format.children[struct_col_idx];
-#ifdef DEBUG
+#ifdef D_ASSERT_IS_ENABLED
 			D_ASSERT(!struct_format.combined_list_data);
 #endif
 			if (!struct_format.combined_list_data) {
@@ -379,13 +467,13 @@ void TupleDataCollection::CollectionWithinCollectionComputeHeapSizes(Vector &hea
                                                                      const UnifiedVectorFormat &list_data) {
 	// Parent list data
 	const auto list_sel = *list_data.sel;
-	const auto list_entries = UnifiedVectorFormat::GetData<list_entry_t>(list_data);
+	const auto list_entries = UnifiedVectorFormat::GetDataUnsafe<list_entry_t>(list_data);
 	const auto &list_validity = list_data.validity;
 
 	// Source
 	const auto &child_list_data = source_format.unified;
 	const auto child_list_sel = *child_list_data.sel;
-	const auto child_list_entries = UnifiedVectorFormat::GetData<list_entry_t>(child_list_data);
+	const auto child_list_entries = UnifiedVectorFormat::GetDataUnsafe<list_entry_t>(child_list_data);
 	const auto &child_list_validity = child_list_data.validity;
 
 	// Target
@@ -427,7 +515,7 @@ void TupleDataCollection::CollectionWithinCollectionComputeHeapSizes(Vector &hea
 
 	D_ASSERT(source_format.children.size() == 1);
 	auto &child_format = source_format.children[0];
-#ifdef DEBUG
+#ifdef D_ASSERT_IS_ENABLED
 	// In debug mode this should be deleted by ResetCombinedListData
 	D_ASSERT(!child_format.combined_list_data);
 #endif
@@ -559,7 +647,7 @@ static void InitializeValidityMask(const data_ptr_t row_locations[], const idx_t
 
 void TupleDataCollection::Scatter(TupleDataChunkState &chunk_state, const DataChunk &new_chunk,
                                   const SelectionVector &append_sel, const idx_t append_count) const {
-#ifdef DEBUG
+#ifdef D_ASSERT_IS_ENABLED
 	Vector heap_locations_copy(LogicalType::POINTER);
 	if (!layout.AllConstant()) {
 		const auto heap_locations = FlatVector::GetData<data_ptr_t>(chunk_state.heap_locations);
@@ -570,26 +658,35 @@ void TupleDataCollection::Scatter(TupleDataChunkState &chunk_state, const DataCh
 	}
 #endif
 
-	const auto row_locations = FlatVector::GetData<data_ptr_t>(chunk_state.row_locations);
+	if (layout.IsSortKeyLayout()) {
+		const auto &scatter_function = scatter_functions[0];
+		scatter_function.function(new_chunk.data[0], chunk_state.vector_data[0], append_sel, append_count, layout,
+		                          chunk_state.row_locations, chunk_state.heap_locations, 0,
+		                          chunk_state.vector_data[0].unified, scatter_function.child_functions);
+	} else {
+		const auto row_locations = FlatVector::GetData<data_ptr_t>(chunk_state.row_locations);
 
-	// Set the validity mask for each row before inserting data
-	InitializeValidityMask(row_locations, append_count, ValidityBytes::SizeInBytes(layout.ColumnCount()));
+		// Set the validity mask for each row before inserting data
+		if (!layout.AllValid()) {
+			InitializeValidityMask(row_locations, append_count, ValidityBytes::SizeInBytes(layout.ColumnCount()));
+		}
 
-	if (!layout.AllConstant()) {
-		// Set the heap size for each row
-		const auto heap_size_offset = layout.GetHeapSizeOffset();
-		const auto heap_sizes = FlatVector::GetData<idx_t>(chunk_state.heap_sizes);
-		for (idx_t i = 0; i < append_count; i++) {
-			Store<uint32_t>(UnsafeNumericCast<uint32_t>(heap_sizes[i]), row_locations[i] + heap_size_offset);
+		if (!layout.AllConstant()) {
+			// Set the heap size for each row
+			const auto heap_size_offset = layout.GetHeapSizeOffset();
+			const auto heap_sizes = FlatVector::GetData<idx_t>(chunk_state.heap_sizes);
+			for (idx_t i = 0; i < append_count; i++) {
+				Store<idx_t>(heap_sizes[i], row_locations[i] + heap_size_offset);
+			}
 		}
-	}
 
-	// Write the data
-	for (const auto &col_idx : chunk_state.column_ids) {
-		Scatter(chunk_state, new_chunk.data[col_idx], col_idx, append_sel, append_count);
+		// Write the data
+		for (const auto &col_idx : chunk_state.column_ids) {
+			Scatter(chunk_state, new_chunk.data[col_idx], col_idx, append_sel, append_count);
+		}
 	}
 
-#ifdef DEBUG
+#ifdef D_ASSERT_IS_ENABLED
 	// Verify that the size of the data written to the heap is the same as the size we computed it would be
 	if (!layout.AllConstant()) {
 		const auto original_heap_locations = FlatVector::GetData<data_ptr_t>(heap_locations_copy);
@@ -612,18 +709,28 @@ void TupleDataCollection::Scatter(TupleDataChunkState &chunk_state, const Vector
 	                          chunk_state.vector_data[column_id].unified, scatter_function.child_functions);
 }
 
+#ifdef DUCKDB_SMALLER_BINARY
 template <class T>
-static void TupleDataTemplatedScatter(const Vector &, const TupleDataVectorFormat &source_format,
-                                      const SelectionVector &append_sel, const idx_t append_count,
-                                      const TupleDataLayout &layout, const Vector &row_locations,
-                                      Vector &heap_locations, const idx_t col_idx, const UnifiedVectorFormat &,
-                                      const vector<TupleDataScatterFunction> &) {
+#else
+template <class T, bool HAS_APPEND_SEL, bool HAS_SOURCE_SEL, bool ALL_VALID>
+#endif
+static void TupleDataTemplatedScatterInternal(const Vector &, const TupleDataVectorFormat &source_format,
+                                              const SelectionVector &append_sel, const idx_t append_count,
+                                              const TupleDataLayout &layout, const Vector &row_locations,
+                                              Vector &heap_locations, const idx_t col_idx, const UnifiedVectorFormat &,
+                                              const vector<TupleDataScatterFunction> &) {
 	// Source
 	const auto &source_data = source_format.unified;
 	const auto &source_sel = *source_data.sel;
 	const auto data = UnifiedVectorFormat::GetData<T>(source_data);
 	const auto &validity = source_data.validity;
 
+#ifdef DUCKDB_SMALLER_BINARY
+	const auto HAS_APPEND_SEL = append_sel.IsSet();
+	const auto HAS_SOURCE_SEL = source_sel.IsSet();
+	const auto ALL_VALID = validity.AllValid();
+#endif
+
 	// Target
 	const auto target_locations = FlatVector::GetData<data_ptr_t>(row_locations);
 	const auto target_heap_locations = FlatVector::GetData<data_ptr_t>(heap_locations);
@@ -633,21 +740,120 @@ static void TupleDataTemplatedScatter(const Vector &, const TupleDataVectorForma
 	idx_t idx_in_entry;
 	ValidityBytes::GetEntryIndex(col_idx, entry_idx, idx_in_entry);
 
+	const auto column_count = layout.ColumnCount();
 	const auto offset_in_row = layout.GetOffsets()[col_idx];
+
+	const auto null_value = NullValue<T>();
+	for (idx_t i = 0; i < append_count; i++) {
+		const auto append_idx = HAS_APPEND_SEL ? append_sel.get_index_unsafe(i) : i;
+		const auto source_idx = HAS_SOURCE_SEL ? source_sel.get_index_unsafe(append_idx) : append_idx;
+		const auto &target_location = target_locations[i];
+		if (ALL_VALID || validity.RowIsValidUnsafe(source_idx)) {
+			TupleDataValueStore<T>(data[source_idx], target_location, offset_in_row, target_heap_locations[i]);
+		} else {
+			D_ASSERT(!layout.AllValid());
+			TupleDataValueStore<T>(null_value, target_location, offset_in_row, target_heap_locations[i]);
+			ValidityBytes(target_location, column_count).SetInvalidUnsafe(entry_idx, idx_in_entry);
+		}
+	}
+}
+
+template <class T>
+static void TupleDataTemplatedScatter(const Vector &source, const TupleDataVectorFormat &source_format,
+                                      const SelectionVector &append_sel, const idx_t append_count,
+                                      const TupleDataLayout &layout, const Vector &row_locations,
+                                      Vector &heap_locations, const idx_t col_idx, const UnifiedVectorFormat &dummy_arg,
+                                      const vector<TupleDataScatterFunction> &child_functions) {
+#ifdef DUCKDB_SMALLER_BINARY
+	TupleDataTemplatedScatterInternal<T>(source, source_format, append_sel, append_count, layout, row_locations,
+	                                     heap_locations, col_idx, dummy_arg, child_functions);
+#else
+	if (append_sel.IsSet()) {
+		if (source_format.unified.sel->IsSet()) {
+			if (source_format.unified.validity.AllValid()) {
+				TupleDataTemplatedScatterInternal<T, true, true, true>(source, source_format, append_sel, append_count,
+				                                                       layout, row_locations, heap_locations, col_idx,
+				                                                       dummy_arg, child_functions);
+			} else {
+				TupleDataTemplatedScatterInternal<T, true, true, false>(source, source_format, append_sel, append_count,
+				                                                        layout, row_locations, heap_locations, col_idx,
+				                                                        dummy_arg, child_functions);
+			}
+		} else {
+			if (source_format.unified.validity.AllValid()) {
+				TupleDataTemplatedScatterInternal<T, true, false, true>(source, source_format, append_sel, append_count,
+				                                                        layout, row_locations, heap_locations, col_idx,
+				                                                        dummy_arg, child_functions);
+			} else {
+				TupleDataTemplatedScatterInternal<T, true, false, false>(
+				    source, source_format, append_sel, append_count, layout, row_locations, heap_locations, col_idx,
+				    dummy_arg, child_functions);
+			}
+		}
+	} else {
+		if (source_format.unified.sel->IsSet()) {
+			if (source_format.unified.validity.AllValid()) {
+				TupleDataTemplatedScatterInternal<T, false, true, true>(source, source_format, append_sel, append_count,
+				                                                        layout, row_locations, heap_locations, col_idx,
+				                                                        dummy_arg, child_functions);
+			} else {
+				TupleDataTemplatedScatterInternal<T, false, true, false>(
+				    source, source_format, append_sel, append_count, layout, row_locations, heap_locations, col_idx,
+				    dummy_arg, child_functions);
+			}
+		} else {
+			if (source_format.unified.validity.AllValid()) {
+				TupleDataTemplatedScatterInternal<T, false, false, true>(
+				    source, source_format, append_sel, append_count, layout, row_locations, heap_locations, col_idx,
+				    dummy_arg, child_functions);
+			} else {
+				TupleDataTemplatedScatterInternal<T, false, false, false>(
+				    source, source_format, append_sel, append_count, layout, row_locations, heap_locations, col_idx,
+				    dummy_arg, child_functions);
+			}
+		}
+	}
+#endif
+}
+
+template <class T, SortKeyType SORT_KEY_TYPE>
+void TupleDataSortKeyScatter(const Vector &, const TupleDataVectorFormat &source_format,
+                             const SelectionVector &append_sel, const idx_t append_count, const TupleDataLayout &layout,
+                             const Vector &row_locations, Vector &heap_locations, const idx_t,
+                             const UnifiedVectorFormat &, const vector<TupleDataScatterFunction> &) {
+	D_ASSERT(layout.IsSortKeyLayout());
+	D_ASSERT(layout.GetSortKeyType() == SORT_KEY_TYPE);
+	using SORT_KEY = SortKey<SORT_KEY_TYPE>;
+
+	// Source
+	const auto &source_data = source_format.unified;
+	const auto &source_sel = *source_data.sel;
+	const auto data = UnifiedVectorFormat::GetData<T>(source_data);
+	const auto &validity = source_data.validity;
+
+	// Target
+	const auto target_locations = FlatVector::GetData<SORT_KEY *>(row_locations);
+	const auto target_heap_locations = FlatVector::GetData<data_ptr_t>(heap_locations);
+
 	if (validity.AllValid()) {
-		for (idx_t i = 0; i < append_count; i++) {
-			const auto source_idx = source_sel.get_index(append_sel.get_index(i));
-			TupleDataValueStore<T>(data[source_idx], target_locations[i], offset_in_row, target_heap_locations[i]);
+		// Fast path
+		if (!append_sel.IsSet() && !source_sel.IsSet()) {
+			for (idx_t i = 0; i < append_count; i++) {
+				target_locations[i]->Construct(data[i], target_heap_locations[i]);
+			}
+		} else {
+			for (idx_t i = 0; i < append_count; i++) {
+				const auto source_idx = source_sel.get_index(append_sel.get_index(i));
+				target_locations[i]->Construct(data[source_idx], target_heap_locations[i]);
+			}
 		}
 	} else {
 		for (idx_t i = 0; i < append_count; i++) {
 			const auto source_idx = source_sel.get_index(append_sel.get_index(i));
-			if (validity.RowIsValid(source_idx)) {
-				TupleDataValueStore<T>(data[source_idx], target_locations[i], offset_in_row, target_heap_locations[i]);
-			} else {
-				TupleDataValueStore<T>(NullValue<T>(), target_locations[i], offset_in_row, target_heap_locations[i]);
-				ValidityBytes(target_locations[i], layout.ColumnCount()).SetInvalidUnsafe(entry_idx, idx_in_entry);
-			}
+			// validity.AllValid() may not be true when doing aggressive vector verification
+			// but the actual values should always all be valid
+			D_ASSERT(validity.RowIsValid(source_idx));
+			target_locations[i]->Construct(data[source_idx], target_heap_locations[i]);
 		}
 	}
 }
@@ -672,6 +878,7 @@ static void TupleDataStructScatter(const Vector &source, const TupleDataVectorFo
 
 	// Set validity of the STRUCT in this layout
 	if (!validity.AllValid()) {
+		D_ASSERT(!layout.AllValid());
 		for (idx_t i = 0; i < append_count; i++) {
 			const auto source_idx = source_sel.get_index(append_sel.get_index(i));
 			if (!validity.RowIsValid(source_idx)) {
@@ -718,7 +925,7 @@ static void TupleDataListScatter(const Vector &source, const TupleDataVectorForm
 	// Source
 	const auto &source_data = source_format.unified;
 	const auto &source_sel = *source_data.sel;
-	const auto data = UnifiedVectorFormat::GetData<list_entry_t>(source_data);
+	const auto data = UnifiedVectorFormat::GetDataUnsafe<list_entry_t>(source_data);
 	const auto &validity = source_data.validity;
 
 	// Target
@@ -742,6 +949,7 @@ static void TupleDataListScatter(const Vector &source, const TupleDataVectorForm
 			Store<uint64_t>(data[source_idx].length, target_heap_location);
 			target_heap_location += sizeof(uint64_t);
 		} else {
+			D_ASSERT(!layout.AllValid());
 			ValidityBytes(target_locations[i], layout.ColumnCount()).SetInvalidUnsafe(entry_idx, idx_in_entry);
 		}
 	}
@@ -767,7 +975,7 @@ static void TupleDataArrayScatter(const Vector &source, const TupleDataVectorFor
 	// The Array vector has fake list_entry_t's set by this point, so this is fine
 	const auto &source_data = source_format.unified;
 	const auto &source_sel = *source_data.sel;
-	const auto data = UnifiedVectorFormat::GetData<list_entry_t>(source_data);
+	const auto data = UnifiedVectorFormat::GetDataUnsafe<list_entry_t>(source_data);
 	const auto &validity = source_data.validity;
 
 	// Target
@@ -791,6 +999,7 @@ static void TupleDataArrayScatter(const Vector &source, const TupleDataVectorFor
 			Store<uint64_t>(data[source_idx].length, target_heap_location);
 			target_heap_location += sizeof(uint64_t);
 		} else {
+			D_ASSERT(!layout.AllValid());
 			ValidityBytes(target_locations[i], layout.ColumnCount()).SetInvalidUnsafe(entry_idx, idx_in_entry);
 		}
 	}
@@ -815,7 +1024,7 @@ static void TupleDataTemplatedWithinCollectionScatter(const Vector &, const Tupl
                                                       const vector<TupleDataScatterFunction> &) {
 	// Parent list data
 	const auto &list_sel = *list_data.sel;
-	const auto list_entries = UnifiedVectorFormat::GetData<list_entry_t>(list_data);
+	const auto list_entries = UnifiedVectorFormat::GetDataUnsafe<list_entry_t>(list_data);
 	const auto &list_validity = list_data.validity;
 
 	// Source
@@ -873,7 +1082,7 @@ static void TupleDataStructWithinCollectionScatter(const Vector &source, const T
                                                    const vector<TupleDataScatterFunction> &child_functions) {
 	// Parent list data
 	const auto &list_sel = *list_data.sel;
-	const auto list_entries = UnifiedVectorFormat::GetData<list_entry_t>(list_data);
+	const auto list_entries = UnifiedVectorFormat::GetDataUnsafe<list_entry_t>(list_data);
 	const auto &list_validity = list_data.validity;
 
 	// Source
@@ -936,13 +1145,13 @@ static void TupleDataCollectionWithinCollectionScatter(const Vector &child_list,
                                                        const vector<TupleDataScatterFunction> &child_functions) {
 	// Parent list data
 	const auto &list_sel = *list_data.sel;
-	const auto list_entries = UnifiedVectorFormat::GetData<list_entry_t>(list_data);
+	const auto list_entries = UnifiedVectorFormat::GetDataUnsafe<list_entry_t>(list_data);
 	const auto &list_validity = list_data.validity;
 
 	// Source
 	const auto &child_list_data = child_list_format.unified;
 	const auto &child_list_sel = *child_list_data.sel;
-	const auto child_list_entries = UnifiedVectorFormat::GetData<list_entry_t>(child_list_data);
+	const auto child_list_entries = UnifiedVectorFormat::GetDataUnsafe<list_entry_t>(child_list_data);
 	const auto &child_list_validity = child_list_data.validity;
 
 	// Target
@@ -1072,6 +1281,55 @@ TupleDataScatterFunction TupleDataCollection::GetScatterFunction(const LogicalTy
 	return result;
 }
 
+template <class T>
+TupleDataScatterFunction GetSortKeyScatterFunctionInternal(SortKeyType sort_key_type) {
+	TupleDataScatterFunction result;
+	switch (sort_key_type) {
+	case SortKeyType::NO_PAYLOAD_FIXED_8:
+		result.function = TupleDataSortKeyScatter<T, SortKeyType::NO_PAYLOAD_FIXED_8>;
+		break;
+	case SortKeyType::NO_PAYLOAD_FIXED_16:
+		result.function = TupleDataSortKeyScatter<T, SortKeyType::NO_PAYLOAD_FIXED_16>;
+		break;
+	case SortKeyType::NO_PAYLOAD_FIXED_24:
+		result.function = TupleDataSortKeyScatter<T, SortKeyType::NO_PAYLOAD_FIXED_24>;
+		break;
+	case SortKeyType::NO_PAYLOAD_FIXED_32:
+		result.function = TupleDataSortKeyScatter<T, SortKeyType::NO_PAYLOAD_FIXED_32>;
+		break;
+	case SortKeyType::NO_PAYLOAD_VARIABLE_32:
+		result.function = TupleDataSortKeyScatter<T, SortKeyType::NO_PAYLOAD_VARIABLE_32>;
+		break;
+	case SortKeyType::PAYLOAD_FIXED_16:
+		result.function = TupleDataSortKeyScatter<T, SortKeyType::PAYLOAD_FIXED_16>;
+		break;
+	case SortKeyType::PAYLOAD_FIXED_24:
+		result.function = TupleDataSortKeyScatter<T, SortKeyType::PAYLOAD_FIXED_24>;
+		break;
+	case SortKeyType::PAYLOAD_FIXED_32:
+		result.function = TupleDataSortKeyScatter<T, SortKeyType::PAYLOAD_FIXED_32>;
+		break;
+	case SortKeyType::PAYLOAD_VARIABLE_32:
+		result.function = TupleDataSortKeyScatter<T, SortKeyType::PAYLOAD_VARIABLE_32>;
+		break;
+	default:
+		throw NotImplementedException("GetSortKeyScatterFunction for %s", EnumUtil::ToString(sort_key_type));
+	}
+	return result;
+}
+
+TupleDataScatterFunction TupleDataCollection::GetSortKeyScatterFunction(const LogicalType &type,
+                                                                        SortKeyType sort_key_type) {
+	switch (type.id()) {
+	case LogicalTypeId::BIGINT:
+		return GetSortKeyScatterFunctionInternal<int64_t>(sort_key_type);
+	case LogicalTypeId::BLOB:
+		return GetSortKeyScatterFunctionInternal<string_t>(sort_key_type);
+	default:
+		throw NotImplementedException("TupleDataCollection::GetSortKeyScatterFunction for %s", type.ToString());
+	}
+}
+
 //-------------------------------------------------------------------------------
 // Gather
 //-------------------------------------------------------------------------------
@@ -1107,11 +1365,20 @@ void TupleDataCollection::Gather(Vector &row_locations, const SelectionVector &s
 	Vector::Verify(result, target_sel, scan_count);
 }
 
+#ifdef DUCKDB_SMALLER_BINARY
 template <class T>
-static void TupleDataTemplatedGather(const TupleDataLayout &layout, Vector &row_locations, const idx_t col_idx,
-                                     const SelectionVector &scan_sel, const idx_t scan_count, Vector &target,
-                                     const SelectionVector &target_sel, optional_ptr<Vector>,
-                                     const vector<TupleDataGatherFunction> &) {
+#else
+template <class T, bool HAS_SCAN_SEL, bool HAS_TARGET_SEL, bool ALL_VALID>
+#endif
+static void TupleDataTemplatedGatherInternal(const TupleDataLayout &layout, Vector &row_locations, const idx_t col_idx,
+                                             const SelectionVector &scan_sel, const idx_t scan_count, Vector &target,
+                                             const SelectionVector &target_sel, optional_ptr<Vector>,
+                                             const vector<TupleDataGatherFunction> &) {
+#ifdef DUCKDB_SMALLER_BINARY
+	const bool HAS_SCAN_SEL = scan_sel.IsSet();
+	const bool HAS_TARGET_SEL = target_sel.IsSet();
+	const bool ALL_VALID = layout.AllValid();
+#endif
 	// Source
 	const auto source_locations = FlatVector::GetData<data_ptr_t>(row_locations);
 
@@ -1125,22 +1392,87 @@ static void TupleDataTemplatedGather(const TupleDataLayout &layout, Vector &row_
 	ValidityBytes::GetEntryIndex(col_idx, entry_idx, idx_in_entry);
 
 	const auto offset_in_row = layout.GetOffsets()[col_idx];
+	const auto column_count = layout.ColumnCount();
 	for (idx_t i = 0; i < scan_count; i++) {
-		const auto &source_row = source_locations[scan_sel.get_index(i)];
-		const auto target_idx = target_sel.get_index(i);
+		const auto source_idx = HAS_SCAN_SEL ? scan_sel.get_index_unsafe(i) : i;
+		const auto target_idx = HAS_TARGET_SEL ? target_sel.get_index_unsafe(i) : i;
+		const auto &source_row = source_locations[source_idx];
 		target_data[target_idx] = Load<T>(source_row + offset_in_row);
-		ValidityBytes row_mask(source_row, layout.ColumnCount());
-		if (!row_mask.RowIsValid(row_mask.GetValidityEntryUnsafe(entry_idx), idx_in_entry)) {
+		if (!ALL_VALID &&
+		    !ValidityBytes::RowIsValid(ValidityBytes(source_row, column_count).GetValidityEntryUnsafe(entry_idx),
+		                               idx_in_entry)) {
 			target_validity.SetInvalid(target_idx);
-		}
-#ifdef DEBUG
-		else {
+		} else {
 			TupleDataValueVerify<T>(target.GetType(), target_data[target_idx]);
 		}
-#endif
 	}
 }
 
+template <class T>
+static void TupleDataTemplatedGather(const TupleDataLayout &layout, Vector &row_locations, const idx_t col_idx,
+                                     const SelectionVector &scan_sel, const idx_t scan_count, Vector &target,
+                                     const SelectionVector &target_sel, optional_ptr<Vector> list_vector,
+                                     const vector<TupleDataGatherFunction> &child_functions) {
+#ifdef DUCKDB_SMALLER_BINARY
+	TupleDataTemplatedGatherInternal<T>(layout, row_locations, col_idx, scan_sel, scan_count, target, target_sel,
+	                                    list_vector, child_functions);
+#else
+	if (scan_sel.IsSet()) {
+		if (target_sel.IsSet()) {
+			if (layout.AllValid()) {
+				TupleDataTemplatedGatherInternal<T, true, true, true>(layout, row_locations, col_idx, scan_sel,
+				                                                      scan_count, target, target_sel, list_vector,
+				                                                      child_functions);
+			} else {
+				TupleDataTemplatedGatherInternal<T, true, true, false>(layout, row_locations, col_idx, scan_sel,
+				                                                       scan_count, target, target_sel, list_vector,
+				                                                       child_functions);
+			}
+		} else {
+			if (layout.AllValid()) {
+				TupleDataTemplatedGatherInternal<T, true, false, true>(layout, row_locations, col_idx, scan_sel,
+				                                                       scan_count, target, target_sel, list_vector,
+				                                                       child_functions);
+			} else {
+				TupleDataTemplatedGatherInternal<T, true, false, false>(layout, row_locations, col_idx, scan_sel,
+				                                                        scan_count, target, target_sel, list_vector,
+				                                                        child_functions);
+			}
+		}
+	} else {
+		if (target_sel.IsSet()) {
+			if (layout.AllValid()) {
+				TupleDataTemplatedGatherInternal<T, false, true, true>(layout, row_locations, col_idx, scan_sel,
+				                                                       scan_count, target, target_sel, list_vector,
+				                                                       child_functions);
+			} else {
+				TupleDataTemplatedGatherInternal<T, false, true, false>(layout, row_locations, col_idx, scan_sel,
+				                                                        scan_count, target, target_sel, list_vector,
+				                                                        child_functions);
+			}
+		} else {
+			if (layout.AllValid()) {
+				TupleDataTemplatedGatherInternal<T, false, false, true>(layout, row_locations, col_idx, scan_sel,
+				                                                        scan_count, target, target_sel, list_vector,
+				                                                        child_functions);
+			} else {
+				TupleDataTemplatedGatherInternal<T, false, false, false>(layout, row_locations, col_idx, scan_sel,
+				                                                         scan_count, target, target_sel, list_vector,
+				                                                         child_functions);
+			}
+		}
+	}
+#endif
+}
+
+template <class T, SortKeyType SORT_KEY_TYPE>
+void TupleDataSortKeyGather(const TupleDataLayout &layout, Vector &row_locations, const idx_t col_idx,
+                            const SelectionVector &scan_sel, const idx_t scan_count, Vector &target,
+                            const SelectionVector &target_sel, optional_ptr<Vector>,
+                            const vector<TupleDataGatherFunction> &) {
+	throw NotImplementedException("Unimplemented type for TupleDataSortKeyGather");
+}
+
 static void TupleDataStructGather(const TupleDataLayout &layout, Vector &row_locations, const idx_t col_idx,
                                   const SelectionVector &scan_sel, const idx_t scan_count, Vector &target,
                                   const SelectionVector &target_sel, optional_ptr<Vector> dummy_vector,
@@ -1165,8 +1497,9 @@ static void TupleDataStructGather(const TupleDataLayout &layout, Vector &row_loc
 		const auto &source_row = source_locations[source_idx];
 
 		// Set the validity
-		ValidityBytes row_mask(source_row, layout.ColumnCount());
-		if (!row_mask.RowIsValid(row_mask.GetValidityEntryUnsafe(entry_idx), idx_in_entry)) {
+		if (!layout.AllValid() &&
+		    !ValidityBytes::RowIsValid(
+		        ValidityBytes(source_row, layout.ColumnCount()).GetValidityEntryUnsafe(entry_idx), idx_in_entry)) {
 			const auto target_idx = target_sel.get_index(i);
 			target_validity.SetInvalid(target_idx);
 		}
@@ -1201,7 +1534,7 @@ static void TupleDataListGather(const TupleDataLayout &layout, Vector &row_locat
 	const auto source_locations = FlatVector::GetData<data_ptr_t>(row_locations);
 
 	// Target
-	const auto target_list_entries = FlatVector::GetData<list_entry_t>(target);
+	const auto target_list_entries = FlatVector::GetDataUnsafe<list_entry_t>(target);
 	auto &target_list_validity = FlatVector::Validity(target);
 
 	// Precompute mask indexes
@@ -1218,10 +1551,11 @@ static void TupleDataListGather(const TupleDataLayout &layout, Vector &row_locat
 	uint64_t target_list_offset = list_size_before;
 	for (idx_t i = 0; i < scan_count; i++) {
 		const auto &source_row = source_locations[scan_sel.get_index(i)];
-		ValidityBytes row_mask(source_row, layout.ColumnCount());
 
 		const auto target_idx = target_sel.get_index(i);
-		if (row_mask.RowIsValid(row_mask.GetValidityEntryUnsafe(entry_idx), idx_in_entry)) {
+		if (layout.AllValid() ||
+		    ValidityBytes::RowIsValid(ValidityBytes(source_row, layout.ColumnCount()).GetValidityEntryUnsafe(entry_idx),
+		                              idx_in_entry)) {
 			auto &source_heap_location = source_heap_locations[i];
 			source_heap_location = Load<data_ptr_t>(source_row + offset_in_row);
 
@@ -1258,7 +1592,7 @@ TupleDataTemplatedWithinCollectionGather(const TupleDataLayout &, Vector &heap_l
                                          const SelectionVector &target_sel, optional_ptr<Vector> list_vector,
                                          const vector<TupleDataGatherFunction> &) {
 	// List parent
-	const auto list_entries = FlatVector::GetData<list_entry_t>(*list_vector);
+	const auto list_entries = FlatVector::GetDataUnsafe<list_entry_t>(*list_vector);
 	const auto &list_validity = FlatVector::Validity(*list_vector);
 
 	// Source
@@ -1310,7 +1644,7 @@ static void TupleDataStructWithinCollectionGather(const TupleDataLayout &layout,
                                                   const SelectionVector &target_sel, optional_ptr<Vector> list_vector,
                                                   const vector<TupleDataGatherFunction> &child_functions) {
 	// List parent
-	const auto list_entries = FlatVector::GetData<list_entry_t>(*list_vector);
+	const auto list_entries = FlatVector::GetDataUnsafe<list_entry_t>(*list_vector);
 	const auto &list_validity = FlatVector::Validity(*list_vector);
 
 	// Source
@@ -1362,21 +1696,21 @@ static void TupleDataCollectionWithinCollectionGather(const TupleDataLayout &lay
                                                       optional_ptr<Vector> list_vector,
                                                       const vector<TupleDataGatherFunction> &child_functions) {
 	// List parent
-	const auto list_entries = FlatVector::GetData<list_entry_t>(*list_vector);
+	const auto list_entries = FlatVector::GetDataUnsafe<list_entry_t>(*list_vector);
 	const auto &list_validity = FlatVector::Validity(*list_vector);
 
 	// Source
 	const auto source_heap_locations = FlatVector::GetData<data_ptr_t>(heap_locations);
 
 	// Target
-	const auto target_list_entries = FlatVector::GetData<list_entry_t>(target);
+	const auto target_list_entries = FlatVector::GetDataUnsafe<list_entry_t>(target);
 	auto &target_validity = FlatVector::Validity(target);
 	const auto child_list_size_before = ListVector::GetListSize(target);
 
 	// We need to create a vector that has the combined list sizes (hugeint_t has same size as list_entry_t)
 	Vector combined_list_vector(LogicalType::HUGEINT);
 	FlatVector::SetValidity(combined_list_vector, list_validity); // Has same validity as list parent
-	const auto combined_list_entries = FlatVector::GetData<list_entry_t>(combined_list_vector);
+	const auto combined_list_entries = FlatVector::GetDataUnsafe<list_entry_t>(combined_list_vector);
 
 	uint64_t target_offset = list_size_before;
 	uint64_t target_child_offset = child_list_size_before;
@@ -1581,4 +1915,53 @@ TupleDataGatherFunction TupleDataCollection::GetGatherFunction(const LogicalType
 	return TupleDataGetGatherFunctionInternal(type, false);
 }
 
+template <class T>
+TupleDataGatherFunction GetSortKeyGatherFunctionInternal(SortKeyType sort_key_type) {
+	TupleDataGatherFunction result;
+	switch (sort_key_type) {
+	case SortKeyType::NO_PAYLOAD_FIXED_8:
+		result.function = TupleDataSortKeyGather<T, SortKeyType::NO_PAYLOAD_FIXED_8>;
+		break;
+	case SortKeyType::NO_PAYLOAD_FIXED_16:
+		result.function = TupleDataSortKeyGather<T, SortKeyType::NO_PAYLOAD_FIXED_16>;
+		break;
+	case SortKeyType::NO_PAYLOAD_FIXED_24:
+		result.function = TupleDataSortKeyGather<T, SortKeyType::NO_PAYLOAD_FIXED_24>;
+		break;
+	case SortKeyType::NO_PAYLOAD_FIXED_32:
+		result.function = TupleDataSortKeyGather<T, SortKeyType::NO_PAYLOAD_FIXED_32>;
+		break;
+	case SortKeyType::NO_PAYLOAD_VARIABLE_32:
+		result.function = TupleDataSortKeyGather<T, SortKeyType::NO_PAYLOAD_VARIABLE_32>;
+		break;
+	case SortKeyType::PAYLOAD_FIXED_16:
+		result.function = TupleDataSortKeyGather<T, SortKeyType::PAYLOAD_FIXED_16>;
+		break;
+	case SortKeyType::PAYLOAD_FIXED_24:
+		result.function = TupleDataSortKeyGather<T, SortKeyType::PAYLOAD_FIXED_24>;
+		break;
+	case SortKeyType::PAYLOAD_FIXED_32:
+		result.function = TupleDataSortKeyGather<T, SortKeyType::PAYLOAD_FIXED_32>;
+		break;
+	case SortKeyType::PAYLOAD_VARIABLE_32:
+		result.function = TupleDataSortKeyGather<T, SortKeyType::PAYLOAD_VARIABLE_32>;
+		break;
+	default:
+		throw NotImplementedException("GetSortKeyGatherFunction for %s", EnumUtil::ToString(sort_key_type));
+	}
+	return result;
+}
+
+TupleDataGatherFunction TupleDataCollection::GetSortKeyGatherFunction(const LogicalType &type,
+                                                                      SortKeyType sort_key_type) {
+	switch (type.id()) {
+	case LogicalTypeId::BIGINT:
+		return GetSortKeyGatherFunctionInternal<int64_t>(sort_key_type);
+	case LogicalTypeId::BLOB:
+		return GetSortKeyGatherFunctionInternal<string_t>(sort_key_type);
+	default:
+		throw NotImplementedException("TupleDataCollection::GetSortKeyGatherFunction for %s", type.ToString());
+	}
+}
+
 } // namespace duckdb