duckdb 1.3.4 → 1.4.0

package/src/duckdb/src/common/types/row/tuple_data_collection.cpp

@@ -25,15 +25,24 @@ void TupleDataCollection::Initialize() {
 	D_ASSERT(!layout.GetTypes().empty());
 	this->count = 0;
 	this->data_size = 0;
-	scatter_functions.reserve(layout.ColumnCount());
-	gather_functions.reserve(layout.ColumnCount());
-	for (idx_t col_idx = 0; col_idx < layout.ColumnCount(); col_idx++) {
-		auto &type = layout.GetTypes()[col_idx];
-		scatter_functions.emplace_back(GetScatterFunction(type));
-		gather_functions.emplace_back(GetGatherFunction(type));
+	if (layout.IsSortKeyLayout()) {
+		scatter_functions.emplace_back(GetSortKeyScatterFunction(layout.GetTypes()[0], layout.GetSortKeyType()));
+		gather_functions.emplace_back(GetSortKeyGatherFunction(layout.GetTypes()[0], layout.GetSortKeyType()));
+	} else {
+		scatter_functions.reserve(layout.ColumnCount());
+		gather_functions.reserve(layout.ColumnCount());
+		for (idx_t col_idx = 0; col_idx < layout.ColumnCount(); col_idx++) {
+			auto &type = layout.GetTypes()[col_idx];
+			scatter_functions.emplace_back(GetScatterFunction(type));
+			gather_functions.emplace_back(GetGatherFunction(type));
+		}
 	}
 }
 
+unique_ptr<TupleDataCollection> TupleDataCollection::CreateUnique() const {
+	return make_uniq<TupleDataCollection>(allocator->GetBufferManager(), layout_ptr);
+}
+
 void GetAllColumnIDsInternal(vector<column_t> &column_ids, const idx_t column_count) {
 	column_ids.reserve(column_count);
 	for (idx_t col_idx = 0; col_idx < column_count; col_idx++) {
@@ -45,10 +54,18 @@ void TupleDataCollection::GetAllColumnIDs(vector<column_t> &column_ids) {
 	GetAllColumnIDsInternal(column_ids, layout.ColumnCount());
 }
 
+shared_ptr<TupleDataLayout> TupleDataCollection::GetLayoutPtr() const {
+	return layout_ptr;
+}
+
 const TupleDataLayout &TupleDataCollection::GetLayout() const {
 	return layout;
 }
 
+idx_t TupleDataCollection::TuplesPerBlock() const {
+	return allocator->GetBufferManager().GetBlockSize() / layout.GetRowWidth();
+}
+
 const idx_t &TupleDataCollection::Count() const {
 	return count;
 }
@@ -56,22 +73,18 @@ const idx_t &TupleDataCollection::Count() const {
 idx_t TupleDataCollection::ChunkCount() const {
 	idx_t total_chunk_count = 0;
 	for (const auto &segment : segments) {
-		total_chunk_count += segment.ChunkCount();
+		total_chunk_count += segment->ChunkCount();
 	}
 	return total_chunk_count;
 }
 
 idx_t TupleDataCollection::SizeInBytes() const {
-	idx_t total_size = 0;
-	for (const auto &segment : segments) {
-		total_size += segment.SizeInBytes();
-	}
-	return total_size;
+	return data_size;
 }
 
 void TupleDataCollection::Unpin() {
 	for (auto &segment : segments) {
-		segment.Unpin();
+		segment->Unpin();
 	}
 }
 
@@ -82,9 +95,53 @@ void TupleDataCollection::SetPartitionIndex(const idx_t index) {
 	allocator->SetPartitionIndex(index);
 }
 
+vector<data_ptr_t> TupleDataCollection::GetRowBlockPointers() const {
+	D_ASSERT(segments.size() == 1);
+	const auto &segment = *segments[0];
+	vector<data_ptr_t> result;
+	result.reserve(segment.pinned_row_handles.size());
+	for (const auto &pinned_row_handle : segment.pinned_row_handles) {
+		result.emplace_back(pinned_row_handle.Ptr());
+	}
+	return result;
+}
+
+void TupleDataCollection::DestroyChunks(const idx_t chunk_idx_begin, const idx_t chunk_idx_end) {
+	D_ASSERT(segments.size() == 1); // Assume 1 segment for now (multi-segment destroys can be implemented if needed)
+	D_ASSERT(chunk_idx_begin <= chunk_idx_end && chunk_idx_end <= ChunkCount());
+	auto &segment = *segments[0];
+	auto &chunk_begin = segment.chunks[chunk_idx_begin];
+
+	const auto row_block_begin = chunk_begin.row_block_ids.Start();
+	if (chunk_idx_end == ChunkCount()) {
+		segment.allocator->DestroyRowBlocks(row_block_begin, segment.allocator->RowBlockCount());
+	} else {
+		auto &chunk_end = segment.chunks[chunk_idx_end];
+		const auto row_block_end = chunk_end.row_block_ids.Start();
+		segment.allocator->DestroyRowBlocks(row_block_begin, row_block_end);
+	}
+
+	if (!layout.AllConstant()) {
+		if (chunk_begin.heap_block_ids.Empty()) {
+			return;
+		}
+		const auto heap_block_begin = chunk_begin.heap_block_ids.Start();
+		if (chunk_idx_end == ChunkCount()) {
+			segment.allocator->DestroyHeapBlocks(heap_block_begin, segment.allocator->HeapBlockCount());
+		} else {
+			auto &chunk_end = segment.chunks[chunk_idx_end];
+			if (chunk_end.heap_block_ids.Empty()) {
+				return;
+			}
+			const auto heap_block_end = chunk_end.heap_block_ids.Start();
+			segment.allocator->DestroyHeapBlocks(heap_block_begin, heap_block_end);
+		}
+	}
+}
+
 // LCOV_EXCL_START
 void VerifyAppendColumns(const TupleDataLayout &layout, const vector<column_t> &column_ids) {
-#ifdef DEBUG
+#ifdef D_ASSERT_IS_ENABLED
 	for (idx_t col_idx = 0; col_idx < layout.ColumnCount(); col_idx++) {
 		if (std::find(column_ids.begin(), column_ids.end(), col_idx) != column_ids.end()) {
 			continue;
@@ -123,7 +180,7 @@ void TupleDataCollection::InitializeAppend(TupleDataAppendState &append_state, v
 void TupleDataCollection::InitializeAppend(TupleDataPinState &pin_state, TupleDataPinProperties properties) {
 	pin_state.properties = properties;
 	if (segments.empty()) {
-		segments.emplace_back(allocator);
+		segments.emplace_back(make_unsafe_uniq<TupleDataSegment>(allocator));
 	}
 }
 
@@ -216,7 +273,12 @@ void TupleDataCollection::AppendUnified(TupleDataPinState &pin_state, TupleDataC
 	}
 
 	if (!layout.AllConstant()) {
-		TupleDataCollection::ComputeHeapSizes(chunk_state, new_chunk, append_sel, actual_append_count);
+		if (layout.IsSortKeyLayout()) {
+			TupleDataCollection::SortKeyComputeHeapSizes(chunk_state, new_chunk, append_sel, actual_append_count,
+			                                             layout.GetSortKeyType());
+		} else {
+			TupleDataCollection::ComputeHeapSizes(chunk_state, new_chunk, append_sel, actual_append_count);
+		}
 	}
 
 	Build(pin_state, chunk_state, 0, actual_append_count);
@@ -289,7 +351,7 @@ void TupleDataCollection::GetVectorData(const TupleDataChunkState &chunk_state,
 
 void TupleDataCollection::Build(TupleDataPinState &pin_state, TupleDataChunkState &chunk_state,
                                 const idx_t append_offset, const idx_t append_count) {
-	auto &segment = segments.back();
+	auto &segment = *segments.back();
 	const auto size_before = segment.SizeInBytes();
 	segment.allocator->Build(segment, pin_state, chunk_state, append_offset, append_count);
 	data_size += segment.SizeInBytes() - size_before;
@@ -300,10 +362,10 @@ void TupleDataCollection::Build(TupleDataPinState &pin_state, TupleDataChunkStat
 // LCOV_EXCL_START
 void VerifyHeapSizes(const data_ptr_t source_locations[], const idx_t heap_sizes[], const SelectionVector &append_sel,
                      const idx_t append_count, const idx_t heap_size_offset) {
-#ifdef DEBUG
+#ifdef D_ASSERT_IS_ENABLED
 	for (idx_t i = 0; i < append_count; i++) {
 		auto idx = append_sel.get_index(i);
-		const auto stored_heap_size = Load<uint32_t>(source_locations[idx] + heap_size_offset);
+		const auto stored_heap_size = Load<idx_t>(source_locations[idx] + heap_size_offset);
 		D_ASSERT(stored_heap_size == heap_sizes[idx]);
 	}
 #endif
@@ -317,9 +379,15 @@ void TupleDataCollection::CopyRows(TupleDataChunkState &chunk_state, TupleDataCh
 
 	// Copy rows
 	const auto row_width = layout.GetRowWidth();
-	for (idx_t i = 0; i < append_count; i++) {
-		auto idx = append_sel.get_index(i);
-		FastMemcpy(target_locations[i], source_locations[idx], row_width);
+	if (append_sel.IsSet()) {
+		for (idx_t i = 0; i < append_count; i++) {
+			const auto idx = append_sel[i];
+			FastMemcpy(target_locations[i], source_locations[idx], row_width);
+		}
+	} else {
+		for (idx_t i = 0; i < append_count; i++) {
+			FastMemcpy(target_locations[i], source_locations[i], row_width);
+		}
 	}
 
 	// Copy heap if we need to
@@ -331,18 +399,32 @@ void TupleDataCollection::CopyRows(TupleDataChunkState &chunk_state, TupleDataCh
 
 		// Check if we need to copy anything at all
 		idx_t total_heap_size = 0;
-		for (idx_t i = 0; i < append_count; i++) {
-			auto idx = append_sel.get_index(i);
-			total_heap_size += heap_sizes[idx];
+		if (!append_sel.IsSet()) {
+			// Fast path
+			for (idx_t i = 0; i < append_count; i++) {
+				total_heap_size += heap_sizes[i];
+			}
+		} else {
+			for (idx_t i = 0; i < append_count; i++) {
+				auto idx = append_sel.get_index(i);
+				total_heap_size += heap_sizes[idx];
+			}
 		}
 		if (total_heap_size == 0) {
 			return;
 		}
 
 		// Copy heap
-		for (idx_t i = 0; i < append_count; i++) {
-			auto idx = append_sel.get_index(i);
-			FastMemcpy(target_heap_locations[i], source_heap_locations[idx], heap_sizes[idx]);
+		if (!append_sel.IsSet()) {
+			// Fast path
+			for (idx_t i = 0; i < append_count; i++) {
+				FastMemcpy(target_heap_locations[i], source_heap_locations[i], heap_sizes[i]);
+			}
+		} else {
+			for (idx_t i = 0; i < append_count; i++) {
+				auto idx = append_sel.get_index(i);
+				FastMemcpy(target_heap_locations[i], source_heap_locations[idx], heap_sizes[idx]);
+			}
 		}
 
 		// Recompute pointers after copying the data
@@ -351,6 +433,28 @@ void TupleDataCollection::CopyRows(TupleDataChunkState &chunk_state, TupleDataCh
 	}
 }
 
+void TupleDataCollection::FindHeapPointers(TupleDataChunkState &chunk_state, const idx_t chunk_count) const {
+	D_ASSERT(!layout.AllConstant());
+	const auto row_locations = FlatVector::GetData<data_ptr_t>(chunk_state.row_locations);
+	const auto heap_sizes = FlatVector::GetData<idx_t>(chunk_state.heap_sizes);
+
+	auto &not_found = chunk_state.utility;
+	idx_t not_found_count = 0;
+
+	const auto &heap_size_offset = layout.GetHeapSizeOffset();
+	for (idx_t i = 0; i < chunk_count; i++) {
+		const auto &row_location = row_locations[i];
+		auto &heap_size = heap_sizes[i];
+
+		heap_size = Load<idx_t>(row_location + heap_size_offset);
+		if (heap_size != 0) {
+			not_found.set_index(not_found_count++, i);
+		}
+	}
+
+	TupleDataAllocator::FindHeapPointers(chunk_state, not_found, not_found_count, layout, 0);
+}
+
 void TupleDataCollection::Combine(TupleDataCollection &other) {
 	if (other.count == 0) {
 		return;
@@ -365,9 +469,9 @@ void TupleDataCollection::Combine(TupleDataCollection &other) {
 	other.Reset();
 }
 
-void TupleDataCollection::AddSegment(TupleDataSegment &&segment) {
-	count += segment.count;
-	data_size += segment.data_size;
+void TupleDataCollection::AddSegment(unsafe_unique_ptr<TupleDataSegment> segment) {
+	count += segment->count;
+	data_size += segment->data_size;
 	segments.emplace_back(std::move(segment));
 	Verify();
 }
@@ -458,19 +562,26 @@ void TupleDataCollection::InitializeScan(TupleDataParallelScanState &state, vect
 	InitializeScan(state.scan_state, std::move(column_ids), properties);
 }
 
+idx_t TupleDataCollection::FetchChunk(TupleDataScanState &state, const idx_t segment_idx, const idx_t chunk_idx,
+                                      const bool init_heap) {
+	auto &segment = *segments[segment_idx];
+	allocator->InitializeChunkState(segment, state.pin_state, state.chunk_state, chunk_idx, init_heap);
+	return segment.chunks[chunk_idx].count;
+}
+
 bool TupleDataCollection::Scan(TupleDataScanState &state, DataChunk &result) {
 	const auto segment_index_before = state.segment_index;
 	idx_t segment_index;
 	idx_t chunk_index;
 	if (!NextScanIndex(state, segment_index, chunk_index)) {
 		if (!segments.empty()) {
-			FinalizePinState(state.pin_state, segments[segment_index_before]);
+			FinalizePinState(state.pin_state, *segments[segment_index_before]);
 		}
 		result.SetCardinality(0);
 		return false;
 	}
 	if (segment_index_before != DConstants::INVALID_INDEX && segment_index != segment_index_before) {
-		FinalizePinState(state.pin_state, segments[segment_index_before]);
+		FinalizePinState(state.pin_state, *segments[segment_index_before]);
 	}
 	ScanAtIndex(state.pin_state, state.chunk_state, state.chunk_state.column_ids, segment_index, chunk_index, result);
 	return true;
@@ -484,14 +595,14 @@ bool TupleDataCollection::Scan(TupleDataParallelScanState &gstate, TupleDataLoca
 		lock_guard<mutex> guard(gstate.lock);
 		if (!NextScanIndex(gstate.scan_state, lstate.segment_index, lstate.chunk_index)) {
 			if (!segments.empty()) {
-				FinalizePinState(lstate.pin_state, segments[segment_index_before]);
+				FinalizePinState(lstate.pin_state, *segments[segment_index_before]);
 			}
 			result.SetCardinality(0);
 			return false;
 		}
 	}
 	if (segment_index_before != DConstants::INVALID_INDEX && segment_index_before != lstate.segment_index) {
-		FinalizePinState(lstate.pin_state, segments[lstate.segment_index]);
+		FinalizePinState(lstate.pin_state, *segments[lstate.segment_index]);
 	}
 	ScanAtIndex(lstate.pin_state, lstate.chunk_state, gstate.scan_state.chunk_state.column_ids, lstate.segment_index,
 	            lstate.chunk_index, result);
@@ -502,7 +613,7 @@ bool TupleDataCollection::ScanComplete(const TupleDataScanState &state) const {
 	if (Count() == 0) {
 		return true;
 	}
-	return state.segment_index == segments.size() - 1 && state.chunk_index == segments.back().ChunkCount();
+	return state.segment_index == segments.size() - 1 && state.chunk_index == segments.back()->ChunkCount();
 }
 
 void TupleDataCollection::FinalizePinState(TupleDataPinState &pin_state, TupleDataSegment &segment) {
@@ -511,7 +622,7 @@ void TupleDataCollection::FinalizePinState(TupleDataPinState &pin_state, TupleDa
 
 void TupleDataCollection::FinalizePinState(TupleDataPinState &pin_state) {
 	D_ASSERT(!segments.empty());
-	FinalizePinState(pin_state, segments.back());
+	FinalizePinState(pin_state, *segments.back());
 }
 
 bool TupleDataCollection::NextScanIndex(TupleDataScanState &state, idx_t &segment_index, idx_t &chunk_index) {
@@ -521,7 +632,7 @@ bool TupleDataCollection::NextScanIndex(TupleDataScanState &state, idx_t &segmen
 		return false;
 	}
 	// Check within the current segment if we still have chunks to scan
-	while (state.chunk_index >= segments[state.segment_index].ChunkCount()) {
+	while (state.chunk_index >= segments[state.segment_index]->ChunkCount()) {
 		// Exhausted all chunks for this segment: Move to the next one
 		state.segment_index++;
 		state.chunk_index = 0;
@@ -536,7 +647,7 @@ bool TupleDataCollection::NextScanIndex(TupleDataScanState &state, idx_t &segmen
 void TupleDataCollection::ScanAtIndex(TupleDataPinState &pin_state, TupleDataChunkState &chunk_state,
                                       const vector<column_t> &column_ids, idx_t segment_index, idx_t chunk_index,
                                       DataChunk &result) {
-	auto &segment = segments[segment_index];
+	auto &segment = *segments[segment_index];
 	auto &chunk = segment.chunks[chunk_index];
 	segment.allocator->InitializeChunkState(segment, pin_state, chunk_state, chunk_index, false);
 	result.Reset();
@@ -582,13 +693,13 @@ void TupleDataCollection::Print() {
 }
 
 void TupleDataCollection::Verify() const {
-#ifdef DEBUG
+#ifdef D_ASSERT_IS_ENABLED
 	idx_t total_count = 0;
 	idx_t total_size = 0;
 	for (const auto &segment : segments) {
-		segment.Verify();
-		total_count += segment.count;
-		total_size += segment.data_size;
+		segment->Verify();
+		total_count += segment->count;
+		total_size += segment->data_size;
 	}
 	D_ASSERT(total_count == this->count);
 	D_ASSERT(total_size == this->data_size);
@@ -596,9 +707,9 @@ void TupleDataCollection::Verify() const {
 }
 
 void TupleDataCollection::VerifyEverythingPinned() const {
-#ifdef DEBUG
+#ifdef D_ASSERT_IS_ENABLED
 	for (const auto &segment : segments) {
-		segment.VerifyEverythingPinned();
+		segment->VerifyEverythingPinned();
 	}
 #endif
 }

package/src/duckdb/src/common/types/row/tuple_data_iterator.cpp

@@ -17,7 +17,7 @@ TupleDataChunkIterator::TupleDataChunkIterator(TupleDataCollection &collection_p
 	D_ASSERT(chunk_idx_to <= collection.ChunkCount());
 	idx_t overall_chunk_index = 0;
 	for (idx_t segment_idx = 0; segment_idx < collection.segments.size(); segment_idx++) {
-		const auto &segment = collection.segments[segment_idx];
+		const auto &segment = *collection.segments[segment_idx];
 		if (chunk_idx_from >= overall_chunk_index && chunk_idx_from <= overall_chunk_index + segment.ChunkCount()) {
 			// We start in this segment
 			start_segment_idx = segment_idx;
@@ -35,7 +35,7 @@ TupleDataChunkIterator::TupleDataChunkIterator(TupleDataCollection &collection_p
 }
 
 void TupleDataChunkIterator::InitializeCurrentChunk() {
-	auto &segment = collection.segments[current_segment_idx];
+	auto &segment = *collection.segments[current_segment_idx];
 	segment.allocator->InitializeChunkState(segment, state.pin_state, state.chunk_state, current_chunk_idx, init_heap);
 }
 
@@ -51,7 +51,7 @@ bool TupleDataChunkIterator::Next() {
 	const auto segment_idx_before = current_segment_idx;
 	if (!collection.NextScanIndex(state, current_segment_idx, current_chunk_idx) || Done()) {
 		// Drop pins / stores them if TupleDataPinProperties::KEEP_EVERYTHING_PINNED
-		collection.FinalizePinState(state.pin_state, collection.segments[segment_idx_before]);
+		collection.FinalizePinState(state.pin_state, *collection.segments[segment_idx_before]);
 		current_segment_idx = end_segment_idx;
 		current_chunk_idx = end_chunk_idx;
 		return false;
@@ -59,7 +59,7 @@ bool TupleDataChunkIterator::Next() {
 
 	// Finalize pin state when moving from one segment to the next
 	if (current_segment_idx != segment_idx_before) {
-		collection.FinalizePinState(state.pin_state, collection.segments[segment_idx_before]);
+		collection.FinalizePinState(state.pin_state, *collection.segments[segment_idx_before]);
 	}
 
 	InitializeCurrentChunk();
@@ -74,7 +74,7 @@ void TupleDataChunkIterator::Reset() {
 }
 
 idx_t TupleDataChunkIterator::GetCurrentChunkCount() const {
-	return collection.segments[current_segment_idx].chunks[current_chunk_idx].count;
+	return collection.segments[current_segment_idx]->chunks[current_chunk_idx].count;
 }
 
 TupleDataChunkState &TupleDataChunkIterator::GetChunkState() {

package/src/duckdb/src/common/types/row/tuple_data_layout.cpp

@@ -1,17 +1,20 @@
 #include "duckdb/common/types/row/tuple_data_layout.hpp"
 
 #include "duckdb/planner/expression/bound_aggregate_expression.hpp"
+#include "duckdb/common/sorting/sort_key.hpp"
 
 namespace duckdb {
 
 TupleDataLayout::TupleDataLayout()
-    : flag_width(0), data_width(0), aggr_width(0), row_width(0), all_constant(true), heap_size_offset(0) {
+    : sort_key_type(SortKeyType::INVALID), flag_width(0), data_width(0), aggr_width(0), row_width(0),
+      all_constant(true), heap_size_offset(0), validity_type(TupleDataValidityType::CAN_HAVE_NULL_VALUES) {
 }
 
 TupleDataLayout TupleDataLayout::Copy() const {
 	TupleDataLayout result;
 	result.types = this->types;
 	result.aggregates = this->aggregates;
+	result.sort_key_type = this->sort_key_type;
 	if (this->struct_layouts) {
 		result.struct_layouts = make_uniq<unordered_map<idx_t, TupleDataLayout>>();
 		for (const auto &entry : *this->struct_layouts) {
@@ -21,20 +24,30 @@ TupleDataLayout TupleDataLayout::Copy() const {
 	result.flag_width = this->flag_width;
 	result.data_width = this->data_width;
 	result.aggr_width = this->aggr_width;
+	result.sort_width = this->sort_width;
+	result.sort_skippable_bytes = this->sort_skippable_bytes;
 	result.row_width = this->row_width;
 	result.offsets = this->offsets;
 	result.all_constant = this->all_constant;
 	result.heap_size_offset = this->heap_size_offset;
 	result.aggr_destructor_idxs = this->aggr_destructor_idxs;
+	result.validity_type = this->validity_type;
 	return result;
 }
 
-void TupleDataLayout::Initialize(vector<LogicalType> types_p, Aggregates aggregates_p, bool align, bool heap_offset_p) {
+void TupleDataLayout::Initialize(vector<LogicalType> types_p, Aggregates aggregates_p,
+                                 TupleDataValidityType validity_type_p, TupleDataNestednessType nestedness_type) {
+	sort_key_type = SortKeyType::INVALID;
 	offsets.clear();
+	aggr_destructor_idxs.clear();
 	types = std::move(types_p);
+	all_constant = true;
+	sort_skippable_bytes.clear();
+	variable_columns.clear();
 
 	// Null mask at the front - 1 bit per value.
-	flag_width = ValidityBytes::ValidityMaskSize(types.size());
+	validity_type = validity_type_p;
+	flag_width = ValidityBytes::ValidityMaskSize(AllValid() ? 0 : types.size());
 	row_width = flag_width;
 
 	// Whether all columns are constant size.
@@ -52,27 +65,34 @@ void TupleDataLayout::Initialize(vector<LogicalType> types_p, Aggregates aggrega
 				struct_layouts = make_uniq<unordered_map<idx_t, TupleDataLayout>>();
 			}
 			auto struct_entry = struct_layouts->emplace(col_idx, TupleDataLayout());
-			struct_entry.first->second.Initialize(std::move(child_type_vector), false, false);
-			all_constant = all_constant && struct_entry.first->second.AllConstant();
-		} else {
-			all_constant = all_constant && TypeIsConstantSize(type.InternalType());
+			struct_entry.first->second.Initialize(std::move(child_type_vector),
+			                                      TupleDataValidityType::CAN_HAVE_NULL_VALUES,
+			                                      TupleDataNestednessType::NESTED_STRUCT_LAYOUT);
+
+			if (!struct_entry.first->second.AllConstant()) {
+				all_constant = false;
+				variable_columns.push_back(col_idx);
+			}
+		} else if (!TypeIsConstantSize(type.InternalType())) {
+			all_constant = false;
+			variable_columns.push_back(col_idx);
 		}
 	}
 
-	// This enables pointer swizzling for out-of-core computation.
-	if (heap_offset_p && !all_constant) {
+	// This enables pointer recomputation for out-of-core.
+	if (nestedness_type == TupleDataNestednessType::TOP_LEVEL_LAYOUT && !all_constant) {
 		heap_size_offset = row_width;
-		row_width += sizeof(uint32_t);
+		row_width += sizeof(idx_t);
 	}
 
 	// Data columns. No alignment required.
 	for (idx_t col_idx = 0; col_idx < types.size(); col_idx++) {
 		const auto &type = types[col_idx];
 		offsets.push_back(row_width);
-		const auto internal_type = type.InternalType();
-		if (TypeIsConstantSize(internal_type) || internal_type == PhysicalType::VARCHAR) {
-			row_width += GetTypeIdSize(type.InternalType());
-		} else if (internal_type == PhysicalType::STRUCT) {
+		const auto physical_type = type.InternalType();
+		if (TypeIsConstantSize(physical_type) || physical_type == PhysicalType::VARCHAR) {
+			row_width += GetTypeIdSize(physical_type);
+		} else if (physical_type == PhysicalType::STRUCT) {
 			// Just get the size of the TupleDataLayout of the struct
 			row_width += GetStructLayout(col_idx).GetRowWidth();
 		} else {
@@ -83,6 +103,7 @@ void TupleDataLayout::Initialize(vector<LogicalType> types_p, Aggregates aggrega
 	}
 
 	// Alignment padding for aggregates
+	const auto align = !aggregates_p.empty();
 #ifndef DUCKDB_ALLOW_UNDEFINED
 	if (align) {
 		row_width = AlignValue(row_width);
@@ -116,12 +137,95 @@ void TupleDataLayout::Initialize(vector<LogicalType> types_p, Aggregates aggrega
 	}
 }
 
-void TupleDataLayout::Initialize(vector<LogicalType> types_p, bool align, bool heap_offset_p) {
-	Initialize(std::move(types_p), Aggregates(), align, heap_offset_p);
+void TupleDataLayout::Initialize(vector<LogicalType> types_p, TupleDataValidityType validity_type,
+                                 TupleDataNestednessType nestedness_type) {
+	Initialize(std::move(types_p), Aggregates(), validity_type, nestedness_type);
+}
+
+void TupleDataLayout::Initialize(Aggregates aggregates_p) {
+	Initialize(vector<LogicalType>(), std::move(aggregates_p), TupleDataValidityType::CANNOT_HAVE_NULL_VALUES,
+	           TupleDataNestednessType::TOP_LEVEL_LAYOUT);
+}
+
+void TupleDataLayout::Initialize(const vector<BoundOrderByNode> &orders, const LogicalType &type, bool has_payload) {
+	// Reset state
+	types.clear();
+	aggregates.clear();
+	struct_layouts.reset();
+	flag_width = DConstants::INVALID_INDEX;
+	data_width = DConstants::INVALID_INDEX;
+	aggr_width = DConstants::INVALID_INDEX;
+	offsets.clear();
+	sort_skippable_bytes.clear();
+	all_constant = true;
+	heap_size_offset = DConstants::INVALID_INDEX;
+	aggr_destructor_idxs.clear();
+	validity_type = TupleDataValidityType::CAN_HAVE_NULL_VALUES;
+
+	// Type is determined by "create_sort_key", if it is <= 8 we get a bigint
+	types.push_back(type);
+
+	// Compute row width
+	sort_width = 0;
+	bool all_valid_and_truely_constant = true;
+	for (idx_t order_idx = 0; order_idx < orders.size(); order_idx++) {
+		const auto &order = orders[order_idx];
+		const auto &logical_type = order.expression->return_type;
+		const auto physical_type = logical_type.InternalType();
+
+		if (all_valid_and_truely_constant && order.stats && !order.stats->CanHaveNull() &&
+		    TypeIsConstantSize(physical_type) && sort_width < 7) {
+			// We don't have to sort by this byte, all values are valid
+			sort_skippable_bytes.emplace_back(sort_width);
+		} else {
+			all_valid_and_truely_constant = false;
+		}
+
+		if (TypeIsConstantSize(physical_type)) {
+			// NULL byte + fixed-width type
+			sort_width += 1 + GetTypeIdSize(physical_type);
+		} else if (logical_type == LogicalType::VARCHAR && order.stats &&
+		           StringStats::HasMaxStringLength(*order.stats)) {
+			// NULL byte + maximum string length + string delimiter
+			sort_width += 1 + StringStats::MaxStringLength(*order.stats) + 1;
+		} else {
+			// We don't know how long the key will be
+			sort_width = DConstants::INVALID_INDEX;
+			break;
+		}
+	}
+
+	// Set row width and sort key type accordingly
+	idx_t temp_row_width = type.id() == LogicalTypeId::BIGINT ? 8 : sort_width;
+	if (sort_width != DConstants::INVALID_INDEX && has_payload) {
+		temp_row_width += 8;
+	}
+	if (temp_row_width <= 8) {
+		D_ASSERT(!has_payload);
+		row_width = 8;
+		sort_key_type = SortKeyType::NO_PAYLOAD_FIXED_8;
+	} else if (temp_row_width <= 16) {
+		row_width = 16;
+		sort_key_type = has_payload ? SortKeyType::PAYLOAD_FIXED_16 : SortKeyType::NO_PAYLOAD_FIXED_16;
+	} else if (temp_row_width <= 24) {
+		row_width = 24;
+		sort_key_type = has_payload ? SortKeyType::PAYLOAD_FIXED_24 : SortKeyType::NO_PAYLOAD_FIXED_24;
+	} else if (temp_row_width <= 32) {
+		row_width = 32;
+		sort_key_type = has_payload ? SortKeyType::PAYLOAD_FIXED_32 : SortKeyType::NO_PAYLOAD_FIXED_32;
+	} else {
+		row_width = 32;
+		sort_key_type = has_payload ? SortKeyType::PAYLOAD_VARIABLE_32 : SortKeyType::NO_PAYLOAD_VARIABLE_32;
+
+		// Variable-size sort key, also set these properties
+		all_constant = false;
+		heap_size_offset = has_payload ? SortKey<SortKeyType::PAYLOAD_VARIABLE_32>::HEAP_SIZE_OFFSET
+		                               : SortKey<SortKeyType::NO_PAYLOAD_VARIABLE_32>::HEAP_SIZE_OFFSET;
+	}
 }
 
-void TupleDataLayout::Initialize(Aggregates aggregates_p, bool align, bool heap_offset_p) {
-	Initialize(vector<LogicalType>(), std::move(aggregates_p), align, heap_offset_p);
+bool TupleDataLayout::IsSortKeyLayout() const {
+	return sort_key_type != SortKeyType::INVALID;
 }
 
 } // namespace duckdb