duckdb 1.0.1-dev21.0 → 1.0.1-dev27.0

package/src/duckdb/src/common/row_operations/row_radix_scatter.cpp

@@ -114,62 +114,67 @@ void RadixScatterListVector(Vector &v, UnifiedVectorFormat &vdata, const Selecti
 		for (idx_t i = 0; i < add_count; i++) {
 			auto idx = sel.get_index(i);
 			auto source_idx = vdata.sel->get_index(idx) + offset;
-			data_ptr_t key_location = key_locations[i] + 1;
+			data_ptr_t &key_location = key_locations[i];
+			const data_ptr_t key_location_start = key_location;
 			// write validity and according value
 			if (validity.RowIsValid(source_idx)) {
-				key_locations[i][0] = valid;
-				key_locations[i]++;
+				*key_location++ = valid;
 				auto &list_entry = list_data[source_idx];
 				if (list_entry.length > 0) {
 					// denote that the list is not empty with a 1
-					key_locations[i][0] = 1;
-					key_locations[i]++;
+					*key_location++ = 1;
 					RowOperations::RadixScatter(child_vector, list_size, *FlatVector::IncrementalSelectionVector(), 1,
 					                            key_locations + i, false, true, false, prefix_len, width - 2,
 					                            list_entry.offset);
 				} else {
 					// denote that the list is empty with a 0
-					key_locations[i][0] = 0;
-					key_locations[i]++;
-					memset(key_locations[i], '\0', width - 2);
+					*key_location++ = 0;
+					// mark rest of bits as empty
+					memset(key_location, '\0', width - 2);
+					key_location += width - 2;
 				}
 				// invert bits if desc
 				if (desc) {
-					for (idx_t s = 0; s < width - 1; s++) {
-						*(key_location + s) = ~*(key_location + s);
+					// skip over validity byte, handled by nulls first/last
+					for (key_location = key_location_start + 1; key_location < key_location_start + width;
+					     key_location++) {
+						*key_location = ~*key_location;
 					}
 				}
 			} else {
-				key_locations[i][0] = invalid;
-				memset(key_locations[i] + 1, '\0', width - 1);
-				key_locations[i] += width;
+				*key_location++ = invalid;
+				memset(key_location, '\0', width - 1);
+				key_location += width - 1;
 			}
+			D_ASSERT(key_location == key_location_start + width);
 		}
 	} else {
 		for (idx_t i = 0; i < add_count; i++) {
 			auto idx = sel.get_index(i);
 			auto source_idx = vdata.sel->get_index(idx) + offset;
 			auto &list_entry = list_data[source_idx];
-			data_ptr_t key_location = key_locations[i];
+			data_ptr_t &key_location = key_locations[i];
+			const data_ptr_t key_location_start = key_location;
 			if (list_entry.length > 0) {
 				// denote that the list is not empty with a 1
-				key_locations[i][0] = 1;
-				key_locations[i]++;
+				*key_location++ = 1;
 				RowOperations::RadixScatter(child_vector, list_size, *FlatVector::IncrementalSelectionVector(), 1,
 				                            key_locations + i, false, true, false, prefix_len, width - 1,
 				                            list_entry.offset);
 			} else {
 				// denote that the list is empty with a 0
-				key_locations[i][0] = 0;
-				key_locations[i]++;
-				memset(key_locations[i], '\0', width - 1);
+				*key_location++ = 0;
+				// mark rest of bits as empty
+				memset(key_location, '\0', width - 1);
+				key_location += width - 1;
 			}
 			// invert bits if desc
 			if (desc) {
-				for (idx_t s = 0; s < width; s++) {
-					*(key_location + s) = ~*(key_location + s);
+				for (key_location = key_location_start; key_location < key_location_start + width; key_location++) {
+					*key_location = ~*key_location;
 				}
 			}
+			D_ASSERT(key_location == key_location_start + width);
 		}
 	}
 }
@@ -177,7 +182,9 @@ void RadixScatterListVector(Vector &v, UnifiedVectorFormat &vdata, const Selecti
 void RadixScatterArrayVector(Vector &v, UnifiedVectorFormat &vdata, idx_t vcount, const SelectionVector &sel,
                              idx_t add_count, data_ptr_t *key_locations, const bool desc, const bool has_null,
                              const bool nulls_first, const idx_t prefix_len, idx_t width, const idx_t offset) {
-	// serialize null values
+	auto &child_vector = ArrayVector::GetEntry(v);
+	auto array_size = ArrayType::GetSize(v.GetType());
+
 	if (has_null) {
 		auto &validity = vdata.validity;
 		const data_t valid = nulls_first ? 1 : 0;
@@ -186,33 +193,48 @@ void RadixScatterArrayVector(Vector &v, UnifiedVectorFormat &vdata, idx_t vcount
 		for (idx_t i = 0; i < add_count; i++) {
 			auto idx = sel.get_index(i);
 			auto source_idx = vdata.sel->get_index(idx) + offset;
-			// write validity and according value
+			data_ptr_t &key_location = key_locations[i];
+			const data_ptr_t key_location_start = key_location;
+
 			if (validity.RowIsValid(source_idx)) {
-				key_locations[i][0] = valid;
+				*key_location++ = valid;
+
+				auto array_offset = source_idx * array_size;
+				RowOperations::RadixScatter(child_vector, array_size, *FlatVector::IncrementalSelectionVector(), 1,
+				                            key_locations + i, false, true, false, prefix_len, width - 1, array_offset);
+
+				// invert bits if desc
+				if (desc) {
+					// skip over validity byte, handled by nulls first/last
+					for (key_location = key_location_start + 1; key_location < key_location_start + width;
+					     key_location++) {
+						*key_location = ~*key_location;
+					}
+				}
 			} else {
-				key_locations[i][0] = invalid;
+				*key_location++ = invalid;
+				memset(key_location, '\0', width - 1);
+				key_location += width - 1;
 			}
-			key_locations[i]++;
+			D_ASSERT(key_location == key_location_start + width);
 		}
-		width--;
-	}
-
-	// serialize the inner child
-	auto &child_vector = ArrayVector::GetEntry(v);
-	auto array_size = ArrayType::GetSize(v.GetType());
-	for (idx_t i = 0; i < add_count; i++) {
-		auto idx = sel.get_index(i);
-		auto source_idx = vdata.sel->get_index(idx) + offset;
-		auto array_offset = source_idx * array_size;
-		data_ptr_t key_location = key_locations[i];
+	} else {
+		for (idx_t i = 0; i < add_count; i++) {
+			auto idx = sel.get_index(i);
+			auto source_idx = vdata.sel->get_index(idx) + offset;
+			data_ptr_t &key_location = key_locations[i];
+			const data_ptr_t key_location_start = key_location;
 
-		RowOperations::RadixScatter(child_vector, array_size, *FlatVector::IncrementalSelectionVector(), 1,
-		                            key_locations + i, false, true, false, prefix_len, width - 1, array_offset);
-		// invert bits if desc
-		if (desc) {
-			for (idx_t s = 0; s < width; s++) {
-				*(key_location + s) = ~*(key_location + s);
+			auto array_offset = source_idx * array_size;
+			RowOperations::RadixScatter(child_vector, array_size, *FlatVector::IncrementalSelectionVector(), 1,
+			                            key_locations + i, false, true, false, prefix_len, width, array_offset);
+			// invert bits if desc
+			if (desc) {
+				for (key_location = key_location_start; key_location < key_location_start + width; key_location++) {
+					*key_location = ~*key_location;
+				}
 			}
+			D_ASSERT(key_location == key_location_start + width);
 		}
 	}
 }
@@ -256,6 +278,14 @@ void RadixScatterStructVector(Vector &v, UnifiedVectorFormat &vdata, idx_t vcoun
 void RowOperations::RadixScatter(Vector &v, idx_t vcount, const SelectionVector &sel, idx_t ser_count,
                                  data_ptr_t *key_locations, bool desc, bool has_null, bool nulls_first,
                                  idx_t prefix_len, idx_t width, idx_t offset) {
+#ifdef DEBUG
+	// initialize to verify written width later
+	auto key_locations_copy = make_uniq_array<data_ptr_t>(ser_count);
+	for (idx_t i = 0; i < ser_count; i++) {
+		key_locations_copy[i] = key_locations[i];
+	}
+#endif
+
 	UnifiedVectorFormat vdata;
 	v.ToUnifiedFormat(vcount, vdata);
 	switch (v.GetType().InternalType()) {
@@ -317,6 +347,12 @@ void RowOperations::RadixScatter(Vector &v, idx_t vcount, const SelectionVector
 	default:
 		throw NotImplementedException("Cannot ORDER BY column with type %s", v.GetType().ToString());
 	}
+
+#ifdef DEBUG
+	for (idx_t i = 0; i < ser_count; i++) {
+		D_ASSERT(key_locations[i] == key_locations_copy[i] + width);
+	}
+#endif
 }
 
 } // namespace duckdb

package/src/duckdb/src/common/serializer/binary_deserializer.cpp

@@ -113,7 +113,7 @@ string BinaryDeserializer::ReadString() {
 	if (len == 0) {
 		return string();
 	}
-	auto buffer = make_unsafe_uniq_array<data_t>(len);
+	auto buffer = make_unsafe_uniq_array_uninitialized<data_t>(len);
 	ReadData(buffer.get(), len);
 	return string(const_char_ptr_cast(buffer.get()), len);
 }

package/src/duckdb/src/common/serializer/buffered_file_reader.cpp

@@ -1,21 +1,23 @@
 #include "duckdb/common/serializer/buffered_file_reader.hpp"
-#include "duckdb/common/serializer/buffered_file_writer.hpp"
+
 #include "duckdb/common/exception.hpp"
+#include "duckdb/common/serializer/buffered_file_writer.hpp"
 
-#include <cstring>
 #include <algorithm>
+#include <cstring>
 
 namespace duckdb {
 
 BufferedFileReader::BufferedFileReader(FileSystem &fs, const char *path, FileLockType lock_type,
                                        optional_ptr<FileOpener> opener)
-    : fs(fs), data(make_unsafe_uniq_array<data_t>(FILE_BUFFER_SIZE)), offset(0), read_data(0), total_read(0) {
+    : fs(fs), data(make_unsafe_uniq_array_uninitialized<data_t>(FILE_BUFFER_SIZE)), offset(0), read_data(0),
+      total_read(0) {
 	handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_READ | lock_type, opener.get());
 	file_size = NumericCast<idx_t>(fs.GetFileSize(*handle));
 }
 
 BufferedFileReader::BufferedFileReader(FileSystem &fs, unique_ptr<FileHandle> handle_p)
-    : fs(fs), data(make_unsafe_uniq_array<data_t>(FILE_BUFFER_SIZE)), offset(0), read_data(0),
+    : fs(fs), data(make_unsafe_uniq_array_uninitialized<data_t>(FILE_BUFFER_SIZE)), offset(0), read_data(0),
       handle(std::move(handle_p)), total_read(0) {
 	file_size = NumericCast<idx_t>(fs.GetFileSize(*handle));
 }

package/src/duckdb/src/common/serializer/buffered_file_writer.cpp

@@ -1,7 +1,9 @@
 #include "duckdb/common/serializer/buffered_file_writer.hpp"
-#include "duckdb/common/exception.hpp"
+
 #include "duckdb/common/algorithm.hpp"
+#include "duckdb/common/exception.hpp"
 #include "duckdb/common/typedefs.hpp"
+
 #include <cstring>
 
 namespace duckdb {
@@ -10,12 +12,13 @@ namespace duckdb {
 constexpr FileOpenFlags BufferedFileWriter::DEFAULT_OPEN_FLAGS;
 
 BufferedFileWriter::BufferedFileWriter(FileSystem &fs, const string &path_p, FileOpenFlags open_flags)
-    : fs(fs), path(path_p), data(make_unsafe_uniq_array<data_t>(FILE_BUFFER_SIZE)), offset(0), total_written(0) {
+    : fs(fs), path(path_p), data(make_unsafe_uniq_array_uninitialized<data_t>(FILE_BUFFER_SIZE)), offset(0),
+      total_written(0) {
 	handle = fs.OpenFile(path, open_flags | FileLockType::WRITE_LOCK);
 }
 
-int64_t BufferedFileWriter::GetFileSize() {
-	return fs.GetFileSize(*handle) + NumericCast<int64_t>(offset);
+idx_t BufferedFileWriter::GetFileSize() {
+	return NumericCast<idx_t>(fs.GetFileSize(*handle)) + offset;
 }
 
 idx_t BufferedFileWriter::GetTotalWritten() {
@@ -65,20 +68,26 @@ void BufferedFileWriter::Flush() {
 	offset = 0;
 }
 
+void BufferedFileWriter::Close() {
+	Flush();
+	handle->Close();
+	handle.reset();
+}
+
 void BufferedFileWriter::Sync() {
 	Flush();
 	handle->Sync();
 }
 
-void BufferedFileWriter::Truncate(int64_t size) {
-	auto persistent = fs.GetFileSize(*handle);
-	D_ASSERT(size <= persistent + NumericCast<int64_t>(offset));
+void BufferedFileWriter::Truncate(idx_t size) {
+	auto persistent = NumericCast<idx_t>(fs.GetFileSize(*handle));
+	D_ASSERT(size <= persistent + offset);
 	if (persistent <= size) {
 		// truncating into the pending write buffer.
-		offset = NumericCast<idx_t>(size - persistent);
+		offset = size - persistent;
 	} else {
 		// truncate the physical file on disk
-		handle->Truncate(size);
+		handle->Truncate(NumericCast<int64_t>(size));
 		// reset anything written in the buffer
 		offset = 0;
 	}

package/src/duckdb/src/common/serializer/memory_stream.cpp

@@ -3,6 +3,7 @@
 namespace duckdb {
 
 MemoryStream::MemoryStream(idx_t capacity) : position(0), capacity(capacity), owns_data(true) {
+	D_ASSERT(capacity != 0 && IsPowerOfTwo(capacity));
 	auto data_malloc_result = malloc(capacity);
 	if (!data_malloc_result) {
 		throw std::bad_alloc();

package/src/duckdb/src/common/sort/partition_state.cpp

@@ -11,7 +11,7 @@ namespace duckdb {
 
 PartitionGlobalHashGroup::PartitionGlobalHashGroup(BufferManager &buffer_manager, const Orders &partitions,
                                                    const Orders &orders, const Types &payload_types, bool external)
-    : count(0), batch_base(0) {
+    : count(0) {
 
 	RowLayout payload_layout;
 	payload_layout.Initialize(payload_types);
@@ -94,7 +94,7 @@ PartitionGlobalSinkState::PartitionGlobalSinkState(ClientContext &context,
 	memory_per_thread = PhysicalOperator::GetMaxThreadMemory(context);
 	external = ClientConfig::GetConfig(context).force_external;
 
-	const auto thread_pages = PreviousPowerOfTwo(memory_per_thread / (4 * idx_t(Storage::BLOCK_ALLOC_SIZE)));
+	const auto thread_pages = PreviousPowerOfTwo(memory_per_thread / (4 * buffer_manager.GetBlockAllocSize()));
 	while (max_bits < 10 && (thread_pages >> max_bits) > 1) {
 		++max_bits;
 	}
@@ -316,9 +316,10 @@ void PartitionLocalSinkState::Sink(DataChunk &input_chunk) {
 		//	No sorts, so build paged row chunks
 		if (!rows) {
 			const auto entry_size = payload_layout.GetRowWidth();
-			const auto capacity = MaxValue<idx_t>(STANDARD_VECTOR_SIZE, (Storage::BLOCK_SIZE / entry_size) + 1);
+			const auto block_size = gstate.buffer_manager.GetBlockSize();
+			const auto capacity = MaxValue<idx_t>(STANDARD_VECTOR_SIZE, block_size / entry_size + 1);
 			rows = make_uniq<RowDataCollection>(gstate.buffer_manager, capacity, entry_size);
-			strings = make_uniq<RowDataCollection>(gstate.buffer_manager, (idx_t)Storage::BLOCK_SIZE, 1U, true);
+			strings = make_uniq<RowDataCollection>(gstate.buffer_manager, block_size, 1U, true);
 		}
 		const auto row_count = input_chunk.size();
 		const auto row_sel = FlatVector::IncrementalSelectionVector();
@@ -401,11 +402,11 @@ void PartitionLocalSinkState::Combine() {
 
 PartitionGlobalMergeState::PartitionGlobalMergeState(PartitionGlobalSinkState &sink, GroupDataPtr group_data_p,
                                                      hash_t hash_bin)
-    : sink(sink), group_data(std::move(group_data_p)), memory_per_thread(sink.memory_per_thread),
+    : sink(sink), group_data(std::move(group_data_p)), group_idx(sink.hash_groups.size()),
+      memory_per_thread(sink.memory_per_thread),
       num_threads(NumericCast<idx_t>(TaskScheduler::GetScheduler(sink.context).NumberOfThreads())),
       stage(PartitionSortStage::INIT), total_tasks(0), tasks_assigned(0), tasks_completed(0) {
 
-	const auto group_idx = sink.hash_groups.size();
 	auto new_group = make_uniq<PartitionGlobalHashGroup>(sink.buffer_manager, sink.partitions, sink.orders,
 	                                                     sink.payload_types, sink.external);
 	sink.hash_groups.emplace_back(std::move(new_group));
@@ -423,12 +424,11 @@ PartitionGlobalMergeState::PartitionGlobalMergeState(PartitionGlobalSinkState &s
 }
 
 PartitionGlobalMergeState::PartitionGlobalMergeState(PartitionGlobalSinkState &sink)
-    : sink(sink), memory_per_thread(sink.memory_per_thread),
+    : sink(sink), group_idx(0), memory_per_thread(sink.memory_per_thread),
       num_threads(NumericCast<idx_t>(TaskScheduler::GetScheduler(sink.context).NumberOfThreads())),
       stage(PartitionSortStage::INIT), total_tasks(0), tasks_assigned(0), tasks_completed(0) {
 
 	const hash_t hash_bin = 0;
-	const size_t group_idx = 0;
 	hash_group = sink.hash_groups[group_idx].get();
 	global_sort = sink.hash_groups[group_idx]->global_sort.get();
 
@@ -448,6 +448,10 @@ void PartitionLocalMergeState::Merge() {
 	merge_sorter.PerformInMergeRound();
 }
 
+void PartitionLocalMergeState::Sorted() {
+	merge_state->sink.OnSortedPartition(merge_state->group_idx);
+}
+
 void PartitionLocalMergeState::ExecuteTask() {
 	switch (stage) {
 	case PartitionSortStage::SCAN:
@@ -459,6 +463,9 @@ void PartitionLocalMergeState::ExecuteTask() {
 	case PartitionSortStage::MERGE:
 		Merge();
 		break;
+	case PartitionSortStage::SORTED:
+		Sorted();
+		break;
 	default:
 		throw InternalException("Unexpected PartitionSortStage in ExecuteTask!");
 	}
@@ -513,30 +520,36 @@ bool PartitionGlobalMergeState::TryPrepareNextStage() {
 		return true;
 
 	case PartitionSortStage::PREPARE:
-		total_tasks = global_sort->sorted_blocks.size() / 2;
-		if (!total_tasks) {
+		if (!(global_sort->sorted_blocks.size() / 2)) {
 			break;
 		}
 		stage = PartitionSortStage::MERGE;
 		global_sort->InitializeMergeRound();
+		total_tasks = num_threads;
 		return true;
 
 	case PartitionSortStage::MERGE:
 		global_sort->CompleteMergeRound(true);
-		total_tasks = global_sort->sorted_blocks.size() / 2;
-		if (!total_tasks) {
+		if (!(global_sort->sorted_blocks.size() / 2)) {
 			break;
 		}
 		global_sort->InitializeMergeRound();
+		total_tasks = num_threads;
 		return true;
 
 	case PartitionSortStage::SORTED:
-		break;
+		stage = PartitionSortStage::FINISHED;
+		total_tasks = 0;
+		return false;
+
+	case PartitionSortStage::FINISHED:
+		return false;
 	}
 
 	stage = PartitionSortStage::SORTED;
+	total_tasks = 1;
 
-	return false;
+	return true;
 }
 
 PartitionGlobalMergeStates::PartitionGlobalMergeStates(PartitionGlobalSinkState &sink) {
@@ -559,13 +572,15 @@ PartitionGlobalMergeStates::PartitionGlobalMergeStates(PartitionGlobalSinkState
 		auto state = make_uniq<PartitionGlobalMergeState>(sink);
 		states.emplace_back(std::move(state));
 	}
+
+	sink.OnBeginMerge();
 }
 
 class PartitionMergeTask : public ExecutorTask {
 public:
 	PartitionMergeTask(shared_ptr<Event> event_p, ClientContext &context_p, PartitionGlobalMergeStates &hash_groups_p,
-	                   PartitionGlobalSinkState &gstate)
-	    : ExecutorTask(context_p, std::move(event_p)), local_state(gstate), hash_groups(hash_groups_p) {
+	                   PartitionGlobalSinkState &gstate, const PhysicalOperator &op)
+	    : ExecutorTask(context_p, std::move(event_p), op), local_state(gstate), hash_groups(hash_groups_p) {
 	}
 
 	TaskExecutionResult ExecuteTask(TaskExecutionMode mode) override;
@@ -602,7 +617,7 @@ bool PartitionGlobalMergeStates::ExecuteTask(PartitionLocalMergeState &local_sta
 		// Thread is done with its assigned task, try to fetch new work
 		for (auto group = sorted; group < states.size(); ++group) {
 			auto &global_state = states[group];
-			if (global_state->IsSorted()) {
+			if (global_state->IsFinished()) {
 				// This hash group is done
 				// Update the high water mark of densely completed groups
 				if (sorted == group) {
@@ -665,7 +680,7 @@ void PartitionMergeEvent::Schedule() {
 
 	vector<shared_ptr<Task>> merge_tasks;
 	for (idx_t tnum = 0; tnum < num_threads; tnum++) {
-		merge_tasks.emplace_back(make_uniq<PartitionMergeTask>(shared_from_this(), context, merge_states, gstate));
+		merge_tasks.emplace_back(make_uniq<PartitionMergeTask>(shared_from_this(), context, merge_states, gstate, op));
 	}
 	SetTasks(std::move(merge_tasks));
 }

package/src/duckdb/src/common/sort/radix_sort.cpp

@@ -17,7 +17,7 @@ static void SortTiedBlobs(BufferManager &buffer_manager, const data_ptr_t datapt
 		return;
 	}
 	// Fill pointer array for sorting
-	auto ptr_block = make_unsafe_uniq_array<data_ptr_t>(end - start);
+	auto ptr_block = make_unsafe_uniq_array_uninitialized<data_ptr_t>(end - start);
 	auto entry_ptrs = (data_ptr_t *)ptr_block.get();
 	for (idx_t i = start; i < end; i++) {
 		entry_ptrs[i - start] = row_ptr;
@@ -73,7 +73,7 @@ static void SortTiedBlobs(BufferManager &buffer_manager, SortedBlock &sb, bool *
 			continue;
 		}
 		idx_t j;
-		for (j = i; j < count; j++) {
+		for (j = i + 1; j < count; j++) {
 			if (!ties[j]) {
 				break;
 			}
@@ -158,7 +158,7 @@ inline void InsertionSort(const data_ptr_t orig_ptr, const data_ptr_t temp_ptr,
 	const data_ptr_t target_ptr = swap ? orig_ptr : temp_ptr;
 	if (count > 1) {
 		const idx_t total_offset = col_offset + offset;
-		auto temp_val = make_unsafe_uniq_array<data_t>(row_width);
+		auto temp_val = make_unsafe_uniq_array_uninitialized<data_t>(row_width);
 		const data_ptr_t val = temp_val.get();
 		const auto comp_width = total_comp_width - offset;
 		for (idx_t i = 1; i < count; i++) {
@@ -238,22 +238,29 @@ void RadixSortMSD(const data_ptr_t orig_ptr, const data_ptr_t temp_ptr, const id
 //! Calls different sort functions, depending on the count and sorting sizes
 void RadixSort(BufferManager &buffer_manager, const data_ptr_t &dataptr, const idx_t &count, const idx_t &col_offset,
                const idx_t &sorting_size, const SortLayout &sort_layout, bool contains_string) {
+
 	if (contains_string) {
 		auto begin = duckdb_pdqsort::PDQIterator(dataptr, sort_layout.entry_size);
 		auto end = begin + count;
 		duckdb_pdqsort::PDQConstants constants(sort_layout.entry_size, col_offset, sorting_size, *end);
-		duckdb_pdqsort::pdqsort_branchless(begin, begin + count, constants);
-	} else if (count <= SortConstants::INSERTION_SORT_THRESHOLD) {
-		InsertionSort(dataptr, nullptr, count, col_offset, sort_layout.entry_size, sorting_size, 0, false);
-	} else if (sorting_size <= SortConstants::MSD_RADIX_SORT_SIZE_THRESHOLD) {
-		RadixSortLSD(buffer_manager, dataptr, count, col_offset, sort_layout.entry_size, sorting_size);
-	} else {
-		auto temp_block = buffer_manager.Allocate(MemoryTag::ORDER_BY,
-		                                          MaxValue(count * sort_layout.entry_size, (idx_t)Storage::BLOCK_SIZE));
-		auto preallocated_array = make_unsafe_uniq_array<idx_t>(sorting_size * SortConstants::MSD_RADIX_LOCATIONS);
-		RadixSortMSD(dataptr, temp_block.Ptr(), count, col_offset, sort_layout.entry_size, sorting_size, 0,
-		             preallocated_array.get(), false);
+		return duckdb_pdqsort::pdqsort_branchless(begin, begin + count, constants);
+	}
+
+	if (count <= SortConstants::INSERTION_SORT_THRESHOLD) {
+		return InsertionSort(dataptr, nullptr, count, col_offset, sort_layout.entry_size, sorting_size, 0, false);
 	}
+
+	if (sorting_size <= SortConstants::MSD_RADIX_SORT_SIZE_THRESHOLD) {
+		return RadixSortLSD(buffer_manager, dataptr, count, col_offset, sort_layout.entry_size, sorting_size);
+	}
+
+	const auto block_size = buffer_manager.GetBlockSize();
+	auto temp_block =
+	    buffer_manager.Allocate(MemoryTag::ORDER_BY, MaxValue(count * sort_layout.entry_size, block_size));
+	auto pre_allocated_array =
+	    make_unsafe_uniq_array_uninitialized<idx_t>(sorting_size * SortConstants::MSD_RADIX_LOCATIONS);
+	RadixSortMSD(dataptr, temp_block.Ptr(), count, col_offset, sort_layout.entry_size, sorting_size, 0,
+	             pre_allocated_array.get(), false);
 }
 
 //! Identifies sequences of rows that are tied, and calls radix sort on these
@@ -306,7 +313,7 @@ void LocalSortState::SortInMemory() {
 		if (!ties) {
 			// This is the first sort
 			RadixSort(*buffer_manager, dataptr, count, col_offset, sorting_size, *sort_layout, contains_string);
-			ties_ptr = make_unsafe_uniq_array<bool>(count);
+			ties_ptr = make_unsafe_uniq_array_uninitialized<bool>(count);
 			ties = ties_ptr.get();
 			std::fill_n(ties, count - 1, true);
 			ties[count - 1] = false;

package/src/duckdb/src/common/sort/sort_state.cpp

@@ -163,22 +163,25 @@ void LocalSortState::Initialize(GlobalSortState &global_sort_state, BufferManage
 	sort_layout = &global_sort_state.sort_layout;
 	payload_layout = &global_sort_state.payload_layout;
 	buffer_manager = &buffer_manager_p;
+	const auto block_size = buffer_manager->GetBlockSize();
+
 	// Radix sorting data
-	radix_sorting_data = make_uniq<RowDataCollection>(
-	    *buffer_manager, RowDataCollection::EntriesPerBlock(sort_layout->entry_size), sort_layout->entry_size);
+	auto entries_per_block = RowDataCollection::EntriesPerBlock(sort_layout->entry_size, block_size);
+	radix_sorting_data = make_uniq<RowDataCollection>(*buffer_manager, entries_per_block, sort_layout->entry_size);
+
 	// Blob sorting data
 	if (!sort_layout->all_constant) {
 		auto blob_row_width = sort_layout->blob_layout.GetRowWidth();
-		blob_sorting_data = make_uniq<RowDataCollection>(
-		    *buffer_manager, RowDataCollection::EntriesPerBlock(blob_row_width), blob_row_width);
-		blob_sorting_heap = make_uniq<RowDataCollection>(*buffer_manager, (idx_t)Storage::BLOCK_SIZE, 1U, true);
+		entries_per_block = RowDataCollection::EntriesPerBlock(blob_row_width, block_size);
+		blob_sorting_data = make_uniq<RowDataCollection>(*buffer_manager, entries_per_block, blob_row_width);
+		blob_sorting_heap = make_uniq<RowDataCollection>(*buffer_manager, block_size, 1U, true);
 	}
+
 	// Payload data
 	auto payload_row_width = payload_layout->GetRowWidth();
-	payload_data = make_uniq<RowDataCollection>(*buffer_manager, RowDataCollection::EntriesPerBlock(payload_row_width),
-	                                            payload_row_width);
-	payload_heap = make_uniq<RowDataCollection>(*buffer_manager, (idx_t)Storage::BLOCK_SIZE, 1U, true);
-	// Init done
+	entries_per_block = RowDataCollection::EntriesPerBlock(payload_row_width, block_size);
+	payload_data = make_uniq<RowDataCollection>(*buffer_manager, entries_per_block, payload_row_width);
+	payload_heap = make_uniq<RowDataCollection>(*buffer_manager, block_size, 1U, true);
 	initialized = true;
 }
 
@@ -266,17 +269,17 @@ unique_ptr<RowDataBlock> LocalSortState::ConcatenateBlocks(RowDataCollection &ro
 		return new_block;
 	}
 	// Create block with the correct capacity
-	auto buffer_manager = &row_data.buffer_manager;
+	auto &buffer_manager = row_data.buffer_manager;
 	const idx_t &entry_size = row_data.entry_size;
-	idx_t capacity = MaxValue(((idx_t)Storage::BLOCK_SIZE + entry_size - 1) / entry_size, row_data.count);
-	auto new_block = make_uniq<RowDataBlock>(MemoryTag::ORDER_BY, *buffer_manager, capacity, entry_size);
+	idx_t capacity = MaxValue((buffer_manager.GetBlockSize() + entry_size - 1) / entry_size, row_data.count);
+	auto new_block = make_uniq<RowDataBlock>(MemoryTag::ORDER_BY, buffer_manager, capacity, entry_size);
 	new_block->count = row_data.count;
-	auto new_block_handle = buffer_manager->Pin(new_block->block);
+	auto new_block_handle = buffer_manager.Pin(new_block->block);
 	data_ptr_t new_block_ptr = new_block_handle.Ptr();
 	// Copy the data of the blocks into a single block
 	for (idx_t i = 0; i < row_data.blocks.size(); i++) {
 		auto &block = row_data.blocks[i];
-		auto block_handle = buffer_manager->Pin(block->block);
+		auto block_handle = buffer_manager.Pin(block->block);
 		memcpy(new_block_ptr, block_handle.Ptr(), block->count * entry_size);
 		new_block_ptr += block->count * entry_size;
 		block.reset();
@@ -322,7 +325,7 @@ void LocalSortState::ReOrder(SortedData &sd, data_ptr_t sorting_ptr, RowDataColl
 		idx_t total_byte_offset =
 		    std::accumulate(heap.blocks.begin(), heap.blocks.end(), (idx_t)0,
 		                    [](idx_t a, const unique_ptr<RowDataBlock> &b) { return a + b->byte_offset; });
-		idx_t heap_block_size = MaxValue(total_byte_offset, (idx_t)Storage::BLOCK_SIZE);
+		idx_t heap_block_size = MaxValue(total_byte_offset, buffer_manager->GetBlockSize());
 		auto ordered_heap_block = make_uniq<RowDataBlock>(MemoryTag::ORDER_BY, *buffer_manager, heap_block_size, 1U);
 		ordered_heap_block->count = count;
 		ordered_heap_block->byte_offset = total_byte_offset;
@@ -401,7 +404,7 @@ void GlobalSortState::PrepareMergePhase() {
 	idx_t total_heap_size =
 	    std::accumulate(sorted_blocks.begin(), sorted_blocks.end(), (idx_t)0,
 	                    [](idx_t a, const unique_ptr<SortedBlock> &b) { return a + b->HeapSize(); });
-	if (external || (pinned_blocks.empty() && total_heap_size > 0.25 * buffer_manager.GetQueryMaxMemory())) {
+	if (external || (pinned_blocks.empty() && total_heap_size * 4 > buffer_manager.GetQueryMaxMemory())) {
 		external = true;
 	}
 	// Use the data that we have to determine which partition size to use during the merge