duckdb 0.6.2-dev971.0 → 0.7.1-dev2.0

package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp

@@ -5,6 +5,7 @@
 #include "duckdb/common/row_operations/row_operations.hpp"
 #include "duckdb/common/sort/sort.hpp"
 #include "duckdb/common/types/chunk_collection.hpp"
+#include "duckdb/common/types/column_data_consumer.hpp"
 #include "duckdb/common/types/row_data_collection_scanner.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/common/windows_undefs.hpp"
@@ -16,6 +17,7 @@
 #include "duckdb/parallel/base_pipeline_event.hpp"
 #include "duckdb/planner/expression/bound_reference_expression.hpp"
 #include "duckdb/planner/expression/bound_window_expression.hpp"
+#include "duckdb/common/radix_partitioning.hpp"
 
 #include <algorithm>
 #include <cmath>
@@ -23,8 +25,6 @@
 
 namespace duckdb {
 
-using counts_t = std::vector<size_t>;
-
 class WindowGlobalHashGroup {
 public:
 	using GlobalSortStatePtr = unique_ptr<GlobalSortState>;
@@ -33,8 +33,8 @@ public:
 	using Types = vector<LogicalType>;
 
 	WindowGlobalHashGroup(BufferManager &buffer_manager, const Orders &partitions, const Orders &orders,
-	                      const Types &payload_types, idx_t max_mem, bool external)
-	    : memory_per_thread(max_mem), count(0) {
+	                      const Types &payload_types, bool external)
+	    : count(0) {
 
 		RowLayout payload_layout;
 		payload_layout.Initialize(payload_types);
@@ -44,17 +44,8 @@ public:
 		partition_layout = global_sort->sort_layout.GetPrefixComparisonLayout(partitions.size());
 	}
 
-	void Combine(LocalSortState &local_sort) {
-		global_sort->AddLocalState(local_sort);
-	}
-
-	void PrepareMergePhase() {
-		global_sort->PrepareMergePhase();
-	}
-
 	void ComputeMasks(ValidityMask &partition_mask, ValidityMask &order_mask);
 
-	const idx_t memory_per_thread;
 	GlobalSortStatePtr global_sort;
 	atomic<idx_t> count;
 
@@ -100,19 +91,19 @@ public:
 	using Orders = vector<BoundOrderByNode>;
 	using Types = vector<LogicalType>;
 
+	using GroupingPartition = unique_ptr<PartitionedColumnData>;
+	using GroupingAppend = unique_ptr<PartitionedColumnDataAppendState>;
+
 	WindowGlobalSinkState(const PhysicalWindow &op_p, ClientContext &context)
 	    : op(op_p), context(context), buffer_manager(BufferManager::GetBufferManager(context)),
-	      allocator(Allocator::Get(context)),
-	      partition_info((idx_t)TaskScheduler::GetScheduler(context).NumberOfThreads()), next_sort(0),
-	      memory_per_thread(0), count(0), mode(DBConfig::GetConfig(context).options.window_mode) {
+	      allocator(Allocator::Get(context)), payload_types(op.children[0]->types), memory_per_thread(0), count(0),
+	      mode(DBConfig::GetConfig(context).options.window_mode) {
 
 		D_ASSERT(op.select_list[0]->GetExpressionClass() == ExpressionClass::BOUND_WINDOW);
 		auto wexpr = reinterpret_cast<BoundWindowExpression *>(op.select_list[0].get());
 
 		// we sort by both 1) partition by expression list and 2) order by expressions
-		payload_types = op.children[0]->types;
-
-		partition_cols = wexpr->partitions.size();
+		const auto partition_cols = wexpr->partitions.size();
 		for (idx_t prt_idx = 0; prt_idx < partition_cols; prt_idx++) {
 			auto &pexpr = wexpr->partitions[prt_idx];
 
@@ -131,77 +122,37 @@ public:
 
 		memory_per_thread = op.GetMaxThreadMemory(context);
 		external = ClientConfig::GetConfig(context).force_external;
-	}
 
-	WindowGlobalHashGroup *GetUngrouped() {
-		lock_guard<mutex> guard(lock);
+		if (!orders.empty()) {
+			grouping_types = payload_types;
+			grouping_types.push_back(LogicalType::HASH);
 
-		if (!ungrouped) {
-			ungrouped = make_unique<WindowGlobalHashGroup>(buffer_manager, partitions, orders, payload_types,
-			                                               memory_per_thread, external);
+			ResizeGroupingData(op.estimated_cardinality);
 		}
-
-		return ungrouped.get();
 	}
 
-	//! Switch to hash grouping
-	size_t Group() {
-		lock_guard<mutex> guard(lock);
-		if (hash_groups.size() < partition_info.n_partitions) {
-			hash_groups.resize(partition_info.n_partitions);
-		}
+	void UpdateLocalPartition(GroupingPartition &local_partition, GroupingAppend &local_append);
+	void CombineLocalPartition(GroupingPartition &local_partition, GroupingAppend &local_append);
 
-		return hash_groups.size();
-	}
-
-	idx_t GroupCount() const {
-		return std::accumulate(
-		    hash_groups.begin(), hash_groups.end(), 0,
-		    [&](const idx_t &n, const HashGroupPtr &group) { return n + (group ? idx_t(group->count) : 0); });
-	}
-
-	WindowGlobalHashGroup *GetHashGroup(idx_t group) {
-		lock_guard<mutex> guard(lock);
-		D_ASSERT(group < hash_groups.size());
-		auto &hash_group = hash_groups[group];
-		if (!hash_group) {
-			const auto maxmem = memory_per_thread / partition_info.n_partitions;
-			hash_group =
-			    make_unique<WindowGlobalHashGroup>(buffer_manager, partitions, orders, payload_types, maxmem, external);
-		}
-
-		return hash_group.get();
-	}
-
-	void Finalize();
-
-	size_t GetNextSortGroup() {
-		for (auto group = next_sort++; group < hash_groups.size(); group = next_sort++) {
-			// Only non-empty groups exist.
-			if (hash_groups[group]) {
-				return group;
-			}
-		}
-
-		return hash_groups.size();
-	}
+	void BuildSortState(ColumnDataCollection &group_data, WindowGlobalHashGroup &global_sort);
 
 	const PhysicalWindow &op;
 	ClientContext &context;
 	BufferManager &buffer_manager;
 	Allocator &allocator;
-	size_t partition_cols;
-	const RadixPartitionInfo partition_info;
 	mutex lock;
 
-	// Sorting
+	// OVER(PARTITION BY...) (hash grouping)
+	unique_ptr<RadixPartitionedColumnData> grouping_data;
+	//! Payload plus hash column
+	Types grouping_types;
+
+	// OVER(...) (sorting)
 	Orders partitions;
 	Orders orders;
-	Types payload_types;
-	HashGroupPtr ungrouped;
+	const Types payload_types;
 	vector<HashGroupPtr> hash_groups;
 	bool external;
-	atomic<size_t> next_sort;
 
 	// OVER() (no sorting)
 	unique_ptr<RowDataCollection> rows;
@@ -211,253 +162,250 @@ public:
 	idx_t memory_per_thread;
 	atomic<idx_t> count;
 	WindowAggregationMode mode;
+
+private:
+	void ResizeGroupingData(idx_t cardinality);
+	void SyncLocalPartition(GroupingPartition &local_partition, GroupingAppend &local_append);
 };
 
-//	Per-thread hash group
-class WindowLocalHashGroup {
-public:
-	using LocalSortStatePtr = unique_ptr<LocalSortState>;
-	using DataChunkPtr = unique_ptr<DataChunk>;
+void WindowGlobalSinkState::ResizeGroupingData(idx_t cardinality) {
+	//	Is the average partition size too large?
+	const idx_t partition_size = STANDARD_ROW_GROUPS_SIZE;
+	const auto bits = grouping_data ? grouping_data->GetRadixBits() : 0;
+	auto new_bits = bits ? bits : 4;
+	while (new_bits < 10 && (cardinality / RadixPartitioning::NumberOfPartitions(new_bits)) > partition_size) {
+		++new_bits;
+	}
+
+	// Repartition the grouping data
+	if (new_bits != bits) {
+		const auto hash_col_idx = payload_types.size();
+		auto new_grouping_data =
+		    make_unique<RadixPartitionedColumnData>(context, grouping_types, new_bits, hash_col_idx);
+
+		// We have to append to a shared copy for some reason
+		if (grouping_data) {
+			auto new_shared = new_grouping_data->CreateShared();
+			PartitionedColumnDataAppendState shared_append;
+			new_shared->InitializeAppendState(shared_append);
+
+			auto &partitions = grouping_data->GetPartitions();
+			for (auto &partition : partitions) {
+				ColumnDataScanState scanner;
+				partition->InitializeScan(scanner);
+
+				DataChunk scan_chunk;
+				partition->InitializeScanChunk(scan_chunk);
+				for (scan_chunk.Reset(); partition->Scan(scanner, scan_chunk); scan_chunk.Reset()) {
+					new_shared->Append(shared_append, scan_chunk);
+				}
+			}
+			new_shared->FlushAppendState(shared_append);
+			new_grouping_data->Combine(*new_shared);
+		}
 
-	explicit WindowLocalHashGroup(WindowGlobalHashGroup &global_group_p) : global_group(global_group_p), count(0) {
+		grouping_data = std::move(new_grouping_data);
+	}
+}
+
+void WindowGlobalSinkState::SyncLocalPartition(GroupingPartition &local_partition, GroupingAppend &local_append) {
+	// We are done if the local_partition is right sized.
+	auto local_radix = (RadixPartitionedColumnData *)local_partition.get();
+	if (local_radix->GetRadixBits() == grouping_data->GetRadixBits()) {
+		return;
 	}
 
-	bool SinkChunk(DataChunk &sort_chunk, DataChunk &payload_chunk);
-	void Combine();
+	// If the local partition is now too small, flush it and reallocate
+	auto new_partition = grouping_data->CreateShared();
+	auto new_append = make_unique<PartitionedColumnDataAppendState>();
+	new_partition->InitializeAppendState(*new_append);
 
-	WindowGlobalHashGroup &global_group;
-	LocalSortStatePtr local_sort;
-	idx_t count;
-};
+	local_partition->FlushAppendState(*local_append);
+	auto &local_groups = local_partition->GetPartitions();
+	for (auto &local_group : local_groups) {
+		ColumnDataScanState scanner;
+		local_group->InitializeScan(scanner);
+
+		DataChunk scan_chunk;
+		local_group->InitializeScanChunk(scan_chunk);
+		for (scan_chunk.Reset(); local_group->Scan(scanner, scan_chunk); scan_chunk.Reset()) {
+			new_partition->Append(*new_append, scan_chunk);
+		}
+	}
+
+	// The append state has stale pointers to the old local partition, so nuke it from orbit.
+	new_partition->FlushAppendState(*new_append);
+
+	local_partition = std::move(new_partition);
+	local_append = make_unique<PartitionedColumnDataAppendState>();
+	local_partition->InitializeAppendState(*local_append);
+}
+
+void WindowGlobalSinkState::UpdateLocalPartition(GroupingPartition &local_partition, GroupingAppend &local_append) {
+	// Make sure grouping_data doesn't change under us.
+	lock_guard<mutex> guard(lock);
 
-bool WindowLocalHashGroup::SinkChunk(DataChunk &sort_buffer, DataChunk &input_chunk) {
-	D_ASSERT(sort_buffer.size() == input_chunk.size());
-	count += input_chunk.size();
-	auto &global_sort = *global_group.global_sort;
-	if (!local_sort) {
-		local_sort = make_unique<LocalSortState>();
-		local_sort->Initialize(global_sort, global_sort.buffer_manager);
+	if (!local_partition) {
+		local_partition = grouping_data->CreateShared();
+		local_append = make_unique<PartitionedColumnDataAppendState>();
+		local_partition->InitializeAppendState(*local_append);
+		return;
 	}
 
-	local_sort->SinkChunk(sort_buffer, input_chunk);
+	// 	Grow the groups if they are too big
+	ResizeGroupingData(count);
+
+	//	Sync local partition to have the same bit count
+	SyncLocalPartition(local_partition, local_append);
+}
 
-	if (local_sort->SizeInBytes() >= global_group.memory_per_thread) {
-		local_sort->Sort(global_sort, true);
+void WindowGlobalSinkState::CombineLocalPartition(GroupingPartition &local_partition, GroupingAppend &local_append) {
+	if (!local_partition) {
+		return;
 	}
+	local_partition->FlushAppendState(*local_append);
 
-	return (local_sort->SizeInBytes() >= global_group.memory_per_thread);
+	// Make sure grouping_data doesn't change under us.
+	// Combine has an internal mutex, so this is single-threaded anyway.
+	lock_guard<mutex> guard(lock);
+	SyncLocalPartition(local_partition, local_append);
+	grouping_data->Combine(*local_partition);
 }
 
-void WindowLocalHashGroup::Combine() {
-	if (local_sort) {
-		global_group.Combine(*local_sort);
-		global_group.count += count;
-		local_sort.reset();
+void WindowGlobalSinkState::BuildSortState(ColumnDataCollection &group_data, WindowGlobalHashGroup &hash_group) {
+	auto &global_sort = *hash_group.global_sort;
+
+	//	 Set up the sort expression computation.
+	vector<LogicalType> sort_types;
+	ExpressionExecutor executor(context);
+	for (auto &order : orders) {
+		auto &oexpr = order.expression;
+		sort_types.emplace_back(oexpr->return_type);
+		executor.AddExpression(*oexpr);
 	}
+	DataChunk sort_chunk;
+	sort_chunk.Initialize(allocator, sort_types);
+
+	// Copy the data from the group into the sort code.
+	LocalSortState local_sort;
+	local_sort.Initialize(global_sort, global_sort.buffer_manager);
+
+	//	Strip hash column
+	DataChunk payload_chunk;
+	payload_chunk.Initialize(allocator, payload_types);
+
+	vector<column_t> column_ids;
+	column_ids.reserve(payload_types.size());
+	for (column_t i = 0; i < payload_types.size(); ++i) {
+		column_ids.emplace_back(i);
+	}
+	ColumnDataConsumer scanner(group_data, column_ids);
+	ColumnDataConsumerScanState chunk_state;
+	chunk_state.current_chunk_state.properties = ColumnDataScanProperties::ALLOW_ZERO_COPY;
+	scanner.InitializeScan();
+	for (auto chunk_idx = scanner.ChunkCount(); chunk_idx-- > 0;) {
+		if (!scanner.AssignChunk(chunk_state)) {
+			break;
+		}
+		scanner.ScanChunk(chunk_state, payload_chunk);
+
+		sort_chunk.Reset();
+		executor.Execute(payload_chunk, sort_chunk);
+
+		local_sort.SinkChunk(sort_chunk, payload_chunk);
+		if (local_sort.SizeInBytes() > memory_per_thread) {
+			local_sort.Sort(global_sort, true);
+		}
+		scanner.FinishChunk(chunk_state);
+	}
+
+	global_sort.AddLocalState(local_sort);
+
+	hash_group.count += group_data.Count();
 }
 
 //	Per-thread sink state
 class WindowLocalSinkState : public LocalSinkState {
 public:
-	using LocalHashGroupPtr = unique_ptr<WindowLocalHashGroup>;
-
 	WindowLocalSinkState(ClientContext &context, const PhysicalWindow &op_p)
-	    : op(op_p), executor(context), count(0), hash_vector(LogicalTypeId::UBIGINT), sel(STANDARD_VECTOR_SIZE) {
-
+	    : op(op_p), allocator(Allocator::Get(context)), executor(context) {
 		D_ASSERT(op.select_list[0]->GetExpressionClass() == ExpressionClass::BOUND_WINDOW);
 		auto wexpr = reinterpret_cast<BoundWindowExpression *>(op.select_list[0].get());
-		partition_cols = wexpr->partitions.size();
 
-		// we sort by both 1) partition by expression list and 2) order by expressions
-		auto &payload_types = op.children[0]->types;
-		vector<LogicalType> over_types;
+		vector<LogicalType> group_types;
 		for (idx_t prt_idx = 0; prt_idx < wexpr->partitions.size(); prt_idx++) {
 			auto &pexpr = wexpr->partitions[prt_idx];
-			over_types.push_back(pexpr->return_type);
+			group_types.push_back(pexpr->return_type);
 			executor.AddExpression(*pexpr);
 		}
+		sort_cols = wexpr->orders.size() + group_types.size();
 
-		for (const auto &order : wexpr->orders) {
-			auto &oexpr = order.expression;
-			over_types.push_back(oexpr->return_type);
-			executor.AddExpression(*oexpr);
-		}
-
-		auto &allocator = Allocator::Get(context);
-		if (!over_types.empty()) {
-			over_chunk.Initialize(allocator, over_types);
-			over_subset.Initialize(allocator, over_types);
+		if (sort_cols) {
+			if (!group_types.empty()) {
+				// OVER(PARTITION BY...)
+				group_chunk.Initialize(allocator, group_types);
+			}
+			// OVER(...)
+			auto payload_types = op.children[0]->types;
+			payload_types.emplace_back(LogicalType::HASH);
+			payload_chunk.Initialize(allocator, payload_types);
+		} else {
+			// OVER()
+			payload_layout.Initialize(op.children[0]->types);
 		}
-
-		payload_chunk.Initialize(allocator, payload_types);
-		payload_subset.Initialize(allocator, payload_types);
-		payload_layout.Initialize(payload_types);
 	}
 
 	// Global state
 	const PhysicalWindow &op;
+	Allocator &allocator;
 
-	// Input
+	// OVER(PARTITION BY...) (hash grouping)
 	ExpressionExecutor executor;
-	DataChunk over_chunk;
+	DataChunk group_chunk;
 	DataChunk payload_chunk;
-	idx_t count;
+	unique_ptr<PartitionedColumnData> local_partition;
+	unique_ptr<PartitionedColumnDataAppendState> local_append;
 
-	// Grouping
-	idx_t partition_cols;
-	counts_t counts;
-	counts_t offsets;
-	Vector hash_vector;
-	SelectionVector sel;
-	DataChunk over_subset;
-	DataChunk payload_subset;
-	LocalHashGroupPtr ungrouped;
-	vector<LocalHashGroupPtr> hash_groups;
+	// OVER(...) (sorting)
+	size_t sort_cols;
 
 	// OVER() (no sorting)
 	RowLayout payload_layout;
 	unique_ptr<RowDataCollection> rows;
 	unique_ptr<RowDataCollection> strings;
 
-	//! Switch to grouping the data
-	void Group(WindowGlobalSinkState &gstate);
-	//! Compute the OVER values
-	void Over(DataChunk &input_chunk);
-	//! Hash the data and group it
-	void Hash(WindowGlobalSinkState &gstate, DataChunk &input_chunk);
+	//! Compute the hash values
+	void Hash(DataChunk &input_chunk, Vector &hash_vector);
 	//! Sink an input chunk
 	void Sink(DataChunk &input_chunk, WindowGlobalSinkState &gstate);
 	//! Merge the state into the global state.
 	void Combine(WindowGlobalSinkState &gstate);
 };
 
-void WindowLocalSinkState::Over(DataChunk &input_chunk) {
-	if (over_chunk.ColumnCount() > 0) {
-		over_chunk.Reset();
-		executor.Execute(input_chunk, over_chunk);
-		over_chunk.Verify();
-	}
-}
-
-void WindowLocalSinkState::Hash(WindowGlobalSinkState &gstate, DataChunk &input_chunk) {
-	// There are three types of hash grouping:
-	// 1. No partitions (no sorting)
-	// 2. One group (sorting, but no hash grouping)
-	// 3. Multiple groups (sorting and hash grouping)
-	if (over_chunk.ColumnCount() == 0) {
-		return;
-	}
-
-	const auto count = over_chunk.size();
-	auto hashes = FlatVector::GetData<hash_t>(hash_vector);
-	if (hash_groups.empty()) {
-		// Ungrouped, so take them all
-		counts.resize(1, count);
-	} else {
-		// First pass: count bins sizes
-		counts.resize(0);
-		counts.resize(hash_groups.size(), 0);
-
-		VectorOperations::Hash(over_chunk.data[0], hash_vector, count);
-		for (idx_t prt_idx = 1; prt_idx < partition_cols; ++prt_idx) {
-			VectorOperations::CombineHash(hash_vector, over_chunk.data[prt_idx], count);
-		}
-
-		const auto &partition_info = gstate.partition_info;
-		if (hash_vector.GetVectorType() == VectorType::CONSTANT_VECTOR) {
-			const auto group = partition_info.GetHashPartition(hashes[0]);
-			counts[group] = count;
-			for (idx_t i = 0; i < count; ++i) {
-				sel.set_index(i, i);
-			}
-		} else {
-			for (idx_t i = 0; i < count; ++i) {
-				const auto group = partition_info.GetHashPartition(hashes[i]);
-				++counts[group];
-			}
-
-			// Second pass: Initialise offsets
-			offsets.resize(counts.size());
-			size_t offset = 0;
-			for (size_t c = 0; c < counts.size(); ++c) {
-				offsets[c] = offset;
-				offset += counts[c];
-			}
-
-			// Third pass: Build sequential selections
-			for (idx_t i = 0; i < count; ++i) {
-				const auto group = partition_info.GetHashPartition(hashes[i]);
-				auto &group_idx = offsets[group];
-				sel.set_index(group_idx++, i);
-			}
-		}
-	}
-
-	idx_t group_offset = 0;
-	for (size_t group = 0; group < counts.size(); ++group) {
-		const auto group_size = counts[group];
-		if (group_size) {
-			auto &local_group = hash_groups[group];
-			if (!local_group) {
-				auto global_group = gstate.GetHashGroup(group);
-				local_group = make_unique<WindowLocalHashGroup>(*global_group);
-			}
-
-			if (counts.size() == 1) {
-				local_group->SinkChunk(over_chunk, input_chunk);
-			} else {
-				SelectionVector psel(sel.data() + group_offset);
-				over_subset.Slice(over_chunk, psel, group_size);
-				payload_subset.Slice(input_chunk, psel, group_size);
-				local_group->SinkChunk(over_subset, payload_subset);
-				group_offset += group_size;
-			}
+void WindowLocalSinkState::Hash(DataChunk &input_chunk, Vector &hash_vector) {
+	const auto count = input_chunk.size();
+	if (group_chunk.ColumnCount() > 0) {
+		// OVER(PARTITION BY...) (hash grouping)
+		group_chunk.Reset();
+		executor.Execute(input_chunk, group_chunk);
+		VectorOperations::Hash(group_chunk.data[0], hash_vector, count);
+		for (idx_t prt_idx = 1; prt_idx < group_chunk.ColumnCount(); ++prt_idx) {
+			VectorOperations::CombineHash(hash_vector, group_chunk.data[prt_idx], count);
 		}
+	} else {
+		// OVER(...) (sorting)
+		// Single partition => single hash value
+		hash_vector.SetVectorType(VectorType::CONSTANT_VECTOR);
+		auto hashes = ConstantVector::GetData<hash_t>(hash_vector);
+		hashes[0] = 0;
 	}
 }
 
-void WindowLocalSinkState::Group(WindowGlobalSinkState &gstate) {
-	if (!gstate.partition_cols) {
-		return;
-	}
-
-	if (!hash_groups.empty()) {
-		return;
-	}
-
-	hash_groups.resize(gstate.Group());
-
-	if (!ungrouped) {
-		return;
-	}
-
-	auto &payload_data = *ungrouped->local_sort->payload_data;
-	auto rows = payload_data.CloneEmpty(payload_data.keep_pinned);
-
-	auto &payload_heap = *ungrouped->local_sort->payload_heap;
-	auto heap = payload_heap.CloneEmpty(payload_heap.keep_pinned);
-
-	RowDataCollectionScanner::AlignHeapBlocks(*rows, *heap, payload_data, payload_heap, payload_layout);
-	RowDataCollectionScanner scanner(*rows, *heap, payload_layout, true);
-	while (scanner.Remaining()) {
-		payload_chunk.Reset();
-		scanner.Scan(payload_chunk);
-
-		Over(payload_chunk);
-		Hash(gstate, payload_chunk);
-	}
-
-	ungrouped.reset();
-}
-
 void WindowLocalSinkState::Sink(DataChunk &input_chunk, WindowGlobalSinkState &gstate) {
 	gstate.count += input_chunk.size();
-	count += input_chunk.size();
-
-	Over(input_chunk);
 
 	// OVER()
-	if (over_chunk.ColumnCount() == 0) {
+	if (sort_cols == 0) {
 		//	No sorts, so build paged row chunks
 		if (!rows) {
 			const auto entry_size = payload_layout.GetRowWidth();
@@ -483,27 +431,23 @@ void WindowLocalSinkState::Sink(DataChunk &input_chunk, WindowGlobalSinkState &g
 		return;
 	}
 
-	// Ungrouped
-	if (hash_groups.empty()) {
-		auto global_ungrouped = gstate.GetUngrouped();
-		if (!ungrouped) {
-			ungrouped = make_unique<WindowLocalHashGroup>(*global_ungrouped);
-		}
+	// OVER(...)
+	gstate.UpdateLocalPartition(local_partition, local_append);
 
-		// If we pass our thread memory budget, then switch to hash grouping.
-		if (ungrouped->SinkChunk(over_chunk, input_chunk) || gstate.count > 100000) {
-			Group(gstate);
-		}
-		return;
+	payload_chunk.Reset();
+	auto &hash_vector = payload_chunk.data.back();
+	Hash(input_chunk, hash_vector);
+	for (idx_t col_idx = 0; col_idx < input_chunk.ColumnCount(); ++col_idx) {
+		payload_chunk.data[col_idx].Reference(input_chunk.data[col_idx]);
 	}
+	payload_chunk.SetCardinality(input_chunk);
 
-	// Grouped, so hash
-	Hash(gstate, input_chunk);
+	local_partition->Append(*local_append, payload_chunk);
 }
 
 void WindowLocalSinkState::Combine(WindowGlobalSinkState &gstate) {
 	// OVER()
-	if (over_chunk.ColumnCount() == 0) {
+	if (sort_cols == 0) {
 		// Only one partition again, so need a global lock.
 		lock_guard<mutex> glock(gstate.lock);
 		if (gstate.rows) {
@@ -514,95 +458,20 @@ void WindowLocalSinkState::Combine(WindowGlobalSinkState &gstate) {
 				strings.reset();
 			}
 		} else {
-			gstate.rows = move(rows);
-			gstate.strings = move(strings);
+			gstate.rows = std::move(rows);
+			gstate.strings = std::move(strings);
 		}
 		return;
 	}
 
-	// Ungrouped data
-	idx_t check = 0;
-	if (ungrouped) {
-		check += ungrouped->count;
-		ungrouped->Combine();
-		ungrouped.reset();
-	}
-
-	// Grouped data
-	for (auto &local_group : hash_groups) {
-		if (local_group) {
-			check += local_group->count;
-			local_group->Combine();
-			local_group.reset();
-		}
-	}
-
-	(void)check;
-	D_ASSERT(check == count);
-}
-
-void WindowGlobalSinkState::Finalize() {
-	if (!ungrouped) {
-		return;
-	}
-
-	if (hash_groups.empty()) {
-		hash_groups.emplace_back(move(ungrouped));
-		return;
-	}
-
-	//	If we have grouped data, merge the remaining ungrouped data into it.
-	//	This can happen if only SOME of the threads ended up regrouping.
-	//	The simplest thing to do is to fake a local thread state,
-	//	push the ungrouped data into it and then use that state to partition the data.
-	// 	This is probably OK because this situation will only happen
-	// 	with relatively small amounts of data.
-
-	//	Sort the data so we can scan it
-	auto &global_sort = *ungrouped->global_sort;
-	if (global_sort.sorted_blocks.empty()) {
-		return;
-	}
-
-	global_sort.PrepareMergePhase();
-	while (global_sort.sorted_blocks.size() > 1) {
-		global_sort.InitializeMergeRound();
-		MergeSorter merge_sorter(global_sort, global_sort.buffer_manager);
-		merge_sorter.PerformInMergeRound();
-		global_sort.CompleteMergeRound(true);
-	}
-
-	// 	Sink it into a temporary local sink state
-	auto lstate = make_unique<WindowLocalSinkState>(context, op);
-
-	//	Match the grouping.
-	lstate->Group(*this);
-
-	// Write into the state chunks directly to hash them
-	auto &payload_chunk = lstate->payload_chunk;
-
-	// 	Now scan the sorted data
-	PayloadScanner scanner(global_sort);
-	while (scanner.Remaining()) {
-		lstate->payload_chunk.Reset();
-		scanner.Scan(lstate->payload_chunk);
-		if (payload_chunk.size() == 0) {
-			break;
-		}
-		lstate->count += payload_chunk.size();
-
-		lstate->Over(payload_chunk);
-		lstate->Hash(*this, payload_chunk);
-	}
-
-	//	Merge the grouped data in.
-	lstate->Combine(*this);
+	// OVER(...)
+	gstate.CombineLocalPartition(local_partition, local_append);
 }
 
 // this implements a sorted window functions variant
 PhysicalWindow::PhysicalWindow(vector<LogicalType> types, vector<unique_ptr<Expression>> select_list_p,
                                idx_t estimated_cardinality, PhysicalOperatorType type)
-    : PhysicalOperator(type, move(types), estimated_cardinality), select_list(move(select_list_p)) {
+    : PhysicalOperator(type, std::move(types), estimated_cardinality), select_list(std::move(select_list_p)) {
 	is_order_dependent = false;
 	for (auto &expr : select_list) {
 		D_ASSERT(expr->expression_class == ExpressionClass::BOUND_WINDOW);
@@ -1215,6 +1084,7 @@ WindowExecutor::WindowExecutor(BoundWindowExpression *wexpr, ClientContext &cont
 
 	// evaluate inner expressions of window functions, could be more complex
 	vector<Expression *> exprs;
+	exprs.reserve(wexpr->children.size());
 	for (auto &child : wexpr->children) {
 		exprs.push_back(child.get());
 	}
@@ -1521,6 +1391,10 @@ public:
 	bool TaskFinished() {
 		return finished;
 	}
+
+	void Prepare();
+	void Merge();
+
 	void ExecuteTask();
 
 	WindowGlobalMergeState *merge_state;
@@ -1530,8 +1404,19 @@ public:
 
 class WindowGlobalMergeState {
 public:
-	explicit WindowGlobalMergeState(GlobalSortState &sort_state)
-	    : sort_state(sort_state), stage(WindowSortStage::INIT), total_tasks(0), tasks_assigned(0), tasks_completed(0) {
+	using GroupDataPtr = unique_ptr<ColumnDataCollection>;
+
+	explicit WindowGlobalMergeState(WindowGlobalSinkState &sink, GroupDataPtr group_data)
+	    : sink(sink), group_data(std::move(group_data)), stage(WindowSortStage::INIT), total_tasks(0),
+	      tasks_assigned(0), tasks_completed(0) {
+
+		const auto group_idx = sink.hash_groups.size();
+		auto new_group = make_unique<WindowGlobalHashGroup>(sink.buffer_manager, sink.partitions, sink.orders,
+		                                                    sink.payload_types, sink.external);
+		sink.hash_groups.emplace_back(std::move(new_group));
+
+		hash_group = sink.hash_groups[group_idx].get();
+		global_sort = sink.hash_groups[group_idx]->global_sort.get();
 	}
 
 	bool IsSorted() const {
@@ -1543,7 +1428,10 @@ public:
 	bool TryPrepareNextStage();
 	void CompleteTask();
 
-	GlobalSortState &sort_state;
+	WindowGlobalSinkState &sink;
+	GroupDataPtr group_data;
+	WindowGlobalHashGroup *hash_group;
+	GlobalSortState *global_sort;
 
 private:
 	mutable mutex lock;
@@ -1553,17 +1441,28 @@ private:
 	idx_t tasks_completed;
 };
 
+void WindowLocalMergeState::Prepare() {
+	auto &global_sort = *merge_state->global_sort;
+	merge_state->sink.BuildSortState(*merge_state->group_data, *merge_state->hash_group);
+	merge_state->group_data.reset();
+
+	global_sort.PrepareMergePhase();
+}
+
+void WindowLocalMergeState::Merge() {
+	auto &global_sort = *merge_state->global_sort;
+	MergeSorter merge_sorter(global_sort, global_sort.buffer_manager);
+	merge_sorter.PerformInMergeRound();
+}
+
 void WindowLocalMergeState::ExecuteTask() {
-	auto &global_sort = merge_state->sort_state;
 	switch (stage) {
 	case WindowSortStage::PREPARE:
-		global_sort.PrepareMergePhase();
+		Prepare();
 		break;
-	case WindowSortStage::MERGE: {
-		MergeSorter merge_sorter(global_sort, global_sort.buffer_manager);
-		merge_sorter.PerformInMergeRound();
+	case WindowSortStage::MERGE:
+		Merge();
 		break;
-	}
 	default:
 		throw InternalException("Unexpected WindowGlobalMergeState in ExecuteTask!");
 	}
@@ -1609,21 +1508,21 @@ bool WindowGlobalMergeState::TryPrepareNextStage() {
 		return true;
 
 	case WindowSortStage::PREPARE:
-		total_tasks = sort_state.sorted_blocks.size() / 2;
+		total_tasks = global_sort->sorted_blocks.size() / 2;
 		if (!total_tasks) {
 			break;
 		}
 		stage = WindowSortStage::MERGE;
-		sort_state.InitializeMergeRound();
+		global_sort->InitializeMergeRound();
 		return true;
 
 	case WindowSortStage::MERGE:
-		sort_state.CompleteMergeRound(true);
-		total_tasks = sort_state.sorted_blocks.size() / 2;
+		global_sort->CompleteMergeRound(true);
+		total_tasks = global_sort->sorted_blocks.size() / 2;
 		if (!total_tasks) {
 			break;
 		}
-		sort_state.InitializeMergeRound();
+		global_sort->InitializeMergeRound();
 		return true;
 
 	case WindowSortStage::SORTED:
@@ -1639,14 +1538,14 @@ class WindowGlobalMergeStates {
 public:
 	using WindowGlobalMergeStatePtr = unique_ptr<WindowGlobalMergeState>;
 
-	WindowGlobalMergeStates(WindowGlobalSinkState &sink, idx_t group) {
+	explicit WindowGlobalMergeStates(WindowGlobalSinkState &sink) {
 		// Schedule all the sorts for maximum thread utilisation
-		for (; group < sink.hash_groups.size(); group = sink.GetNextSortGroup()) {
-			auto &hash_group = *sink.hash_groups[group];
-
+		for (auto &group_data : sink.grouping_data->GetPartitions()) {
 			// Prepare for merge sort phase
-			auto state = make_unique<WindowGlobalMergeState>(*hash_group.global_sort);
-			states.emplace_back(move(state));
+			if (group_data->Count()) {
+				auto state = make_unique<WindowGlobalMergeState>(sink, std::move(group_data));
+				states.emplace_back(std::move(state));
+			}
 		}
 	}
 
@@ -1656,7 +1555,7 @@ public:
 class WindowMergeTask : public ExecutorTask {
 public:
 	WindowMergeTask(shared_ptr<Event> event_p, ClientContext &context_p, WindowGlobalMergeStates &hash_groups_p)
-	    : ExecutorTask(context_p), event(move(event_p)), hash_groups(hash_groups_p) {
+	    : ExecutorTask(context_p), event(std::move(event_p)), hash_groups(hash_groups_p) {
 	}
 
 	TaskExecutionResult ExecuteTask(TaskExecutionMode mode) override;
@@ -1726,8 +1625,8 @@ TaskExecutionResult WindowMergeTask::ExecuteTask(TaskExecutionMode mode) {
 
 class WindowMergeEvent : public BasePipelineEvent {
 public:
-	WindowMergeEvent(WindowGlobalSinkState &gstate_p, Pipeline &pipeline_p, idx_t group)
-	    : BasePipelineEvent(pipeline_p), gstate(gstate_p), merge_states(gstate_p, group) {
+	WindowMergeEvent(WindowGlobalSinkState &gstate_p, Pipeline &pipeline_p)
+	    : BasePipelineEvent(pipeline_p), gstate(gstate_p), merge_states(gstate_p) {
 	}
 
 	WindowGlobalSinkState &gstate;
@@ -1745,7 +1644,7 @@ public:
 		for (idx_t tnum = 0; tnum < num_threads; tnum++) {
 			merge_tasks.push_back(make_unique<WindowMergeTask>(shared_from_this(), context, merge_states));
 		}
-		SetTasks(move(merge_tasks));
+		SetTasks(std::move(merge_tasks));
 	}
 };
 
@@ -1753,24 +1652,27 @@ SinkFinalizeType PhysicalWindow::Finalize(Pipeline &pipeline, Event &event, Clie
                                           GlobalSinkState &gstate_p) const {
 	auto &state = (WindowGlobalSinkState &)gstate_p;
 
+	//	Did we get any data?
+	if (!state.count) {
+		return SinkFinalizeType::NO_OUTPUT_POSSIBLE;
+	}
+
 	// Do we have any sorting to schedule?
 	if (state.rows) {
-		D_ASSERT(state.hash_groups.empty());
+		D_ASSERT(!state.grouping_data);
 		return state.rows->count ? SinkFinalizeType::READY : SinkFinalizeType::NO_OUTPUT_POSSIBLE;
 	}
 
 	// Find the first group to sort
-	state.Finalize();
-	D_ASSERT(state.count == state.GroupCount());
-	auto group = state.GetNextSortGroup();
-	if (group >= state.hash_groups.size()) {
+	auto &groups = state.grouping_data->GetPartitions();
+	if (groups.empty()) {
 		// Empty input!
 		return SinkFinalizeType::NO_OUTPUT_POSSIBLE;
 	}
 
 	// Schedule all the sorts for maximum thread utilisation
-	auto new_event = make_shared<WindowMergeEvent>(state, pipeline, group);
-	event.InsertEvent(move(new_event));
+	auto new_event = make_shared<WindowMergeEvent>(state, pipeline);
+	event.InsertEvent(std::move(new_event));
 
 	return SinkFinalizeType::READY;
 }
@@ -1792,18 +1694,16 @@ public:
 		auto &state = (WindowGlobalSinkState &)*op.sink_state;
 
 		// If there is only one partition, we have to process it on one thread.
-		if (state.hash_groups.empty()) {
+		if (!state.grouping_data) {
 			return 1;
 		}
 
-		idx_t max_threads = 0;
-		for (const auto &hash_group : state.hash_groups) {
-			if (hash_group) {
-				max_threads++;
-			}
+		// If there is not a lot of data, process serially.
+		if (state.count < STANDARD_ROW_GROUPS_SIZE) {
+			return 1;
 		}
 
-		return max_threads;
+		return state.hash_groups.size();
 	}
 };
 
@@ -1814,8 +1714,8 @@ public:
 	using WindowExecutorPtr = unique_ptr<WindowExecutor>;
 	using WindowExecutors = vector<WindowExecutorPtr>;
 
-	WindowLocalSourceState(Allocator &allocator_p, const PhysicalWindow &op, ExecutionContext &context)
-	    : context(context.client), allocator(allocator_p) {
+	WindowLocalSourceState(const PhysicalWindow &op, ExecutionContext &context, WindowGlobalSourceState &gstate)
+	    : context(context.client), allocator(Allocator::Get(context.client)) {
 		vector<LogicalType> output_types;
 		for (idx_t expr_idx = 0; expr_idx < op.select_list.size(); ++expr_idx) {
 			D_ASSERT(op.select_list[expr_idx]->GetExpressionClass() == ExpressionClass::BOUND_WINDOW);
@@ -1882,7 +1782,7 @@ void WindowLocalSourceState::MaterializeSortedData() {
 	D_ASSERT(!sd.data_blocks.empty());
 	auto &block = sd.data_blocks[0];
 	rows = make_unique<RowDataCollection>(buffer_manager, block->capacity, block->entry_size);
-	rows->blocks = move(sd.data_blocks);
+	rows->blocks = std::move(sd.data_blocks);
 	rows->count = std::accumulate(rows->blocks.begin(), rows->blocks.end(), idx_t(0),
 	                              [&](idx_t c, const unique_ptr<RowDataBlock> &b) { return c + b->count; });
 
@@ -1890,7 +1790,7 @@ void WindowLocalSourceState::MaterializeSortedData() {
 	if (!sd.heap_blocks.empty()) {
 		auto &block = sd.heap_blocks[0];
 		heap = make_unique<RowDataCollection>(buffer_manager, block->capacity, block->entry_size);
-		heap->blocks = move(sd.heap_blocks);
+		heap->blocks = std::move(sd.heap_blocks);
 		hash_group.reset();
 	} else {
 		heap = make_unique<RowDataCollection>(buffer_manager, (idx_t)Storage::BLOCK_SIZE, 1, true);
@@ -1904,7 +1804,6 @@ void WindowLocalSourceState::GeneratePartition(WindowGlobalSinkState &gstate, co
 
 	//	Get rid of any stale data
 	hash_bin = hash_bin_p;
-	hash_group.reset();
 
 	// There are three types of partitions:
 	// 1. No partition (no sorting)
@@ -1927,7 +1826,7 @@ void WindowLocalSourceState::GeneratePartition(WindowGlobalSinkState &gstate, co
 		D_ASSERT(op.select_list[expr_idx]->GetExpressionClass() == ExpressionClass::BOUND_WINDOW);
 		auto wexpr = reinterpret_cast<BoundWindowExpression *>(op.select_list[expr_idx].get());
 		auto wexec = make_unique<WindowExecutor>(wexpr, context, count);
-		window_execs.emplace_back(move(wexec));
+		window_execs.emplace_back(std::move(wexec));
 	}
 
 	//	Initialise masks to false
@@ -1953,7 +1852,7 @@ void WindowLocalSourceState::GeneratePartition(WindowGlobalSinkState &gstate, co
 		external = true;
 	} else if (hash_bin < gstate.hash_groups.size() && gstate.hash_groups[hash_bin]) {
 		// Overwrite the collections with the sorted data
-		hash_group = move(gstate.hash_groups[hash_bin]);
+		hash_group = std::move(gstate.hash_groups[hash_bin]);
 		hash_group->ComputeMasks(partition_mask, order_mask);
 		MaterializeSortedData();
 	} else {
@@ -1987,13 +1886,12 @@ void WindowLocalSourceState::GeneratePartition(WindowGlobalSinkState &gstate, co
 	scanner->ReSwizzle();
 
 	//	Second pass can flush
-	scanner = make_unique<RowDataCollectionScanner>(*rows, *heap, layout, external, true);
+	scanner->Reset(true);
 }
 
 void WindowLocalSourceState::Scan(DataChunk &result) {
 	D_ASSERT(scanner);
 	if (!scanner->Remaining()) {
-		scanner.reset();
 		return;
 	}
 
@@ -2021,8 +1919,9 @@ void WindowLocalSourceState::Scan(DataChunk &result) {
 }
 
 unique_ptr<LocalSourceState> PhysicalWindow::GetLocalSourceState(ExecutionContext &context,
-                                                                 GlobalSourceState &gstate) const {
-	return make_unique<WindowLocalSourceState>(Allocator::Get(context.client), *this, context);
+                                                                 GlobalSourceState &gstate_p) const {
+	auto &gstate = (WindowGlobalSourceState &)gstate_p;
+	return make_unique<WindowLocalSourceState>(*this, context, gstate);
 }
 
 unique_ptr<GlobalSourceState> PhysicalWindow::GetGlobalSourceState(ClientContext &context) const {
@@ -2037,25 +1936,28 @@ void PhysicalWindow::GetData(ExecutionContext &context, DataChunk &chunk, Global
 
 	const auto bin_count = gstate.hash_groups.empty() ? 1 : gstate.hash_groups.size();
 
-	//	Move to the next bin if we are done.
-	while (!state.scanner || !state.scanner->Remaining()) {
-		state.scanner.reset();
-		state.rows.reset();
-		state.heap.reset();
-		auto hash_bin = global_source.next_bin++;
-		if (hash_bin >= bin_count) {
-			return;
-		}
+	while (chunk.size() == 0) {
+		//	Move to the next bin if we are done.
+		while (!state.scanner || !state.scanner->Remaining()) {
+			state.scanner.reset();
+			state.rows.reset();
+			state.heap.reset();
+			state.hash_group.reset();
+			auto hash_bin = global_source.next_bin++;
+			if (hash_bin >= bin_count) {
+				return;
+			}
 
-		for (; hash_bin < gstate.hash_groups.size(); hash_bin = global_source.next_bin++) {
-			if (gstate.hash_groups[hash_bin]) {
-				break;
+			for (; hash_bin < gstate.hash_groups.size(); hash_bin = global_source.next_bin++) {
+				if (gstate.hash_groups[hash_bin]) {
+					break;
+				}
 			}
+			state.GeneratePartition(gstate, hash_bin);
 		}
-		state.GeneratePartition(gstate, hash_bin);
-	}
 
-	state.Scan(chunk);
+		state.Scan(chunk);
+	}
 }
 
 string PhysicalWindow::ParamsToString() const {