duckdb 1.0.1-dev21.0 → 1.0.1-dev27.0

package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp

@@ -30,22 +30,148 @@
 namespace duckdb {
 
 //	Global sink state
-class WindowGlobalSinkState : public GlobalSinkState {
+class WindowGlobalSinkState;
+
+enum WindowGroupStage : uint8_t { SINK, FINALIZE, GETDATA, DONE };
+
+class WindowHashGroup {
 public:
-	WindowGlobalSinkState(const PhysicalWindow &op, ClientContext &context)
-	    : op(op), mode(DBConfig::GetConfig(context).options.window_mode) {
+	using HashGroupPtr = unique_ptr<PartitionGlobalHashGroup>;
+	using OrderMasks = PartitionGlobalHashGroup::OrderMasks;
+	using ExecutorGlobalStatePtr = unique_ptr<WindowExecutorGlobalState>;
+	using ExecutorGlobalStates = vector<ExecutorGlobalStatePtr>;
+	using ExecutorLocalStatePtr = unique_ptr<WindowExecutorLocalState>;
+	using ExecutorLocalStates = vector<ExecutorLocalStatePtr>;
+	using ThreadLocalStates = vector<ExecutorLocalStates>;
+
+	WindowHashGroup(WindowGlobalSinkState &gstate, const idx_t hash_bin_p);
 
-		D_ASSERT(op.select_list[op.order_idx]->GetExpressionClass() == ExpressionClass::BOUND_WINDOW);
-		auto &wexpr = op.select_list[op.order_idx]->Cast<BoundWindowExpression>();
+	ExecutorGlobalStates &Initialize(WindowGlobalSinkState &gstate);
 
-		global_partition =
-		    make_uniq<PartitionGlobalSinkState>(context, wexpr.partitions, wexpr.orders, op.children[0]->types,
-		                                        wexpr.partitions_stats, op.estimated_cardinality);
+	// Scan all of the blocks during the build phase
+	unique_ptr<RowDataCollectionScanner> GetBuildScanner(idx_t block_idx) const {
+		if (!rows) {
+			return nullptr;
+		}
+		return make_uniq<RowDataCollectionScanner>(*rows, *heap, layout, external, block_idx, false);
 	}
 
+	// Scan a single block during the evaluate phase
+	unique_ptr<RowDataCollectionScanner> GetEvaluateScanner(idx_t block_idx) const {
+		//	Second pass can flush
+		D_ASSERT(rows);
+		return make_uniq<RowDataCollectionScanner>(*rows, *heap, layout, external, block_idx, true);
+	}
+
+	// The processing stage for this group
+	WindowGroupStage GetStage() const {
+		return stage;
+	}
+
+	bool TryPrepareNextStage() {
+		lock_guard<mutex> prepare_guard(lock);
+		switch (stage.load()) {
+		case WindowGroupStage::SINK:
+			if (sunk == count) {
+				stage = WindowGroupStage::FINALIZE;
+				return true;
+			}
+			return false;
+		case WindowGroupStage::FINALIZE:
+			if (finalized == blocks) {
+				stage = WindowGroupStage::GETDATA;
+				return true;
+			}
+			return false;
+		default:
+			// never block in GETDATA
+			return true;
+		}
+	}
+
+	//! The hash partition data
+	HashGroupPtr hash_group;
+	//! The size of the group
+	idx_t count = 0;
+	//! The number of blocks in the group
+	idx_t blocks = 0;
+	unique_ptr<RowDataCollection> rows;
+	unique_ptr<RowDataCollection> heap;
+	RowLayout layout;
+	//! The partition boundary mask
+	ValidityMask partition_mask;
+	//! The order boundary mask
+	OrderMasks order_masks;
+	//! External paging
+	bool external;
+	// The processing stage for this group
+	atomic<WindowGroupStage> stage;
+	//! The function global states for this hash group
+	ExecutorGlobalStates gestates;
+	//! Executor local states, one per thread
+	ThreadLocalStates thread_states;
+
+	//! The bin number
+	idx_t hash_bin;
+	//! Single threading lock
+	mutex lock;
+	//! Count of sunk rows
+	std::atomic<idx_t> sunk;
+	//! Count of finalized blocks
+	std::atomic<idx_t> finalized;
+	//! The number of tasks left before we should be deleted
+	std::atomic<idx_t> tasks_remaining;
+	//! The output ordering batch index this hash group starts at
+	idx_t batch_base;
+
+private:
+	void MaterializeSortedData();
+};
+
+class WindowPartitionGlobalSinkState;
+
+class WindowGlobalSinkState : public GlobalSinkState {
+public:
+	using ExecutorPtr = unique_ptr<WindowExecutor>;
+	using Executors = vector<ExecutorPtr>;
+
+	WindowGlobalSinkState(const PhysicalWindow &op, ClientContext &context);
+
+	//! Parent operator
 	const PhysicalWindow &op;
-	unique_ptr<PartitionGlobalSinkState> global_partition;
-	WindowAggregationMode mode;
+	//! Execution context
+	ClientContext &context;
+	//! The partitioned sunk data
+	unique_ptr<WindowPartitionGlobalSinkState> global_partition;
+	//! The execution functions
+	Executors executors;
+};
+
+class WindowPartitionGlobalSinkState : public PartitionGlobalSinkState {
+public:
+	using WindowHashGroupPtr = unique_ptr<WindowHashGroup>;
+
+	WindowPartitionGlobalSinkState(WindowGlobalSinkState &gsink, const BoundWindowExpression &wexpr)
+	    : PartitionGlobalSinkState(gsink.context, wexpr.partitions, wexpr.orders, gsink.op.children[0]->types,
+	                               wexpr.partitions_stats, gsink.op.estimated_cardinality),
+	      gsink(gsink) {
+	}
+	~WindowPartitionGlobalSinkState() override = default;
+
+	void OnBeginMerge() override {
+		PartitionGlobalSinkState::OnBeginMerge();
+		window_hash_groups.resize(hash_groups.size());
+	}
+
+	void OnSortedPartition(const idx_t group_idx) override {
+		PartitionGlobalSinkState::OnSortedPartition(group_idx);
+		window_hash_groups[group_idx] = make_uniq<WindowHashGroup>(gsink, group_idx);
+	}
+
+	//! Operator global sink state
+	WindowGlobalSinkState &gsink;
+	//! The sorted hash groups
+	vector<WindowHashGroupPtr> window_hash_groups;
 };
 
 //	Per-thread sink state
@@ -89,39 +215,54 @@ PhysicalWindow::PhysicalWindow(vector<LogicalType> types, vector<unique_ptr<Expr
 }
 
 static unique_ptr<WindowExecutor> WindowExecutorFactory(BoundWindowExpression &wexpr, ClientContext &context,
-                                                        const ValidityMask &partition_mask,
-                                                        const ValidityMask &order_mask, const idx_t payload_count,
                                                         WindowAggregationMode mode) {
 	switch (wexpr.type) {
 	case ExpressionType::WINDOW_AGGREGATE:
-		return make_uniq<WindowAggregateExecutor>(wexpr, context, payload_count, partition_mask, order_mask, mode);
+		return make_uniq<WindowAggregateExecutor>(wexpr, context, mode);
 	case ExpressionType::WINDOW_ROW_NUMBER:
-		return make_uniq<WindowRowNumberExecutor>(wexpr, context, payload_count, partition_mask, order_mask);
+		return make_uniq<WindowRowNumberExecutor>(wexpr, context);
 	case ExpressionType::WINDOW_RANK_DENSE:
-		return make_uniq<WindowDenseRankExecutor>(wexpr, context, payload_count, partition_mask, order_mask);
+		return make_uniq<WindowDenseRankExecutor>(wexpr, context);
 	case ExpressionType::WINDOW_RANK:
-		return make_uniq<WindowRankExecutor>(wexpr, context, payload_count, partition_mask, order_mask);
+		return make_uniq<WindowRankExecutor>(wexpr, context);
 	case ExpressionType::WINDOW_PERCENT_RANK:
-		return make_uniq<WindowPercentRankExecutor>(wexpr, context, payload_count, partition_mask, order_mask);
+		return make_uniq<WindowPercentRankExecutor>(wexpr, context);
 	case ExpressionType::WINDOW_CUME_DIST:
-		return make_uniq<WindowCumeDistExecutor>(wexpr, context, payload_count, partition_mask, order_mask);
+		return make_uniq<WindowCumeDistExecutor>(wexpr, context);
 	case ExpressionType::WINDOW_NTILE:
-		return make_uniq<WindowNtileExecutor>(wexpr, context, payload_count, partition_mask, order_mask);
+		return make_uniq<WindowNtileExecutor>(wexpr, context);
 	case ExpressionType::WINDOW_LEAD:
 	case ExpressionType::WINDOW_LAG:
-		return make_uniq<WindowLeadLagExecutor>(wexpr, context, payload_count, partition_mask, order_mask);
+		return make_uniq<WindowLeadLagExecutor>(wexpr, context);
 	case ExpressionType::WINDOW_FIRST_VALUE:
-		return make_uniq<WindowFirstValueExecutor>(wexpr, context, payload_count, partition_mask, order_mask);
+		return make_uniq<WindowFirstValueExecutor>(wexpr, context);
 	case ExpressionType::WINDOW_LAST_VALUE:
-		return make_uniq<WindowLastValueExecutor>(wexpr, context, payload_count, partition_mask, order_mask);
+		return make_uniq<WindowLastValueExecutor>(wexpr, context);
 	case ExpressionType::WINDOW_NTH_VALUE:
-		return make_uniq<WindowNthValueExecutor>(wexpr, context, payload_count, partition_mask, order_mask);
+		return make_uniq<WindowNthValueExecutor>(wexpr, context);
 		break;
 	default:
 		throw InternalException("Window aggregate type %s", ExpressionTypeToString(wexpr.type));
 	}
 }
 
+WindowGlobalSinkState::WindowGlobalSinkState(const PhysicalWindow &op, ClientContext &context)
+    : op(op), context(context) {
+
+	D_ASSERT(op.select_list[op.order_idx]->GetExpressionClass() == ExpressionClass::BOUND_WINDOW);
+	auto &wexpr = op.select_list[op.order_idx]->Cast<BoundWindowExpression>();
+
+	const auto mode = DBConfig::GetConfig(context).options.window_mode;
+	for (idx_t expr_idx = 0; expr_idx < op.select_list.size(); ++expr_idx) {
+		D_ASSERT(op.select_list[expr_idx]->GetExpressionClass() == ExpressionClass::BOUND_WINDOW);
+		auto &wexpr = op.select_list[expr_idx]->Cast<BoundWindowExpression>();
+		auto wexec = WindowExecutorFactory(wexpr, context, mode);
+		executors.emplace_back(std::move(wexec));
+	}
+
+	global_partition = make_uniq<WindowPartitionGlobalSinkState>(*this, wexpr);
+}
+
 //===--------------------------------------------------------------------===//
 // Sink
 //===--------------------------------------------------------------------===//
@@ -171,7 +312,7 @@ SinkFinalizeType PhysicalWindow::Finalize(Pipeline &pipeline, Event &event, Clie
 	}
 
 	// Schedule all the sorts for maximum thread utilisation
-	auto new_event = make_shared_ptr<PartitionMergeEvent>(*state.global_partition, pipeline);
+	auto new_event = make_shared_ptr<PartitionMergeEvent>(*state.global_partition, pipeline, *this);
 	event.InsertEvent(std::move(new_event));
 
 	return SinkFinalizeType::READY;
@@ -180,123 +321,156 @@ SinkFinalizeType PhysicalWindow::Finalize(Pipeline &pipeline, Event &event, Clie
 //===--------------------------------------------------------------------===//
 // Source
 //===--------------------------------------------------------------------===//
-class WindowPartitionSourceState;
-
 class WindowGlobalSourceState : public GlobalSourceState {
 public:
-	using HashGroupSourcePtr = unique_ptr<WindowPartitionSourceState>;
 	using ScannerPtr = unique_ptr<RowDataCollectionScanner>;
-	using Task = std::pair<WindowPartitionSourceState *, ScannerPtr>;
+
+	struct Task {
+		Task(WindowGroupStage stage, idx_t group_idx, idx_t max_idx)
+		    : stage(stage), group_idx(group_idx), thread_idx(0), max_idx(max_idx) {
+		}
+		WindowGroupStage stage;
+		//! The hash group
+		idx_t group_idx;
+		//! The thread index (for local state)
+		idx_t thread_idx;
+		//! The total block index count
+		idx_t max_idx;
+		//! The first block index count
+		idx_t begin_idx = 0;
+		//! The end block index count
+		idx_t end_idx = 0;
+	};
+	using TaskPtr = optional_ptr<Task>;
 
 	WindowGlobalSourceState(ClientContext &context_p, WindowGlobalSinkState &gsink_p);
 
-	//! Get the next task
-	Task NextTask(idx_t hash_bin);
+	//! Build task list
+	void CreateTaskList();
+
+	//! Are there any more tasks?
+	bool HasMoreTasks() const {
+		return !stopped && next_task < tasks.size();
+	}
+	bool HasUnfinishedTasks() const {
+		return !stopped && finished < tasks.size();
+	}
+	//! Try to advance the group stage
+	bool TryPrepareNextStage();
+	//! Get the next task given the current state
+	bool TryNextTask(TaskPtr &task);
+	//! Finish a task
+	void FinishTask(TaskPtr task);
 
 	//! Context for executing computations
 	ClientContext &context;
 	//! All the sunk data
 	WindowGlobalSinkState &gsink;
-	//! The next group to build.
-	atomic<idx_t> next_build;
-	//! The built groups
-	vector<HashGroupSourcePtr> built;
-	//! Serialise access to the built hash groups
-	mutable mutex built_lock;
-	//! The number of unfinished tasks
-	atomic<idx_t> tasks_remaining;
+	//! The total number of blocks to process;
+	idx_t total_blocks = 0;
+	//! The number of local states
+	atomic<idx_t> locals;
+	//! The list of tasks
+	vector<Task> tasks;
+	//! The the next task
+	atomic<idx_t> next_task;
+	//! The the number of finished tasks
+	atomic<idx_t> finished;
+	//! Stop producing tasks
+	atomic<bool> stopped;
 	//! The number of rows returned
 	atomic<idx_t> returned;
 
 public:
 	idx_t MaxThreads() override {
-		return tasks_remaining;
+		return total_blocks;
 	}
-
-private:
-	Task CreateTask(idx_t hash_bin);
-	Task StealWork();
 };
 
 WindowGlobalSourceState::WindowGlobalSourceState(ClientContext &context_p, WindowGlobalSinkState &gsink_p)
-    : context(context_p), gsink(gsink_p), next_build(0), tasks_remaining(0), returned(0) {
-	auto &hash_groups = gsink.global_partition->hash_groups;
-
+    : context(context_p), gsink(gsink_p), locals(0), next_task(0), finished(0), stopped(false), returned(0) {
 	auto &gpart = gsink.global_partition;
-	if (hash_groups.empty()) {
+	auto &window_hash_groups = gsink.global_partition->window_hash_groups;
+
+	if (window_hash_groups.empty()) {
 		//	OVER()
-		built.resize(1);
-		if (gpart->rows) {
-			tasks_remaining += gpart->rows->blocks.size();
+		if (gpart->rows && !gpart->rows->blocks.empty()) {
+			// We need to construct the single WindowHashGroup here because the sort tasks will not be run.
+			window_hash_groups.emplace_back(make_uniq<WindowHashGroup>(gsink, idx_t(0)));
+			total_blocks = gpart->rows->blocks.size();
 		}
 	} else {
-		built.resize(hash_groups.size());
 		idx_t batch_base = 0;
-		for (auto &hash_group : hash_groups) {
-			if (!hash_group) {
+		for (auto &window_hash_group : window_hash_groups) {
+			if (!window_hash_group) {
 				continue;
 			}
-			auto &global_sort_state = *hash_group->global_sort;
-			if (global_sort_state.sorted_blocks.empty()) {
+			auto &rows = window_hash_group->rows;
+			if (!rows) {
 				continue;
 			}
 
-			D_ASSERT(global_sort_state.sorted_blocks.size() == 1);
-			auto &sb = *global_sort_state.sorted_blocks[0];
-			auto &sd = *sb.payload_data;
-			tasks_remaining += sd.data_blocks.size();
-
-			hash_group->batch_base = batch_base;
-			batch_base += sd.data_blocks.size();
+			const auto block_count = window_hash_group->rows->blocks.size();
+			window_hash_group->batch_base = batch_base;
+			batch_base += block_count;
 		}
+		total_blocks = batch_base;
 	}
 }
 
-// Per-bin evaluation state (build and evaluate)
-class WindowPartitionSourceState {
-public:
-	using HashGroupPtr = unique_ptr<PartitionGlobalHashGroup>;
-	using ExecutorPtr = unique_ptr<WindowExecutor>;
-	using Executors = vector<ExecutorPtr>;
-	using OrderMasks = PartitionGlobalHashGroup::OrderMasks;
-
-	WindowPartitionSourceState(ClientContext &context, WindowGlobalSourceState &gsource)
-	    : context(context), op(gsource.gsink.op), gsource(gsource), read_block_idx(0), unscanned(0) {
-		layout.Initialize(gsource.gsink.global_partition->payload_types);
+void WindowGlobalSourceState::CreateTaskList() {
+	//	Check whether we have a task list outside the mutex.
+	if (next_task.load()) {
+		return;
 	}
 
-	unique_ptr<RowDataCollectionScanner> GetScanner() const;
-	void MaterializeSortedData();
-	void BuildPartition(WindowGlobalSinkState &gstate, const idx_t hash_bin);
+	auto guard = Lock();
 
-	ClientContext &context;
-	const PhysicalWindow &op;
-	WindowGlobalSourceState &gsource;
-
-	HashGroupPtr hash_group;
-	//! The generated input chunks
-	unique_ptr<RowDataCollection> rows;
-	unique_ptr<RowDataCollection> heap;
-	RowLayout layout;
-	//! The partition boundary mask
-	ValidityMask partition_mask;
-	//! The order boundary mask
-	OrderMasks order_masks;
-	//! External paging
-	bool external;
-	//! The current execution functions
-	Executors executors;
+	auto &window_hash_groups = gsink.global_partition->window_hash_groups;
+	if (!tasks.empty()) {
+		return;
+	}
 
-	//! The bin number
-	idx_t hash_bin;
+	//    Sort the groups from largest to smallest
+	if (window_hash_groups.empty()) {
+		return;
+	}
 
-	//! The next block to read.
-	mutable atomic<idx_t> read_block_idx;
-	//! The number of remaining unscanned blocks.
-	atomic<idx_t> unscanned;
-};
+	using PartitionBlock = std::pair<idx_t, idx_t>;
+	vector<PartitionBlock> partition_blocks;
+	for (idx_t group_idx = 0; group_idx < window_hash_groups.size(); ++group_idx) {
+		auto &window_hash_group = window_hash_groups[group_idx];
+		partition_blocks.emplace_back(window_hash_group->rows->blocks.size(), group_idx);
+	}
+	std::sort(partition_blocks.begin(), partition_blocks.end(), std::greater<PartitionBlock>());
+
+	//	Schedule the largest group on as many threads as possible
+	const auto threads = locals.load();
+	const auto &max_block = partition_blocks.front();
+	const auto per_thread = (max_block.first + threads - 1) / threads;
+	if (!per_thread) {
+		throw InternalException("No blocks per thread! %ld threads, %ld groups, %ld blocks, %ld hash group", threads,
+		                        partition_blocks.size(), max_block.first, max_block.second);
+	}
+
+	//	TODO: Generate dynamically instead of building a big list?
+	vector<WindowGroupStage> states {WindowGroupStage::SINK, WindowGroupStage::FINALIZE, WindowGroupStage::GETDATA};
+	for (const auto &b : partition_blocks) {
+		auto &window_hash_group = *window_hash_groups[b.second];
+		for (const auto &state : states) {
+			idx_t thread_count = 0;
+			for (Task task(state, b.second, b.first); task.begin_idx < task.max_idx; task.begin_idx += per_thread) {
+				task.end_idx = MinValue<idx_t>(task.begin_idx + per_thread, task.max_idx);
+				tasks.emplace_back(task);
+				window_hash_group.tasks_remaining++;
+				thread_count = ++task.thread_idx;
+			}
+			window_hash_group.thread_states.resize(thread_count);
+		}
+	}
+}
 
-void WindowPartitionSourceState::MaterializeSortedData() {
+void WindowHashGroup::MaterializeSortedData() {
 	auto &global_sort_state = *hash_group->global_sort;
 	if (global_sort_state.sorted_blocks.empty()) {
 		return;
@@ -329,38 +503,23 @@ void WindowPartitionSourceState::MaterializeSortedData() {
 		heap->blocks = std::move(sd.heap_blocks);
 		hash_group.reset();
 	} else {
-		heap = make_uniq<RowDataCollection>(buffer_manager, (idx_t)Storage::BLOCK_SIZE, 1U, true);
+		heap = make_uniq<RowDataCollection>(buffer_manager, buffer_manager.GetBlockSize(), 1U, true);
 	}
 	heap->count = std::accumulate(heap->blocks.begin(), heap->blocks.end(), idx_t(0),
 	                              [&](idx_t c, const unique_ptr<RowDataBlock> &b) { return c + b->count; });
 }
 
-unique_ptr<RowDataCollectionScanner> WindowPartitionSourceState::GetScanner() const {
-	auto &gsink = *gsource.gsink.global_partition;
-	if ((gsink.rows && !hash_bin) || hash_bin < gsink.hash_groups.size()) {
-		const auto block_idx = read_block_idx++;
-		if (block_idx >= rows->blocks.size()) {
-			return nullptr;
-		}
-		//	Second pass can flush
-		--gsource.tasks_remaining;
-		return make_uniq<RowDataCollectionScanner>(*rows, *heap, layout, external, block_idx, true);
-	}
-	return nullptr;
-}
-
-void WindowPartitionSourceState::BuildPartition(WindowGlobalSinkState &gstate, const idx_t hash_bin_p) {
-	//	Get rid of any stale data
-	hash_bin = hash_bin_p;
-
+WindowHashGroup::WindowHashGroup(WindowGlobalSinkState &gstate, const idx_t hash_bin_p)
+    : count(0), blocks(0), stage(WindowGroupStage::SINK), hash_bin(hash_bin_p), sunk(0), finalized(0),
+      tasks_remaining(0), batch_base(0) {
 	// There are three types of partitions:
 	// 1. No partition (no sorting)
 	// 2. One partition (sorting, but no hashing)
 	// 3. Multiple partitions (sorting and hashing)
 
 	//	How big is the partition?
-	auto &gpart = *gsource.gsink.global_partition;
-	idx_t count = 0;
+	auto &gpart = *gstate.global_partition;
+	layout.Initialize(gpart.payload_types);
 	if (hash_bin < gpart.hash_groups.size() && gpart.hash_groups[hash_bin]) {
 		count = gpart.hash_groups[hash_bin]->count;
 	} else if (gpart.rows && !hash_bin) {
@@ -373,9 +532,9 @@ void WindowPartitionSourceState::BuildPartition(WindowGlobalSinkState &gstate, c
 	partition_mask.Initialize(count);
 	partition_mask.SetAllInvalid(count);
 
-	for (idx_t expr_idx = 0; expr_idx < op.select_list.size(); ++expr_idx) {
-		D_ASSERT(op.select_list[expr_idx]->GetExpressionClass() == ExpressionClass::BOUND_WINDOW);
-		auto &wexpr = op.select_list[expr_idx]->Cast<BoundWindowExpression>();
+	const auto &executors = gstate.executors;
+	for (auto &wexec : executors) {
+		auto &wexpr = wexec->wexpr;
 		auto &order_mask = order_masks[wexpr.partitions.size() + wexpr.orders.size()];
 		if (order_mask.IsMaskSet()) {
 			continue;
@@ -404,248 +563,263 @@ void WindowPartitionSourceState::BuildPartition(WindowGlobalSinkState &gstate, c
 		hash_group->ComputeMasks(partition_mask, order_masks);
 		external = hash_group->global_sort->external;
 		MaterializeSortedData();
-	} else {
-		return;
-	}
-
-	// Create the executors for each function
-	executors.clear();
-	for (idx_t expr_idx = 0; expr_idx < op.select_list.size(); ++expr_idx) {
-		D_ASSERT(op.select_list[expr_idx]->GetExpressionClass() == ExpressionClass::BOUND_WINDOW);
-		auto &wexpr = op.select_list[expr_idx]->Cast<BoundWindowExpression>();
-		auto &order_mask = order_masks[wexpr.partitions.size() + wexpr.orders.size()];
-		auto wexec = WindowExecutorFactory(wexpr, context, partition_mask, order_mask, count, gstate.mode);
-		executors.emplace_back(std::move(wexec));
-	}
-
-	//	First pass over the input without flushing
-	DataChunk input_chunk;
-	input_chunk.Initialize(gpart.allocator, gpart.payload_types);
-	auto scanner = make_uniq<RowDataCollectionScanner>(*rows, *heap, layout, external, false);
-	idx_t input_idx = 0;
-	while (true) {
-		input_chunk.Reset();
-		scanner->Scan(input_chunk);
-		if (input_chunk.size() == 0) {
-			break;
-		}
-
-		//	TODO: Parallelization opportunity
-		for (auto &wexec : executors) {
-			wexec->Sink(input_chunk, input_idx, scanner->Count());
-		}
-		input_idx += input_chunk.size();
 	}
 
-	//	TODO: Parallelization opportunity
-	for (auto &wexec : executors) {
-		wexec->Finalize();
+	if (rows) {
+		blocks = rows->blocks.size();
 	}
-
-	// External scanning assumes all blocks are swizzled.
-	scanner->ReSwizzle();
-
-	//	Start the block countdown
-	unscanned = rows->blocks.size();
 }
 
 // Per-thread scan state
 class WindowLocalSourceState : public LocalSourceState {
 public:
-	using ReadStatePtr = unique_ptr<WindowExecutorState>;
-	using ReadStates = vector<ReadStatePtr>;
+	using Task = WindowGlobalSourceState::Task;
+	using TaskPtr = optional_ptr<Task>;
 
 	explicit WindowLocalSourceState(WindowGlobalSourceState &gsource);
-	void UpdateBatchIndex();
-	bool NextPartition();
-	void Scan(DataChunk &chunk);
+
+	//! Does the task have more work to do?
+	bool TaskFinished() const {
+		return !task || task->begin_idx == task->end_idx;
+	}
+	//! Assign the next task
+	bool TryAssignTask();
+	//! Execute a step in the current task
+	void ExecuteTask(DataChunk &chunk);
 
 	//! The shared source state
 	WindowGlobalSourceState &gsource;
-	//! The current bin being processed
-	idx_t hash_bin;
 	//! The current batch index (for output reordering)
 	idx_t batch_index;
+	//! The task this thread is working on
+	TaskPtr task;
 	//! The current source being processed
-	optional_ptr<WindowPartitionSourceState> partition_source;
-	//! The read cursor
+	optional_ptr<WindowHashGroup> window_hash_group;
+	//! The scan cursor
 	unique_ptr<RowDataCollectionScanner> scanner;
 	//! Buffer for the inputs
 	DataChunk input_chunk;
-	//! Executor read states.
-	ReadStates read_states;
 	//! Buffer for window results
 	DataChunk output_chunk;
-};
 
-WindowLocalSourceState::WindowLocalSourceState(WindowGlobalSourceState &gsource)
-    : gsource(gsource), hash_bin(gsource.built.size()), batch_index(0) {
-	auto &gsink = *gsource.gsink.global_partition;
-	auto &op = gsource.gsink.op;
+protected:
+	void Sink();
+	void Finalize();
+	void GetData(DataChunk &chunk);
+};
 
-	input_chunk.Initialize(gsink.allocator, gsink.payload_types);
+WindowHashGroup::ExecutorGlobalStates &WindowHashGroup::Initialize(WindowGlobalSinkState &gsink) {
+	//	Single-threaded building as this is mostly memory allocation
+	lock_guard<mutex> gestate_guard(lock);
+	const auto &executors = gsink.executors;
+	if (gestates.size() == executors.size()) {
+		return gestates;
+	}
 
-	vector<LogicalType> output_types;
-	for (idx_t expr_idx = 0; expr_idx < op.select_list.size(); ++expr_idx) {
-		D_ASSERT(op.select_list[expr_idx]->GetExpressionClass() == ExpressionClass::BOUND_WINDOW);
-		auto &wexpr = op.select_list[expr_idx]->Cast<BoundWindowExpression>();
-		output_types.emplace_back(wexpr.return_type);
+	// These can be large so we defer building them until we are ready.
+	for (auto &wexec : executors) {
+		auto &wexpr = wexec->wexpr;
+		auto &order_mask = order_masks[wexpr.partitions.size() + wexpr.orders.size()];
+		gestates.emplace_back(wexec->GetGlobalState(count, partition_mask, order_mask));
 	}
-	output_chunk.Initialize(Allocator::Get(gsource.context), output_types);
+
+	return gestates;
 }
 
-WindowGlobalSourceState::Task WindowGlobalSourceState::CreateTask(idx_t hash_bin) {
-	//	Build outside the lock so no one tries to steal before we are done.
-	auto partition_source = make_uniq<WindowPartitionSourceState>(context, *this);
-	partition_source->BuildPartition(gsink, hash_bin);
-	Task result(partition_source.get(), partition_source->GetScanner());
+void WindowLocalSourceState::Sink() {
+	D_ASSERT(task);
+	D_ASSERT(task->stage == WindowGroupStage::SINK);
 
-	//	Is there any data to scan?
-	if (result.second) {
-		lock_guard<mutex> built_guard(built_lock);
-		built[hash_bin] = std::move(partition_source);
+	auto &gsink = gsource.gsink;
+	const auto &executors = gsink.executors;
 
-		return result;
-	}
+	// Create the global state for each function
+	// These can be large so we defer building them until we are ready.
+	auto &gestates = window_hash_group->Initialize(gsink);
 
-	return Task();
-}
+	//	Set up the local states
+	auto &local_states = window_hash_group->thread_states.at(task->thread_idx);
+	if (local_states.empty()) {
+		for (idx_t w = 0; w < executors.size(); ++w) {
+			local_states.emplace_back(executors[w]->GetLocalState(*gestates[w]));
+		}
+	}
 
-WindowGlobalSourceState::Task WindowGlobalSourceState::StealWork() {
-	for (idx_t hash_bin = 0; hash_bin < built.size(); ++hash_bin) {
-		lock_guard<mutex> built_guard(built_lock);
-		auto &partition_source = built[hash_bin];
-		if (!partition_source) {
-			continue;
+	//	First pass over the input without flushing
+	for (; task->begin_idx < task->end_idx; ++task->begin_idx) {
+		scanner = window_hash_group->GetBuildScanner(task->begin_idx);
+		if (!scanner) {
+			break;
 		}
+		while (true) {
+			//	TODO: Try to align on validity mask boundaries by starting ragged?
+			idx_t input_idx = scanner->Scanned();
+			input_chunk.Reset();
+			scanner->Scan(input_chunk);
+			if (input_chunk.size() == 0) {
+				break;
+			}
 
-		Task result(partition_source.get(), partition_source->GetScanner());
+			for (idx_t w = 0; w < executors.size(); ++w) {
+				executors[w]->Sink(input_chunk, input_idx, window_hash_group->count, *gestates[w], *local_states[w]);
+			}
 
-		//	Is there any data to scan?
-		if (result.second) {
-			return result;
+			window_hash_group->sunk += input_chunk.size();
 		}
+
+		// External scanning assumes all blocks are swizzled.
+		scanner->SwizzleBlock(task->begin_idx);
+		scanner.reset();
+	}
+}
+
+void WindowLocalSourceState::Finalize() {
+	D_ASSERT(task);
+	D_ASSERT(task->stage == WindowGroupStage::FINALIZE);
+
+	// Finalize all the executors.
+	// Parallel finalisation is handled internally by the executor,
+	// and should not return until all threads have completed work.
+	auto &gsink = gsource.gsink;
+	const auto &executors = gsink.executors;
+	auto &gestates = window_hash_group->gestates;
+	auto &local_states = window_hash_group->thread_states.at(task->thread_idx);
+	for (idx_t w = 0; w < executors.size(); ++w) {
+		executors[w]->Finalize(*gestates[w], *local_states[w]);
 	}
 
-	//	Nothing to steal
-	return Task();
+	//	Mark this range as done
+	window_hash_group->finalized += (task->end_idx - task->begin_idx);
+	task->begin_idx = task->end_idx;
 }
 
-WindowGlobalSourceState::Task WindowGlobalSourceState::NextTask(idx_t hash_bin) {
-	auto &hash_groups = gsink.global_partition->hash_groups;
-	const auto bin_count = built.size();
-
-	//	Flush unneeded data
-	if (hash_bin < bin_count) {
-		//	Lock and delete when all blocks have been scanned
-		//	We do this here instead of in NextScan so the WindowLocalSourceState
-		//	has a chance to delete its state objects first,
-		//	which may reference the partition_source
-
-		//	Delete data outside the lock in case it is slow
-		HashGroupSourcePtr killed;
-		lock_guard<mutex> built_guard(built_lock);
-		auto &partition_source = built[hash_bin];
-		if (partition_source && !partition_source->unscanned) {
-			killed = std::move(partition_source);
-		}
+WindowLocalSourceState::WindowLocalSourceState(WindowGlobalSourceState &gsource) : gsource(gsource), batch_index(0) {
+	auto &gsink = gsource.gsink;
+	auto &global_partition = *gsink.global_partition;
+
+	input_chunk.Initialize(global_partition.allocator, global_partition.payload_types);
+
+	vector<LogicalType> output_types;
+	for (auto &wexec : gsink.executors) {
+		auto &wexpr = wexec->wexpr;
+		output_types.emplace_back(wexpr.return_type);
 	}
+	output_chunk.Initialize(global_partition.allocator, output_types);
 
-	hash_bin = next_build++;
-	if (hash_bin < bin_count) {
-		//	Find a non-empty hash group.
-		for (; hash_bin < hash_groups.size(); hash_bin = next_build++) {
-			if (hash_groups[hash_bin] && hash_groups[hash_bin]->count) {
-				auto result = CreateTask(hash_bin);
-				if (result.second) {
-					return result;
-				}
-			}
-		}
+	++gsource.locals;
+}
 
-		//	OVER() doesn't have a hash_group
-		if (hash_groups.empty()) {
-			auto result = CreateTask(hash_bin);
-			if (result.second) {
-				return result;
-			}
-		}
+bool WindowGlobalSourceState::TryNextTask(TaskPtr &task) {
+	auto guard = Lock();
+	if (next_task >= tasks.size() || stopped) {
+		task = nullptr;
+		return false;
 	}
 
-	//	Work stealing
-	while (!context.interrupted && tasks_remaining) {
-		auto result = StealWork();
-		if (result.second) {
-			return result;
-		}
+	//	If the next task matches the current state of its group, then we can use it
+	//	Otherwise block.
+	task = &tasks[next_task];
+
+	auto &gpart = *gsink.global_partition;
+	auto &window_hash_group = gpart.window_hash_groups[task->group_idx];
+	auto group_stage = window_hash_group->GetStage();
 
-		//	If there is nothing to steal but there are unfinished partitions,
-		//	yield until any pending builds are done.
-		TaskScheduler::YieldThread();
+	if (task->stage == group_stage) {
+		++next_task;
+		return true;
 	}
 
-	return Task();
+	task = nullptr;
+	return false;
 }
 
-void WindowLocalSourceState::UpdateBatchIndex() {
-	D_ASSERT(partition_source);
-	D_ASSERT(scanner.get());
+void WindowGlobalSourceState::FinishTask(TaskPtr task) {
+	if (!task) {
+		return;
+	}
+
+	auto &gpart = *gsink.global_partition;
+	auto &finished_hash_group = gpart.window_hash_groups[task->group_idx];
+	D_ASSERT(finished_hash_group);
 
-	batch_index = partition_source->hash_group ? partition_source->hash_group->batch_base : 0;
-	batch_index += scanner->BlockIndex();
+	if (!--finished_hash_group->tasks_remaining) {
+		finished_hash_group.reset();
+	}
 }
 
-bool WindowLocalSourceState::NextPartition() {
-	//	Release old states before the source
+bool WindowLocalSourceState::TryAssignTask() {
+	// Because downstream operators may be using our internal buffers,
+	// we can't "finish" a task until we are about to get the next one.
+
+	// Scanner first, as it may be referencing sort blocks in the hash group
 	scanner.reset();
-	read_states.clear();
+	gsource.FinishTask(task);
 
-	//	Get a partition_source that is not finished
-	while (!scanner) {
-		auto task = gsource.NextTask(hash_bin);
-		if (!task.first) {
-			return false;
-		}
-		partition_source = task.first;
-		scanner = std::move(task.second);
-		hash_bin = partition_source->hash_bin;
-		UpdateBatchIndex();
-	}
+	return gsource.TryNextTask(task);
+}
 
-	for (auto &wexec : partition_source->executors) {
-		read_states.emplace_back(wexec->GetExecutorState());
+bool WindowGlobalSourceState::TryPrepareNextStage() {
+	if (next_task >= tasks.size() || stopped) {
+		return true;
 	}
 
-	return true;
+	auto task = &tasks[next_task];
+	auto window_hash_group = gsink.global_partition->window_hash_groups[task->group_idx].get();
+	return window_hash_group->TryPrepareNextStage();
 }
 
-void WindowLocalSourceState::Scan(DataChunk &result) {
-	D_ASSERT(scanner);
-	if (!scanner->Remaining()) {
-		lock_guard<mutex> built_guard(gsource.built_lock);
-		--partition_source->unscanned;
-		scanner = partition_source->GetScanner();
+void WindowLocalSourceState::ExecuteTask(DataChunk &result) {
+	auto &gsink = gsource.gsink;
 
-		if (!scanner) {
-			partition_source = nullptr;
-			read_states.clear();
-			return;
-		}
+	// Update the hash group
+	window_hash_group = gsink.global_partition->window_hash_groups[task->group_idx].get();
+
+	// Process the new state
+	switch (task->stage) {
+	case WindowGroupStage::SINK:
+		Sink();
+		D_ASSERT(TaskFinished());
+		break;
+	case WindowGroupStage::FINALIZE:
+		Finalize();
+		D_ASSERT(TaskFinished());
+		break;
+	case WindowGroupStage::GETDATA:
+		D_ASSERT(!TaskFinished());
+		GetData(result);
+		break;
+	default:
+		throw InternalException("Invalid window source state.");
+	}
 
-		UpdateBatchIndex();
+	// Count this task as finished.
+	if (TaskFinished()) {
+		++gsource.finished;
+	}
+}
+
+void WindowLocalSourceState::GetData(DataChunk &result) {
+	D_ASSERT(window_hash_group->GetStage() == WindowGroupStage::GETDATA);
+
+	if (!scanner || !scanner->Remaining()) {
+		scanner = window_hash_group->GetEvaluateScanner(task->begin_idx);
+		batch_index = window_hash_group->batch_base + task->begin_idx;
 	}
 
 	const auto position = scanner->Scanned();
 	input_chunk.Reset();
 	scanner->Scan(input_chunk);
 
-	auto &executors = partition_source->executors;
+	const auto &executors = gsource.gsink.executors;
+	auto &gestates = window_hash_group->gestates;
+	auto &local_states = window_hash_group->thread_states.at(task->thread_idx);
 	output_chunk.Reset();
 	for (idx_t expr_idx = 0; expr_idx < executors.size(); ++expr_idx) {
 		auto &executor = *executors[expr_idx];
-		auto &lstate = *read_states[expr_idx];
+		auto &gstate = *gestates[expr_idx];
+		auto &lstate = *local_states[expr_idx];
 		auto &result = output_chunk.data[expr_idx];
-		executor.Evaluate(position, input_chunk, result, lstate);
+		executor.Evaluate(position, input_chunk, result, lstate, gstate);
 	}
 	output_chunk.SetCardinality(input_chunk);
 	output_chunk.Verify();
@@ -658,6 +832,16 @@ void WindowLocalSourceState::Scan(DataChunk &result) {
 	for (idx_t col_idx = 0; col_idx < output_chunk.ColumnCount(); col_idx++) {
 		result.data[out_idx++].Reference(output_chunk.data[col_idx]);
 	}
+
+	// If we done with this block, move to the next one
+	if (!scanner->Remaining()) {
+		++task->begin_idx;
+	}
+
+	// If that was the last block, release out local state memory.
+	if (TaskFinished()) {
+		local_states.clear();
+	}
 	result.Verify();
 }
 
@@ -676,11 +860,22 @@ bool PhysicalWindow::SupportsBatchIndex() const {
 	//	We can only preserve order for single partitioning
 	//	or work stealing causes out of order batch numbers
 	auto &wexpr = select_list[order_idx]->Cast<BoundWindowExpression>();
-	return wexpr.partitions.empty() && !wexpr.orders.empty();
+	return wexpr.partitions.empty(); // NOLINT
 }
 
 OrderPreservationType PhysicalWindow::SourceOrder() const {
-	return SupportsBatchIndex() ? OrderPreservationType::FIXED_ORDER : OrderPreservationType::NO_ORDER;
+	auto &wexpr = select_list[order_idx]->Cast<BoundWindowExpression>();
+	if (!wexpr.partitions.empty()) {
+		// if we have partitions the window order is not defined
+		return OrderPreservationType::NO_ORDER;
+	}
+	// without partitions we can maintain order
+	if (wexpr.orders.empty()) {
+		// if we have no orders we maintain insertion order
+		return OrderPreservationType::INSERTION_ORDER;
+	}
+	// otherwise we can maintain the fixed order
+	return OrderPreservationType::FIXED_ORDER;
 }
 
 double PhysicalWindow::GetProgress(ClientContext &context, GlobalSourceState &gsource_p) const {
@@ -689,7 +884,7 @@ double PhysicalWindow::GetProgress(ClientContext &context, GlobalSourceState &gs
 
 	auto &gsink = gsource.gsink;
 	const auto count = gsink.global_partition->count.load();
-	return count ? (returned / double(count)) : -1;
+	return count ? (double(returned) / double(count)) : -1;
 }
 
 idx_t PhysicalWindow::GetBatchIndex(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate_p,
@@ -702,29 +897,53 @@ SourceResultType PhysicalWindow::GetData(ExecutionContext &context, DataChunk &c
                                          OperatorSourceInput &input) const {
 	auto &gsource = input.global_state.Cast<WindowGlobalSourceState>();
 	auto &lsource = input.local_state.Cast<WindowLocalSourceState>();
-	while (chunk.size() == 0) {
-		//	Move to the next bin if we are done.
-		while (!lsource.scanner) {
-			if (!lsource.NextPartition()) {
-				return chunk.size() > 0 ? SourceResultType::HAVE_MORE_OUTPUT : SourceResultType::FINISHED;
+
+	gsource.CreateTaskList();
+
+	while (gsource.HasUnfinishedTasks() && chunk.size() == 0) {
+		if (!lsource.TaskFinished() || lsource.TryAssignTask()) {
+			try {
+				lsource.ExecuteTask(chunk);
+			} catch (...) {
+				gsource.stopped = true;
+				throw;
+			}
+		} else {
+			auto guard = gsource.Lock();
+			if (!gsource.HasMoreTasks()) {
+				// no more tasks - exit
+				gsource.UnblockTasks(guard);
+				break;
+			}
+			if (gsource.TryPrepareNextStage()) {
+				// we successfully prepared the next stage - unblock tasks
+				gsource.UnblockTasks(guard);
+			} else {
+				// there are more tasks available, but we can't execute them yet
+				// block the source
+				return gsource.BlockSource(guard, input.interrupt_state);
 			}
 		}
-
-		lsource.Scan(chunk);
-		gsource.returned += chunk.size();
 	}
 
-	return chunk.size() == 0 ? SourceResultType::FINISHED : SourceResultType::HAVE_MORE_OUTPUT;
+	gsource.returned += chunk.size();
+
+	if (chunk.size() == 0) {
+		return SourceResultType::FINISHED;
+	}
+	return SourceResultType::HAVE_MORE_OUTPUT;
 }
 
-string PhysicalWindow::ParamsToString() const {
-	string result;
+InsertionOrderPreservingMap<string> PhysicalWindow::ParamsToString() const {
+	InsertionOrderPreservingMap<string> result;
+	string projections;
 	for (idx_t i = 0; i < select_list.size(); i++) {
 		if (i > 0) {
-			result += "\n";
+			projections += "\n";
 		}
-		result += select_list[i]->GetName();
+		projections += select_list[i]->GetName();
 	}
+	result["Projections"] = projections;
 	return result;
 }