duckdb 1.0.1-dev21.0 → 1.0.1-dev27.0

package/src/duckdb/src/optimizer/rule/join_dependent_filter.cpp

@@ -0,0 +1,135 @@
+#include "duckdb/optimizer/rule/join_dependent_filter.hpp"
+
+#include "duckdb/optimizer/expression_rewriter.hpp"
+#include "duckdb/planner/expression/bound_columnref_expression.hpp"
+#include "duckdb/planner/expression/bound_comparison_expression.hpp"
+#include "duckdb/planner/expression/bound_conjunction_expression.hpp"
+#include "duckdb/planner/expression_iterator.hpp"
+
+namespace duckdb {
+
+JoinDependentFilterRule::JoinDependentFilterRule(ExpressionRewriter &rewriter) : Rule(rewriter) {
+	// Match on a ConjunctionExpression that has at least two ConjunctionExpressions as children
+	auto op = make_uniq<ConjunctionExpressionMatcher>();
+	op->matchers.push_back(make_uniq<ConjunctionExpressionMatcher>());
+	op->policy = SetMatcher::Policy::SOME;
+	root = std::move(op);
+}
+
+static inline void GetTableIndices(const Expression &expression, unordered_set<idx_t> &table_idxs) {
+	ExpressionIterator::EnumerateChildren(expression, [&](const Expression &child) {
+		if (child.GetExpressionClass() == ExpressionClass::BOUND_COLUMN_REF) {
+			auto &colref = child.Cast<BoundColumnRefExpression>();
+			table_idxs.insert(colref.binding.table_index);
+		} else {
+			GetTableIndices(child, table_idxs);
+		}
+	});
+}
+
+static inline bool ExpressionReferencesMultipleTables(const Expression &binding) {
+	unordered_set<idx_t> table_idxs;
+	GetTableIndices(binding, table_idxs);
+	return table_idxs.size() > 1;
+}
+
+static inline void ExtractConjunctedExpressions(Expression &expression,
+                                                unordered_map<idx_t, unique_ptr<Expression>> &expressions) {
+	if (expression.GetExpressionType() == ExpressionType::CONJUNCTION_AND) {
+		auto &conjunction = expression.Cast<BoundConjunctionExpression>();
+		for (auto &child : conjunction.children) {
+			ExtractConjunctedExpressions(*child, expressions);
+		}
+	} else if (!expression.IsVolatile()) {
+		unordered_set<idx_t> table_idxs;
+		GetTableIndices(expression, table_idxs);
+		if (table_idxs.size() != 1) {
+			return; // Needs to reference exactly one table
+		}
+
+		// If there was already an expression, AND it together
+		auto &table_expressions = expressions[*table_idxs.begin()];
+		table_expressions = table_expressions
+		                        ? make_uniq<BoundConjunctionExpression>(ExpressionType::CONJUNCTION_AND,
+		                                                                std::move(table_expressions), expression.Copy())
+		                        : expression.Copy();
+	}
+}
+
+unique_ptr<Expression> JoinDependentFilterRule::Apply(LogicalOperator &op, vector<reference<Expression>> &bindings,
+                                                      bool &changes_made, bool is_root) {
+	// Only applies to top-level FILTER expressions
+	if ((op.type != LogicalOperatorType::LOGICAL_FILTER && op.type != LogicalOperatorType::LOGICAL_ANY_JOIN) ||
+	    !is_root) {
+		return nullptr;
+	}
+
+	// The expression must be an OR
+	auto &conjunction = bindings[0].get().Cast<BoundConjunctionExpression>();
+	if (conjunction.GetExpressionType() != ExpressionType::CONJUNCTION_OR) {
+		return nullptr;
+	}
+
+	// Must have at least one join-dependent AND expression
+	auto &children = conjunction.children;
+	bool eligible = false;
+	for (const auto &child : children) {
+		if (child->GetExpressionClass() == ExpressionClass::BOUND_CONJUNCTION &&
+		    child->GetExpressionType() == ExpressionType::CONJUNCTION_AND &&
+		    ExpressionReferencesMultipleTables(*child)) {
+			eligible = true;
+			break;
+		}
+	}
+	if (!eligible) {
+		return nullptr;
+	}
+
+	// Extract all comparison expressions between column references and constants that are AND'ed together
+	auto conjuncted_expressions = make_uniq_array<unordered_map<idx_t, unique_ptr<Expression>>>(children.size());
+	for (idx_t child_idx = 0; child_idx < children.size(); child_idx++) {
+		conjuncted_expressions[child_idx] = unordered_map<idx_t, unique_ptr<Expression>>();
+		ExtractConjunctedExpressions(*children[child_idx], conjuncted_expressions[child_idx]);
+	}
+
+	auto derived_filter = make_uniq<BoundConjunctionExpression>(ExpressionType::CONJUNCTION_AND);
+	for (auto &entry : conjuncted_expressions[0]) {
+		auto derived_entry_filter = make_uniq<BoundConjunctionExpression>(ExpressionType::CONJUNCTION_OR);
+		derived_entry_filter->children.push_back(entry.second->Copy());
+
+		bool found = true;
+		for (idx_t conj_idx = 1; conj_idx < children.size(); conj_idx++) {
+			auto &other_entry = conjuncted_expressions[conj_idx];
+			auto other_it = other_entry.find(entry.first);
+			if (other_it == other_entry.end()) {
+				found = false;
+				break; // Expression does not appear in every conjuncted expression, cannot derive any restriction
+			}
+			derived_entry_filter->children.push_back(other_it->second->Copy());
+		}
+
+		if (!found) {
+			continue; // Expression must show up in every entry
+		}
+
+		derived_filter->children.push_back(std::move(derived_entry_filter));
+	}
+
+	if (derived_filter->children.empty()) {
+		return nullptr; // Could not derive filters that can be pushed down
+	}
+
+	// Add the derived expression to the original expression with an AND
+	auto result = make_uniq_base<Expression, BoundConjunctionExpression>(ExpressionType::CONJUNCTION_AND);
+	auto &result_conjunction = result->Cast<BoundConjunctionExpression>();
+	result_conjunction.children.push_back(conjunction.Copy());
+
+	if (derived_filter->children.size() == 1) {
+		result_conjunction.children.push_back(std::move(derived_filter->children[0]));
+	} else {
+		result_conjunction.children.push_back(std::move(derived_filter));
+	}
+	return result;
+}
+
+} // namespace duckdb

package/src/duckdb/src/optimizer/rule/like_optimizations.cpp

@@ -132,7 +132,7 @@ unique_ptr<Expression> LikeOptimizationRule::Apply(LogicalOperator &op, vector<r
 		return ApplyRule(root, SuffixFun::GetFunction(), patt_str, is_not_like);
 	} else if (PatternIsContains(patt_str)) {
 		// Contains LIKE pattern: [%]+[^%_]*[%]+, ignoring underscore
-		return ApplyRule(root, ContainsFun::GetFunction(), patt_str, is_not_like);
+		return ApplyRule(root, ContainsFun::GetStringContains(), patt_str, is_not_like);
 	}
 	return nullptr;
 }

package/src/duckdb/src/optimizer/rule/regex_optimizations.cpp

@@ -184,7 +184,7 @@ unique_ptr<Expression> RegexOptimizationRule::Apply(LogicalOperator &op, vector<
 			return nullptr;
 		}
 		auto parameter = make_uniq<BoundConstantExpression>(Value(std::move(escaped_like_string.like_string)));
-		auto contains = make_uniq<BoundFunctionExpression>(root.return_type, ContainsFun::GetFunction(),
+		auto contains = make_uniq<BoundFunctionExpression>(root.return_type, ContainsFun::GetStringContains(),
 		                                                   std::move(root.children), nullptr);
 		contains->children[1] = std::move(parameter);
 

package/src/duckdb/src/optimizer/statistics/operator/propagate_filter.cpp

@@ -241,7 +241,12 @@ unique_ptr<NodeStatistics> StatisticsPropagator::PropagateStatistics(LogicalFilt
 			filter.expressions.erase_at(i);
 			i--;
 			if (filter.expressions.empty()) {
-				// just break. The physical filter planner will plan a projection instead
+				// if there is a projection map, we should keep the filter
+				// the physical planner will eventually skip the filter, but will keep
+				// the correct columns.
+				if (filter.projection_map.empty()) {
+					node_ptr = std::move(filter.children[0]);
+				}
 				break;
 			}
 		} else if (ExpressionIsConstant(*condition, Value::BOOLEAN(false)) ||

package/src/duckdb/src/optimizer/statistics/operator/propagate_get.cpp

@@ -39,8 +39,9 @@ unique_ptr<NodeStatistics> StatisticsPropagator::PropagateStatistics(LogicalGet
 		// no column statistics to get
 		return std::move(node_stats);
 	}
-	for (idx_t i = 0; i < get.column_ids.size(); i++) {
-		auto stats = get.function.statistics(context, get.bind_data.get(), get.column_ids[i]);
+	auto &column_ids = get.GetColumnIds();
+	for (idx_t i = 0; i < column_ids.size(); i++) {
+		auto stats = get.function.statistics(context, get.bind_data.get(), column_ids[i]);
 		if (stats) {
 			ColumnBinding binding(get.table_index, i);
 			statistics_map.insert(make_pair(binding, std::move(stats)));
@@ -55,13 +56,13 @@ unique_ptr<NodeStatistics> StatisticsPropagator::PropagateStatistics(LogicalGet
 
 	for (auto &table_filter_column : column_indexes) {
 		idx_t column_index;
-		for (column_index = 0; column_index < get.column_ids.size(); column_index++) {
-			if (get.column_ids[column_index] == table_filter_column) {
+		for (column_index = 0; column_index < column_ids.size(); column_index++) {
+			if (column_ids[column_index] == table_filter_column) {
 				break;
 			}
 		}
-		D_ASSERT(column_index < get.column_ids.size());
-		D_ASSERT(get.column_ids[column_index] == table_filter_column);
+		D_ASSERT(column_index < column_ids.size());
+		D_ASSERT(column_ids[column_index] == table_filter_column);
 
 		// find the stats
 		ColumnBinding stats_binding(get.table_index, column_index);

package/src/duckdb/src/optimizer/statistics/operator/propagate_join.cpp

@@ -358,7 +358,7 @@ void StatisticsPropagator::CreateFilterFromJoinStats(unique_ptr<LogicalOperator>
 		child->expressions.emplace_back(std::move(filter_expr));
 	}
 
-	// not allowed to let filter pushdowwn change mark joins to semi joins.
+	// not allowed to let filter pushdown change mark joins to semi joins.
 	// semi joins are potentially slower AND the conversion can ruin column binding information
 	FilterPushdown filter_pushdown(optimizer, false);
 	child = filter_pushdown.Rewrite(std::move(child));

package/src/duckdb/src/optimizer/topn_optimizer.cpp

@@ -8,8 +8,7 @@
 namespace duckdb {
 
 bool TopN::CanOptimize(LogicalOperator &op) {
-	if (op.type == LogicalOperatorType::LOGICAL_LIMIT &&
-	    op.children[0]->type == LogicalOperatorType::LOGICAL_ORDER_BY) {
+	if (op.type == LogicalOperatorType::LOGICAL_LIMIT) {
 		auto &limit = op.Cast<LogicalLimit>();
 
 		if (limit.limit_val.Type() != LimitNodeType::CONSTANT_VALUE) {
@@ -20,15 +19,42 @@ bool TopN::CanOptimize(LogicalOperator &op) {
 			// we need offset to be either not set (i.e. limit without offset) OR have offset be
 			return false;
 		}
-		return true;
+
+		auto child_op = op.children[0].get();
+
+		while (child_op->type == LogicalOperatorType::LOGICAL_PROJECTION) {
+			D_ASSERT(!child_op->children.empty());
+			child_op = child_op->children[0].get();
+		}
+
+		return child_op->type == LogicalOperatorType::LOGICAL_ORDER_BY;
 	}
 	return false;
 }
 
 unique_ptr<LogicalOperator> TopN::Optimize(unique_ptr<LogicalOperator> op) {
 	if (CanOptimize(*op)) {
+
+		vector<unique_ptr<LogicalOperator>> projections;
+
+		// traverse operator tree and collect all projection nodes until we reach
+		// the order by operator
+
+		auto child = std::move(op->children[0]);
+		// collect all projections until we get to the order by
+		while (child->type == LogicalOperatorType::LOGICAL_PROJECTION) {
+			D_ASSERT(!child->children.empty());
+			auto tmp = std::move(child->children[0]);
+			projections.push_back(std::move(child));
+			child = std::move(tmp);
+		}
+		D_ASSERT(child->type == LogicalOperatorType::LOGICAL_ORDER_BY);
+		auto &order_by = child->Cast<LogicalOrder>();
+
+		// Move order by operator into children of limit operator
+		op->children[0] = std::move(child);
+
 		auto &limit = op->Cast<LogicalLimit>();
-		auto &order_by = (op->children[0])->Cast<LogicalOrder>();
 		auto limit_val = limit.limit_val.GetConstantValue();
 		idx_t offset_val = 0;
 		if (limit.offset_val.Type() == LimitNodeType::CONSTANT_VALUE) {
@@ -36,12 +62,25 @@ unique_ptr<LogicalOperator> TopN::Optimize(unique_ptr<LogicalOperator> op) {
 		}
 		auto topn = make_uniq<LogicalTopN>(std::move(order_by.orders), limit_val, offset_val);
 		topn->AddChild(std::move(order_by.children[0]));
+		auto cardinality = limit_val;
+		if (topn->children[0]->has_estimated_cardinality && topn->children[0]->estimated_cardinality < limit_val) {
+			cardinality = topn->children[0]->estimated_cardinality;
+		}
+		topn->SetEstimatedCardinality(cardinality);
 		op = std::move(topn);
-	} else {
-		for (auto &child : op->children) {
-			child = Optimize(std::move(child));
+
+		// reconstruct all projection nodes above limit operator
+		while (!projections.empty()) {
+			auto node = std::move(projections.back());
+			node->children[0] = std::move(op);
+			op = std::move(node);
+			projections.pop_back();
 		}
 	}
+
+	for (auto &child : op->children) {
+		child = Optimize(std::move(child));
+	}
 	return op;
 }
 

package/src/duckdb/src/parallel/executor.cpp

@@ -2,6 +2,7 @@
 
 #include "duckdb/execution/execution_context.hpp"
 #include "duckdb/execution/operator/helper/physical_result_collector.hpp"
+#include "duckdb/execution/operator/scan/physical_table_scan.hpp"
 #include "duckdb/execution/operator/set/physical_cte.hpp"
 #include "duckdb/execution/operator/set/physical_recursive_cte.hpp"
 #include "duckdb/execution/physical_operator.hpp"
@@ -13,14 +14,16 @@
 #include "duckdb/parallel/pipeline_executor.hpp"
 #include "duckdb/parallel/pipeline_finish_event.hpp"
 #include "duckdb/parallel/pipeline_initialize_event.hpp"
+#include "duckdb/parallel/pipeline_prepare_finish_event.hpp"
 #include "duckdb/parallel/task_scheduler.hpp"
 #include "duckdb/parallel/thread_context.hpp"
 
 #include <algorithm>
+#include <chrono>
 
 namespace duckdb {
 
-Executor::Executor(ClientContext &context) : context(context), executor_tasks(0) {
+Executor::Executor(ClientContext &context) : context(context), executor_tasks(0), blocked_thread_time(0) {
 }
 
 Executor::~Executor() {
@@ -40,14 +43,16 @@ void Executor::AddEvent(shared_ptr<Event> event) {
 }
 
 struct PipelineEventStack {
-	PipelineEventStack(Event &pipeline_initialize_event, Event &pipeline_event, Event &pipeline_finish_event,
-	                   Event &pipeline_complete_event)
+	PipelineEventStack(Event &pipeline_initialize_event, Event &pipeline_event, Event &pipeline_prepare_finish_event,
+	                   Event &pipeline_finish_event, Event &pipeline_complete_event)
 	    : pipeline_initialize_event(pipeline_initialize_event), pipeline_event(pipeline_event),
-	      pipeline_finish_event(pipeline_finish_event), pipeline_complete_event(pipeline_complete_event) {
+	      pipeline_prepare_finish_event(pipeline_prepare_finish_event), pipeline_finish_event(pipeline_finish_event),
+	      pipeline_complete_event(pipeline_complete_event) {
 	}
 
 	Event &pipeline_initialize_event;
 	Event &pipeline_event;
+	Event &pipeline_prepare_finish_event;
 	Event &pipeline_finish_event;
 	Event &pipeline_complete_event;
 };
@@ -75,18 +80,22 @@ void Executor::SchedulePipeline(const shared_ptr<MetaPipeline> &meta_pipeline, S
 	auto base_pipeline = meta_pipeline->GetBasePipeline();
 	auto base_initialize_event = make_shared_ptr<PipelineInitializeEvent>(base_pipeline);
 	auto base_event = make_shared_ptr<PipelineEvent>(base_pipeline);
+	auto base_prepare_finish_event = make_shared_ptr<PipelinePrepareFinishEvent>(base_pipeline);
 	auto base_finish_event = make_shared_ptr<PipelineFinishEvent>(base_pipeline);
 	auto base_complete_event =
 	    make_shared_ptr<PipelineCompleteEvent>(base_pipeline->executor, event_data.initial_schedule);
-	PipelineEventStack base_stack(*base_initialize_event, *base_event, *base_finish_event, *base_complete_event);
+	PipelineEventStack base_stack(*base_initialize_event, *base_event, *base_prepare_finish_event, *base_finish_event,
+	                              *base_complete_event);
 	events.push_back(std::move(base_initialize_event));
 	events.push_back(std::move(base_event));
+	events.push_back(std::move(base_prepare_finish_event));
 	events.push_back(std::move(base_finish_event));
 	events.push_back(std::move(base_complete_event));
 
-	// dependencies: initialize -> event -> finish -> complete
+	// dependencies: initialize -> event -> prepare finish -> finish -> complete
 	base_stack.pipeline_event.AddDependency(base_stack.pipeline_initialize_event);
-	base_stack.pipeline_finish_event.AddDependency(base_stack.pipeline_event);
+	base_stack.pipeline_prepare_finish_event.AddDependency(base_stack.pipeline_event);
+	base_stack.pipeline_finish_event.AddDependency(base_stack.pipeline_prepare_finish_event);
 	base_stack.pipeline_complete_event.AddDependency(base_stack.pipeline_finish_event);
 
 	// create an event and stack for all pipelines in the MetaPipeline
@@ -106,24 +115,30 @@ void Executor::SchedulePipeline(const shared_ptr<MetaPipeline> &meta_pipeline, S
 			D_ASSERT(group_entry != event_map.end());
 			auto &group_stack = group_entry->second;
 			PipelineEventStack pipeline_stack(base_stack.pipeline_initialize_event, *pipeline_event,
+			                                  group_stack.pipeline_prepare_finish_event,
 			                                  group_stack.pipeline_finish_event, base_stack.pipeline_complete_event);
 
-			// dependencies: base_finish -> pipeline_event -> group_finish
+			// dependencies: base_finish -> pipeline_event -> group_prepare_finish
 			pipeline_stack.pipeline_event.AddDependency(base_stack.pipeline_finish_event);
-			group_stack.pipeline_finish_event.AddDependency(pipeline_stack.pipeline_event);
+			group_stack.pipeline_prepare_finish_event.AddDependency(pipeline_stack.pipeline_event);
 
 			// add pipeline stack to event map
 			event_map.insert(make_pair(reference<Pipeline>(*pipeline), pipeline_stack));
 		} else if (meta_pipeline->HasFinishEvent(*pipeline)) {
 			// this pipeline has its own finish event (despite going into the same sink - Finalize twice!)
+			auto pipeline_prepare_finish_event = make_shared_ptr<PipelinePrepareFinishEvent>(pipeline);
 			auto pipeline_finish_event = make_shared_ptr<PipelineFinishEvent>(pipeline);
 			PipelineEventStack pipeline_stack(base_stack.pipeline_initialize_event, *pipeline_event,
-			                                  *pipeline_finish_event, base_stack.pipeline_complete_event);
+			                                  *pipeline_prepare_finish_event, *pipeline_finish_event,
+			                                  base_stack.pipeline_complete_event);
+			events.push_back(std::move(pipeline_prepare_finish_event));
 			events.push_back(std::move(pipeline_finish_event));
 
-			// dependencies: base_finish -> pipeline_event -> pipeline_finish -> base_complete
+			// dependencies:
+			// base_finish -> pipeline_event -> pipeline_prepare_finish -> pipeline_finish -> base_complete
 			pipeline_stack.pipeline_event.AddDependency(base_stack.pipeline_finish_event);
-			pipeline_stack.pipeline_finish_event.AddDependency(pipeline_stack.pipeline_event);
+			pipeline_stack.pipeline_prepare_finish_event.AddDependency(pipeline_stack.pipeline_event);
+			pipeline_stack.pipeline_finish_event.AddDependency(pipeline_stack.pipeline_prepare_finish_event);
 			base_stack.pipeline_complete_event.AddDependency(pipeline_stack.pipeline_finish_event);
 
 			// add pipeline stack to event map
@@ -131,11 +146,12 @@ void Executor::SchedulePipeline(const shared_ptr<MetaPipeline> &meta_pipeline, S
 		} else {
 			// no additional finish event
 			PipelineEventStack pipeline_stack(base_stack.pipeline_initialize_event, *pipeline_event,
+			                                  base_stack.pipeline_prepare_finish_event,
 			                                  base_stack.pipeline_finish_event, base_stack.pipeline_complete_event);
 
-			// dependencies: base_initialize -> pipeline_event -> base_finish
+			// dependencies: base_initialize -> pipeline_event -> base_prepare_finish
 			pipeline_stack.pipeline_event.AddDependency(base_stack.pipeline_initialize_event);
-			base_stack.pipeline_finish_event.AddDependency(pipeline_stack.pipeline_event);
+			base_stack.pipeline_prepare_finish_event.AddDependency(pipeline_stack.pipeline_event);
 
 			// add pipeline stack to event map
 			event_map.insert(make_pair(reference<Pipeline>(*pipeline), pipeline_stack));
@@ -146,27 +162,15 @@ void Executor::SchedulePipeline(const shared_ptr<MetaPipeline> &meta_pipeline, S
 	// add base stack to the event data too
 	event_map.insert(make_pair(reference<Pipeline>(*base_pipeline), base_stack));
 
-	// set up the dependencies within this MetaPipeline
 	for (auto &pipeline : pipelines) {
 		auto source = pipeline->GetSource();
 		if (source->type == PhysicalOperatorType::TABLE_SCAN) {
-			// we have to reset the source here (in the main thread), because some of our clients (looking at you, R)
-			// do not like it when threads other than the main thread call into R, for e.g., arrow scans
-			pipeline->ResetSource(true);
-		}
-
-		auto dependencies = meta_pipeline->GetDependencies(*pipeline);
-		if (!dependencies) {
-			continue;
-		}
-		auto root_entry = event_map.find(*pipeline);
-		D_ASSERT(root_entry != event_map.end());
-		auto &pipeline_stack = root_entry->second;
-		for (auto &dependency : *dependencies) {
-			auto event_entry = event_map.find(dependency);
-			D_ASSERT(event_entry != event_map.end());
-			auto &dependency_stack = event_entry->second;
-			pipeline_stack.pipeline_event.AddDependency(dependency_stack.pipeline_event);
+			auto &table_function = source->Cast<PhysicalTableScan>();
+			if (table_function.function.global_initialization == TableFunctionInitialization::INITIALIZE_ON_SCHEDULE) {
+				// certain functions have to be eagerly initialized during scheduling
+				// if that is the case - initialize the function here
+				pipeline->ResetSource(true);
+			}
 		}
 	}
 }
@@ -180,7 +184,7 @@ void Executor::ScheduleEventsInternal(ScheduleEventData &event_data) {
 		SchedulePipeline(meta_pipeline, event_data);
 	}
 
-	// set up the dependencies across MetaPipelines
+	// set up the dependencies for complete event
 	auto &event_map = event_data.event_map;
 	for (auto &entry : event_map) {
 		auto &pipeline = entry.first.get();
@@ -197,23 +201,56 @@ void Executor::ScheduleEventsInternal(ScheduleEventData &event_data) {
 		}
 	}
 
-	// make pipeline_finish_event of each MetaPipeline depend on the pipeline_event of the base pipeline of its sublings
-	// this allows TemporaryMemoryManager to more fairly distribute memory
+	// set the dependencies for pipeline event
+	for (auto &meta_pipeline : event_data.meta_pipelines) {
+		for (auto &entry : meta_pipeline->GetDependencies()) {
+			auto &pipeline = entry.first.get();
+			auto root_entry = event_map.find(pipeline);
+			D_ASSERT(root_entry != event_map.end());
+			auto &pipeline_stack = root_entry->second;
+			for (auto &dependency : entry.second) {
+				auto event_entry = event_map.find(dependency);
+				D_ASSERT(event_entry != event_map.end());
+				auto &dependency_stack = event_entry->second;
+				pipeline_stack.pipeline_event.AddDependency(dependency_stack.pipeline_event);
+			}
+		}
+	}
+
+	// these dependencies make it so that things happen in this order:
+	// 1. all join build child pipelines run until Combine
+	// 2. all join build child pipeline PrepareFinalize
+	// 3. all join build child pipelines Finalize
+	// operators communicate their memory usage through the TemporaryMemoryManger (TMM) in PrepareFinalize
+	// then, when the child pipelines Finalize, all required memory is known, and TMM can make an informed decision
 	for (auto &meta_pipeline : event_data.meta_pipelines) {
 		vector<shared_ptr<MetaPipeline>> children;
 		meta_pipeline->GetMetaPipelines(children, false, true);
 		for (auto &child1 : children) {
+			if (child1->Type() != MetaPipelineType::JOIN_BUILD) {
+				continue; // We only want to do this for join builds
+			}
 			auto &child1_base = *child1->GetBasePipeline();
 			auto child1_entry = event_map.find(child1_base);
 			D_ASSERT(child1_entry != event_map.end());
+
 			for (auto &child2 : children) {
-				if (RefersToSameObject(*child1, *child2)) {
-					continue;
+				if (child2->Type() != MetaPipelineType::JOIN_BUILD || RefersToSameObject(*child1, *child2)) {
+					continue; // We don't want to depend on itself
+				}
+				if (!RefersToSameObject(*child1->GetParent(), *child2->GetParent())) {
+					continue; // Different parents, skip
 				}
+
 				auto &child2_base = *child2->GetBasePipeline();
 				auto child2_entry = event_map.find(child2_base);
 				D_ASSERT(child2_entry != event_map.end());
-				child1_entry->second.pipeline_finish_event.AddDependency(child2_entry->second.pipeline_event);
+
+				// all children PrepareFinalize must wait until all Combine
+				child1_entry->second.pipeline_prepare_finish_event.AddDependency(child2_entry->second.pipeline_event);
+				// all children Finalize must wait until all PrepareFinalize
+				child1_entry->second.pipeline_finish_event.AddDependency(
+				    child2_entry->second.pipeline_prepare_finish_event);
 			}
 		}
 	}
@@ -426,6 +463,27 @@ void Executor::WorkOnTasks() {
 	}
 }
 
+void Executor::SignalTaskRescheduled(lock_guard<mutex> &) {
+	task_reschedule.notify_one();
+}
+
+void Executor::WaitForTask() {
+#ifndef DUCKDB_NO_THREADS
+	static constexpr std::chrono::milliseconds WAIT_TIME_MS = std::chrono::milliseconds(WAIT_TIME);
+	std::unique_lock<mutex> l(executor_lock);
+	if (to_be_rescheduled_tasks.empty()) {
+		return;
+	}
+	if (ResultCollectorIsBlocked()) {
+		// If the result collector is blocked, it won't get unblocked until the connection calls Fetch
+		return;
+	}
+
+	blocked_thread_time++;
+	task_reschedule.wait_for(l, WAIT_TIME_MS);
+#endif
+}
+
 void Executor::RescheduleTask(shared_ptr<Task> &task_p) {
 	// This function will spin lock until the task provided is added to the to_be_rescheduled_tasks
 	while (true) {
@@ -438,17 +496,20 @@ void Executor::RescheduleTask(shared_ptr<Task> &task_p) {
 			auto &scheduler = TaskScheduler::GetScheduler(context);
 			to_be_rescheduled_tasks.erase(task_p.get());
 			scheduler.ScheduleTask(GetToken(), task_p);
+			SignalTaskRescheduled(l);
 			break;
 		}
 	}
 }
 
 bool Executor::ResultCollectorIsBlocked() {
+	if (!HasStreamingResultCollector()) {
+		return false;
+	}
 	if (completed_pipelines + 1 != total_pipelines) {
 		// The result collector is always in the last pipeline
 		return false;
 	}
-	lock_guard<mutex> l(executor_lock);
 	if (to_be_rescheduled_tasks.empty()) {
 		return false;
 	}
@@ -483,12 +544,12 @@ bool Executor::ExecutionIsFinished() {
 PendingExecutionResult Executor::ExecuteTask(bool dry_run) {
 	// Only executor should return NO_TASKS_AVAILABLE
 	D_ASSERT(execution_result != PendingExecutionResult::NO_TASKS_AVAILABLE);
-	if (execution_result != PendingExecutionResult::RESULT_NOT_READY) {
+	if (execution_result != PendingExecutionResult::RESULT_NOT_READY && ExecutionIsFinished()) {
 		return execution_result;
 	}
 	// check if there are any incomplete pipelines
 	auto &scheduler = TaskScheduler::GetScheduler(context);
-	while (completed_pipelines < total_pipelines) {
+	if (completed_pipelines < total_pipelines) {
 		// there are! if we don't already have a task, fetch one
 		auto current_task = task.get();
 		if (dry_run) {
@@ -503,13 +564,15 @@ PendingExecutionResult Executor::ExecuteTask(bool dry_run) {
 
 		if (!current_task && !HasError()) {
 			// there are no tasks to be scheduled and there are tasks blocked
+			lock_guard<mutex> l(executor_lock);
+			if (to_be_rescheduled_tasks.empty()) {
+				return PendingExecutionResult::NO_TASKS_AVAILABLE;
+			}
+			// At least one task is blocked
 			if (ResultCollectorIsBlocked()) {
-				// The blocked tasks are processing the Sink of a BufferedResultCollector
-				// We return here so the query result can be made and fetched from
-				// which will in turn unblock the Sink tasks.
-				return PendingExecutionResult::BLOCKED;
+				return PendingExecutionResult::RESULT_READY;
 			}
-			return PendingExecutionResult::NO_TASKS_AVAILABLE;
+			return PendingExecutionResult::BLOCKED;
 		}
 
 		if (current_task) {
@@ -545,7 +608,7 @@ PendingExecutionResult Executor::ExecuteTask(bool dry_run) {
 		execution_result = PendingExecutionResult::EXECUTION_ERROR;
 		ThrowException();
 	} // LCOV_EXCL_STOP
-	execution_result = PendingExecutionResult::RESULT_READY;
+	execution_result = PendingExecutionResult::EXECUTION_FINISHED;
 	return execution_result;
 }
 
@@ -610,8 +673,15 @@ void Executor::ThrowException() {
 	error_manager.ThrowException();
 }
 
-void Executor::Flush(ThreadContext &tcontext) {
-	profiler->Flush(tcontext.profiler);
+void Executor::Flush(ThreadContext &thread_context) {
+	static constexpr std::chrono::milliseconds WAIT_TIME_MS = std::chrono::milliseconds(WAIT_TIME);
+	auto global_profiler = profiler;
+	if (global_profiler) {
+		global_profiler->Flush(thread_context.profiler);
+
+		auto blocked_time = blocked_thread_time.load();
+		global_profiler->SetInfo(double(blocked_time * WAIT_TIME_MS.count()) / 1000);
+	}
 }
 
 bool Executor::GetPipelinesProgress(double &current_progress, uint64_t &current_cardinality,
@@ -640,8 +710,8 @@ bool Executor::GetPipelinesProgress(double &current_progress, uint64_t &current_
 
 	for (size_t i = 0; i < progress.size(); i++) {
 		progress[i] = MaxValue(0.0, MinValue(100.0, progress[i]));
-		current_cardinality = NumericCast<idx_t>(static_cast<double>(
-		    current_cardinality +
+		current_cardinality = LossyNumericCast<idx_t>(static_cast<double>(
+		    static_cast<double>(current_cardinality) +
 		    static_cast<double>(progress[i]) * static_cast<double>(cardinality[i]) / static_cast<double>(100)));
 		current_progress += progress[i] * double(cardinality[i]) / double(total_cardinality);
 		D_ASSERT(current_cardinality <= total_cardinality);
@@ -653,6 +723,14 @@ bool Executor::HasResultCollector() {
 	return physical_plan->type == PhysicalOperatorType::RESULT_COLLECTOR;
 }
 
+bool Executor::HasStreamingResultCollector() {
+	if (!HasResultCollector()) {
+		return false;
+	}
+	auto &result_collector = physical_plan->Cast<PhysicalResultCollector>();
+	return result_collector.IsStreaming();
+}
+
 unique_ptr<QueryResult> Executor::GetResult() {
 	D_ASSERT(HasResultCollector());
 	auto &result_collector = physical_plan->Cast<PhysicalResultCollector>();