duckdb 0.8.2-dev37.0 → 0.8.2-dev3949.0

package/src/duckdb/src/optimizer/remove_duplicate_groups.cpp

@@ -0,0 +1,127 @@
+#include "duckdb/optimizer/remove_duplicate_groups.hpp"
+
+#include "duckdb/common/pair.hpp"
+#include "duckdb/planner/expression/bound_columnref_expression.hpp"
+#include "duckdb/planner/operator/logical_aggregate.hpp"
+
+namespace duckdb {
+
+void RemoveDuplicateGroups::VisitOperator(LogicalOperator &op) {
+	switch (op.type) {
+	case LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY:
+		VisitAggregate(op.Cast<LogicalAggregate>());
+		break;
+	default:
+		break;
+	}
+	LogicalOperatorVisitor::VisitOperatorExpressions(op);
+	LogicalOperatorVisitor::VisitOperatorChildren(op);
+}
+
+void RemoveDuplicateGroups::VisitAggregate(LogicalAggregate &aggr) {
+	if (!aggr.grouping_functions.empty()) {
+		return;
+	}
+
+	auto &groups = aggr.groups;
+
+	column_binding_map_t<idx_t> duplicate_map;
+	vector<pair<idx_t, idx_t>> duplicates;
+	for (idx_t group_idx = 0; group_idx < groups.size(); group_idx++) {
+		const auto &group = groups[group_idx];
+		if (group->type != ExpressionType::BOUND_COLUMN_REF) {
+			continue;
+		}
+		const auto &colref = group->Cast<BoundColumnRefExpression>();
+		const auto &binding = colref.binding;
+		const auto it = duplicate_map.find(binding);
+		if (it == duplicate_map.end()) {
+			duplicate_map.emplace(binding, group_idx);
+		} else {
+			duplicates.emplace_back(it->second, group_idx);
+		}
+	}
+
+	if (duplicates.empty()) {
+		return;
+	}
+
+	// Sort duplicates by max duplicate group idx, because we want to remove groups from the back
+	sort(duplicates.begin(), duplicates.end(),
+	     [](const pair<idx_t, idx_t> &lhs, const pair<idx_t, idx_t> &rhs) { return lhs.second > rhs.second; });
+
+	// Now we want to remove the duplicates, but this alters the column bindings coming out of the aggregate,
+	// so we keep track of how they shift and do another round of column binding replacements
+	column_binding_map_t<ColumnBinding> group_binding_map;
+	for (idx_t group_idx = 0; group_idx < groups.size(); group_idx++) {
+		group_binding_map.emplace(ColumnBinding(aggr.group_index, group_idx),
+		                          ColumnBinding(aggr.group_index, group_idx));
+	}
+
+	for (idx_t duplicate_idx = 0; duplicate_idx < duplicates.size(); duplicate_idx++) {
+		const auto &duplicate = duplicates[duplicate_idx];
+		const auto &remaining_idx = duplicate.first;
+		const auto &removed_idx = duplicate.second;
+
+		// Store expression and remove it from groups
+		stored_expressions.emplace_back(std::move(groups[removed_idx]));
+		groups.erase(groups.begin() + removed_idx);
+
+		// This optimizer should run before statistics propagation, so this should be empty
+		// If it runs after, then group_stats should be updated too
+		D_ASSERT(aggr.group_stats.empty());
+
+		// Remove from grouping sets too
+		for (auto &grouping_set : aggr.grouping_sets) {
+			// Replace removed group with duplicate remaining group
+			if (grouping_set.erase(removed_idx) != 0) {
+				grouping_set.insert(remaining_idx);
+			}
+
+			// Indices shifted: Reinsert groups in the set with group_idx - 1
+			vector<idx_t> group_indices_to_reinsert;
+			for (auto &entry : grouping_set) {
+				if (entry > removed_idx) {
+					group_indices_to_reinsert.emplace_back(entry);
+				}
+			}
+			for (const auto group_idx : group_indices_to_reinsert) {
+				grouping_set.erase(group_idx);
+			}
+			for (const auto group_idx : group_indices_to_reinsert) {
+				grouping_set.insert(group_idx - 1);
+			}
+		}
+
+		// Update mapping
+		auto it = group_binding_map.find(ColumnBinding(aggr.group_index, removed_idx));
+		D_ASSERT(it != group_binding_map.end());
+		it->second.column_index = remaining_idx;
+
+		for (auto &map_entry : group_binding_map) {
+			auto &new_binding = map_entry.second;
+			if (new_binding.column_index > removed_idx) {
+				new_binding.column_index--;
+			}
+		}
+	}
+
+	// Replace all references to the old group binding with the new group binding
+	for (const auto &map_entry : group_binding_map) {
+		auto it = column_references.find(map_entry.first);
+		if (it != column_references.end()) {
+			for (auto expr : it->second) {
+				expr.get().binding = map_entry.second;
+			}
+		}
+	}
+}
+
+unique_ptr<Expression> RemoveDuplicateGroups::VisitReplace(BoundColumnRefExpression &expr,
+                                                           unique_ptr<Expression> *expr_ptr) {
+	// add a column reference
+	column_references[expr.binding].push_back(expr);
+	return nullptr;
+}
+
+} // namespace duckdb

package/src/duckdb/src/optimizer/remove_unused_columns.cpp

@@ -302,6 +302,10 @@ void RemoveUnusedColumns::VisitOperator(LogicalOperator &op) {
 		everything_referenced = true;
 		break;
 	}
+	case LogicalOperatorType::LOGICAL_MATERIALIZED_CTE: {
+		everything_referenced = true;
+		break;
+	}
 	case LogicalOperatorType::LOGICAL_CTE_REF: {
 		everything_referenced = true;
 		break;

package/src/duckdb/src/optimizer/rule/regex_optimizations.cpp

@@ -3,6 +3,8 @@
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/planner/expression/bound_function_expression.hpp"
 #include "duckdb/planner/expression/bound_constant_expression.hpp"
+#include "duckdb/function/scalar/string_functions.hpp"
+#include "duckdb/function/scalar/regexp.hpp"
 
 #include "re2/re2.h"
 #include "re2/regexp.h"
@@ -12,17 +14,138 @@ namespace duckdb {
 RegexOptimizationRule::RegexOptimizationRule(ExpressionRewriter &rewriter) : Rule(rewriter) {
 	auto func = make_uniq<FunctionExpressionMatcher>();
 	func->function = make_uniq<SpecificFunctionMatcher>("regexp_matches");
-	func->policy = SetMatcher::Policy::ORDERED;
+	func->policy = SetMatcher::Policy::SOME_ORDERED;
 	func->matchers.push_back(make_uniq<ExpressionMatcher>());
 	func->matchers.push_back(make_uniq<ConstantExpressionMatcher>());
+
 	root = std::move(func);
 }
 
+struct LikeString {
+	bool exists = true;
+	bool escaped = false;
+	string like_string = "";
+};
+
+static void AddCharacter(char chr, LikeString &ret, bool contains) {
+	// if we are not converting into a contains, and the string has LIKE special characters
+	// then don't return a possible LIKE match
+	// same if the character is a control character
+	if (iscntrl(chr) || (!contains && (chr == '%' || chr == '_'))) {
+		ret.exists = false;
+		return;
+	}
+	auto run_as_str {chr};
+	ret.like_string += run_as_str;
+}
+
+static LikeString GetLikeStringEscaped(duckdb_re2::Regexp *regexp, bool contains = false) {
+	D_ASSERT(regexp->op() == duckdb_re2::kRegexpLiteralString || regexp->op() == duckdb_re2::kRegexpLiteral);
+	LikeString ret;
+
+	if (regexp->parse_flags() & duckdb_re2::Regexp::FoldCase ||
+	    !(regexp->parse_flags() & duckdb_re2::Regexp::OneLine)) {
+		// parse flags can turn on and off within a regex match, return no optimization
+		// For now, we just don't optimize if these every turn on.
+		// TODO: logic to attempt the optimization, then if the parse flags change, then abort
+		ret.exists = false;
+		return ret;
+	}
+
+	// case insensitivity may be on now, but it can also turn off.
+	if (regexp->op() == duckdb_re2::kRegexpLiteralString) {
+		auto nrunes = (idx_t)regexp->nrunes();
+		auto runes = regexp->runes();
+		for (idx_t i = 0; i < nrunes; i++) {
+			char chr = toascii(runes[i]);
+			AddCharacter(chr, ret, contains);
+			if (!ret.exists) {
+				return ret;
+			}
+		}
+	} else {
+		auto rune = regexp->rune();
+		char chr = toascii(rune);
+		AddCharacter(chr, ret, contains);
+	}
+	D_ASSERT(ret.like_string.size() >= 1 || !ret.exists);
+	return ret;
+}
+
+static LikeString LikeMatchFromRegex(duckdb_re2::RE2 &pattern) {
+	LikeString ret = LikeString();
+	auto num_subs = pattern.Regexp()->nsub();
+	auto subs = pattern.Regexp()->sub();
+	auto cur_sub_index = 0;
+	while (cur_sub_index < num_subs) {
+		switch (subs[cur_sub_index]->op()) {
+		case duckdb_re2::kRegexpAnyChar:
+			if (cur_sub_index == 0) {
+				ret.like_string += "%";
+			}
+			ret.like_string += "_";
+			if (cur_sub_index + 1 == num_subs) {
+				ret.like_string += "%";
+			}
+			break;
+		case duckdb_re2::kRegexpStar:
+			// .* is a Star operator is a anyChar operator as a child.
+			// any other child operator would represent a pattern LIKE cannot match.
+			if (subs[cur_sub_index]->nsub() == 1 && subs[cur_sub_index]->sub()[0]->op() == duckdb_re2::kRegexpAnyChar) {
+				ret.like_string += "%";
+				break;
+			}
+			ret.exists = false;
+			return ret;
+		case duckdb_re2::kRegexpLiteralString:
+		case duckdb_re2::kRegexpLiteral: {
+			// if this is the only matching op, we should have directly called
+			// GetEscapedLikeString
+			D_ASSERT(!(cur_sub_index == 0 && cur_sub_index + 1 == num_subs));
+			if (cur_sub_index == 0) {
+				ret.like_string += "%";
+			}
+			// if the kRegexpLiteral or kRegexpLiteralString is the only op to match
+			// the string can directly be converted into a contains
+			LikeString escaped_like_string = GetLikeStringEscaped(subs[cur_sub_index], false);
+			if (!escaped_like_string.exists) {
+				return escaped_like_string;
+			}
+			ret.like_string += escaped_like_string.like_string;
+			ret.escaped = escaped_like_string.escaped;
+			if (cur_sub_index + 1 == num_subs) {
+				ret.like_string += "%";
+			}
+			break;
+		}
+		case duckdb_re2::kRegexpEndText:
+		case duckdb_re2::kRegexpEmptyMatch:
+		case duckdb_re2::kRegexpBeginText: {
+			break;
+		}
+		default:
+			// some other regexp op that doesn't have an equivalent to a like string
+			// return false;
+			ret.exists = false;
+			return ret;
+		}
+		cur_sub_index += 1;
+	}
+	return ret;
+}
+
 unique_ptr<Expression> RegexOptimizationRule::Apply(LogicalOperator &op, vector<reference<Expression>> &bindings,
                                                     bool &changes_made, bool is_root) {
 	auto &root = bindings[0].get().Cast<BoundFunctionExpression>();
 	auto &constant_expr = bindings[2].get().Cast<BoundConstantExpression>();
-	D_ASSERT(root.children.size() == 2);
+	D_ASSERT(root.children.size() == 2 || root.children.size() == 3);
+	auto regexp_bind_data = root.bind_info.get()->Cast<RegexpMatchesBindData>();
+
+	auto constant_value = ExpressionExecutor::EvaluateScalar(GetContext(), constant_expr);
+	D_ASSERT(constant_value.type() == constant_expr.return_type);
+	auto patt_str = StringValue::Get(constant_value);
+
+	duckdb_re2::RE2::Options parsed_options = regexp_bind_data.options;
 
 	if (constant_expr.value.IsNull()) {
 		return make_uniq<BoundConstantExpression>(Value(root.return_type));
@@ -31,34 +154,50 @@ unique_ptr<Expression> RegexOptimizationRule::Apply(LogicalOperator &op, vector<
 	// the constant_expr is a scalar expression that we have to fold
 	if (!constant_expr.IsFoldable()) {
 		return nullptr;
-	}
+	};
 
-	auto constant_value = ExpressionExecutor::EvaluateScalar(GetContext(), constant_expr);
-	D_ASSERT(constant_value.type() == constant_expr.return_type);
-	auto patt_str = StringValue::Get(constant_value);
-
-	duckdb_re2::RE2 pattern(patt_str);
+	duckdb_re2::RE2 pattern(patt_str, parsed_options);
 	if (!pattern.ok()) {
 		return nullptr; // this should fail somewhere else
 	}
 
+	LikeString like_string;
+	// check for a like string. If we can convert it to a like string, the like string
+	// optimizer will further optimize suffix and prefix things.
 	if (pattern.Regexp()->op() == duckdb_re2::kRegexpLiteralString ||
 	    pattern.Regexp()->op() == duckdb_re2::kRegexpLiteral) {
-
-		string min;
-		string max;
-		pattern.PossibleMatchRange(&min, &max, patt_str.size() + 1);
-		if (min != max) {
+		// convert to contains.
+		LikeString escaped_like_string = GetLikeStringEscaped(pattern.Regexp(), true);
+		if (!escaped_like_string.exists) {
 			return nullptr;
 		}
-		auto parameter = make_uniq<BoundConstantExpression>(Value(std::move(min)));
+		auto parameter = make_uniq<BoundConstantExpression>(Value(std::move(escaped_like_string.like_string)));
 		auto contains = make_uniq<BoundFunctionExpression>(root.return_type, ContainsFun::GetFunction(),
 		                                                   std::move(root.children), nullptr);
 		contains->children[1] = std::move(parameter);
 
 		return std::move(contains);
+	} else if (pattern.Regexp()->op() == duckdb_re2::kRegexpConcat) {
+		like_string = LikeMatchFromRegex(pattern);
+	} else {
+		like_string.exists = false;
+	}
+
+	if (!like_string.exists) {
+		return nullptr;
+	}
+
+	// if regexp had options, remove them so the new Like Expression can be matched for other optimizers.
+	if (root.children.size() == 3) {
+		root.children.pop_back();
+		D_ASSERT(root.children.size() == 2);
 	}
-	return nullptr;
+
+	auto like_expression = make_uniq<BoundFunctionExpression>(root.return_type, LikeFun::GetLikeFunction(),
+	                                                          std::move(root.children), nullptr);
+	auto parameter = make_uniq<BoundConstantExpression>(Value(std::move(like_string.like_string)));
+	like_expression->children[1] = std::move(parameter);
+	return std::move(like_expression);
 }
 
 } // namespace duckdb

package/src/duckdb/src/optimizer/statistics/operator/propagate_join.cpp

@@ -1,9 +1,13 @@
 #include "duckdb/common/types/hugeint.hpp"
+#include "duckdb/optimizer/filter_pushdown.hpp"
 #include "duckdb/optimizer/statistics_propagator.hpp"
 #include "duckdb/planner/expression/bound_columnref_expression.hpp"
+#include "duckdb/planner/expression/bound_comparison_expression.hpp"
+#include "duckdb/planner/expression/bound_constant_expression.hpp"
 #include "duckdb/planner/operator/logical_any_join.hpp"
 #include "duckdb/planner/operator/logical_comparison_join.hpp"
 #include "duckdb/planner/operator/logical_cross_product.hpp"
+#include "duckdb/planner/operator/logical_filter.hpp"
 #include "duckdb/planner/operator/logical_join.hpp"
 #include "duckdb/planner/operator/logical_limit.hpp"
 #include "duckdb/planner/operator/logical_positional_join.hpp"
@@ -13,8 +17,8 @@ namespace duckdb {
 void StatisticsPropagator::PropagateStatistics(LogicalComparisonJoin &join, unique_ptr<LogicalOperator> *node_ptr) {
 	for (idx_t i = 0; i < join.conditions.size(); i++) {
 		auto &condition = join.conditions[i];
-		auto stats_left = PropagateExpression(condition.left);
-		auto stats_right = PropagateExpression(condition.right);
+		const auto stats_left = PropagateExpression(condition.left);
+		const auto stats_right = PropagateExpression(condition.right);
 		if (stats_left && stats_right) {
 			if ((condition.comparison == ExpressionType::COMPARE_DISTINCT_FROM ||
 			     condition.comparison == ExpressionType::COMPARE_NOT_DISTINCT_FROM) &&
@@ -25,8 +29,8 @@ void StatisticsPropagator::PropagateStatistics(LogicalComparisonJoin &join, uniq
 			}
 			auto prune_result = PropagateComparison(*stats_left, *stats_right, condition.comparison);
 			// Add stats to logical_join for perfect hash join
-			join.join_stats.push_back(std::move(stats_left));
-			join.join_stats.push_back(std::move(stats_right));
+			join.join_stats.push_back(stats_left->ToUnique());
+			join.join_stats.push_back(stats_right->ToUnique());
 			switch (prune_result) {
 			case FilterPropagateResult::FILTER_FALSE_OR_NULL:
 			case FilterPropagateResult::FILTER_ALWAYS_FALSE:
@@ -123,12 +127,21 @@ void StatisticsPropagator::PropagateStatistics(LogicalComparisonJoin &join, uniq
 		case JoinType::INNER:
 		case JoinType::SEMI: {
 			UpdateFilterStatistics(*condition.left, *condition.right, condition.comparison);
-			auto stats_left = PropagateExpression(condition.left);
-			auto stats_right = PropagateExpression(condition.right);
+			auto updated_stats_left = PropagateExpression(condition.left);
+			auto updated_stats_right = PropagateExpression(condition.right);
+
+			// Try to push lhs stats down rhs and vice versa
+			if (!context.config.force_index_join && stats_left && stats_right && updated_stats_left &&
+			    updated_stats_right && condition.left->type == ExpressionType::BOUND_COLUMN_REF &&
+			    condition.right->type == ExpressionType::BOUND_COLUMN_REF) {
+				CreateFilterFromJoinStats(join.children[0], condition.left, *stats_left, *updated_stats_left);
+				CreateFilterFromJoinStats(join.children[1], condition.right, *stats_right, *updated_stats_right);
+			}
+
 			// Update join_stats when is already part of the join
 			if (join.join_stats.size() == 2) {
-				join.join_stats[0] = std::move(stats_left);
-				join.join_stats[1] = std::move(stats_right);
+				join.join_stats[0] = std::move(updated_stats_left);
+				join.join_stats[1] = std::move(updated_stats_right);
 			}
 			break;
 		}
@@ -281,4 +294,48 @@ unique_ptr<NodeStatistics> StatisticsPropagator::PropagateStatistics(LogicalPosi
 	return std::move(node_stats);
 }
 
+void StatisticsPropagator::CreateFilterFromJoinStats(unique_ptr<LogicalOperator> &child, unique_ptr<Expression> &expr,
+                                                     const BaseStatistics &stats_before,
+                                                     const BaseStatistics &stats_after) {
+	// Only do this for integral colref's that have stats
+	if (expr->type != ExpressionType::BOUND_COLUMN_REF || !expr->return_type.IsIntegral() ||
+	    !NumericStats::HasMinMax(stats_before) || !NumericStats::HasMinMax(stats_after)) {
+		return;
+	}
+
+	// Retrieve min/max
+	auto min_before = NumericStats::Min(stats_before);
+	auto max_before = NumericStats::Max(stats_before);
+	auto min_after = NumericStats::Min(stats_after);
+	auto max_after = NumericStats::Max(stats_after);
+
+	vector<unique_ptr<Expression>> filter_exprs;
+	if (min_after > min_before) {
+		filter_exprs.emplace_back(
+		    make_uniq<BoundComparisonExpression>(ExpressionType::COMPARE_GREATERTHANOREQUALTO, expr->Copy(),
+		                                         make_uniq<BoundConstantExpression>(std::move(min_after))));
+	}
+	if (max_after < max_before) {
+		filter_exprs.emplace_back(
+		    make_uniq<BoundComparisonExpression>(ExpressionType::COMPARE_LESSTHANOREQUALTO, expr->Copy(),
+		                                         make_uniq<BoundConstantExpression>(std::move(max_after))));
+	}
+
+	if (filter_exprs.empty()) {
+		return;
+	}
+
+	auto filter = make_uniq<LogicalFilter>();
+	filter->children.emplace_back(std::move(child));
+	child = std::move(filter);
+
+	for (auto &filter_expr : filter_exprs) {
+		child->expressions.emplace_back(std::move(filter_expr));
+	}
+
+	FilterPushdown filter_pushdown(optimizer);
+	child = filter_pushdown.Rewrite(std::move(child));
+	PropagateExpression(expr);
+}
+
 } // namespace duckdb

package/src/duckdb/src/optimizer/statistics/operator/propagate_order.cpp

@@ -11,7 +11,7 @@ unique_ptr<NodeStatistics> StatisticsPropagator::PropagateStatistics(LogicalOrde
 
 	// then propagate to each of the order expressions
 	for (auto &bound_order : order.orders) {
-		PropagateAndCompress(bound_order.expression, bound_order.stats);
+		bound_order.stats = PropagateExpression(bound_order.expression);
 	}
 	return std::move(node_stats);
 }

package/src/duckdb/src/optimizer/statistics_propagator.cpp

@@ -1,24 +1,26 @@
 #include "duckdb/optimizer/statistics_propagator.hpp"
 
 #include "duckdb/main/client_context.hpp"
+#include "duckdb/optimizer/optimizer.hpp"
+#include "duckdb/planner/expression/list.hpp"
 #include "duckdb/planner/expression_iterator.hpp"
 #include "duckdb/planner/logical_operator.hpp"
 #include "duckdb/planner/operator/logical_aggregate.hpp"
-#include "duckdb/planner/operator/logical_empty_result.hpp"
 #include "duckdb/planner/operator/logical_cross_product.hpp"
+#include "duckdb/planner/operator/logical_empty_result.hpp"
 #include "duckdb/planner/operator/logical_filter.hpp"
 #include "duckdb/planner/operator/logical_get.hpp"
 #include "duckdb/planner/operator/logical_join.hpp"
-#include "duckdb/planner/operator/logical_projection.hpp"
+#include "duckdb/planner/operator/logical_order.hpp"
 #include "duckdb/planner/operator/logical_positional_join.hpp"
+#include "duckdb/planner/operator/logical_projection.hpp"
 #include "duckdb/planner/operator/logical_set_operation.hpp"
-#include "duckdb/planner/operator/logical_order.hpp"
 #include "duckdb/planner/operator/logical_window.hpp"
-#include "duckdb/planner/expression/list.hpp"
 
 namespace duckdb {
 
-StatisticsPropagator::StatisticsPropagator(ClientContext &context) : context(context) {
+StatisticsPropagator::StatisticsPropagator(Optimizer &optimizer_p)
+    : optimizer(optimizer_p), context(optimizer.context) {
 }
 
 void StatisticsPropagator::ReplaceWithEmptyResult(unique_ptr<LogicalOperator> &node) {

package/src/duckdb/src/optimizer/topn_optimizer.cpp

@@ -1,24 +1,41 @@
 #include "duckdb/optimizer/topn_optimizer.hpp"
-#include "duckdb/planner/operator/logical_order.hpp"
+
+#include "duckdb/common/limits.hpp"
 #include "duckdb/planner/operator/logical_limit.hpp"
+#include "duckdb/planner/operator/logical_order.hpp"
 #include "duckdb/planner/operator/logical_top_n.hpp"
-#include "duckdb/common/limits.hpp"
 
 namespace duckdb {
 
-unique_ptr<LogicalOperator> TopN::Optimize(unique_ptr<LogicalOperator> op) {
-	if (op->type == LogicalOperatorType::LOGICAL_LIMIT &&
-	    op->children[0]->type == LogicalOperatorType::LOGICAL_ORDER_BY) {
-		auto &limit = op->Cast<LogicalLimit>();
-		auto &order_by = (op->children[0])->Cast<LogicalOrder>();
+bool TopN::CanOptimize(LogicalOperator &op) {
+	if (op.type == LogicalOperatorType::LOGICAL_LIMIT &&
+	    op.children[0]->type == LogicalOperatorType::LOGICAL_ORDER_BY) {
+		auto &limit = op.Cast<LogicalLimit>();
+
+		// When there are some expressions in the limit operator,
+		// we shouldn't use this optimizations. Because of the expressions
+		// will be lost when it convert to TopN operator.
+		if (limit.limit || limit.offset) {
+			return false;
+		}
 
 		// This optimization doesn't apply when OFFSET is present without LIMIT
 		// Or if offset is not constant
 		if (limit.limit_val != NumericLimits<int64_t>::Maximum() || limit.offset) {
-			auto topn = make_uniq<LogicalTopN>(std::move(order_by.orders), limit.limit_val, limit.offset_val);
-			topn->AddChild(std::move(order_by.children[0]));
-			op = std::move(topn);
+			return true;
 		}
+	}
+	return false;
+}
+
+unique_ptr<LogicalOperator> TopN::Optimize(unique_ptr<LogicalOperator> op) {
+	if (CanOptimize(*op)) {
+		auto &limit = op->Cast<LogicalLimit>();
+		auto &order_by = (op->children[0])->Cast<LogicalOrder>();
+
+		auto topn = make_uniq<LogicalTopN>(std::move(order_by.orders), limit.limit_val, limit.offset_val);
+		topn->AddChild(std::move(order_by.children[0]));
+		op = std::move(topn);
 	} else {
 		for (auto &child : op->children) {
 			child = Optimize(std::move(child));

package/src/duckdb/src/optimizer/unnest_rewriter.cpp

@@ -2,7 +2,7 @@
 
 #include "duckdb/common/pair.hpp"
 #include "duckdb/planner/operator/logical_delim_get.hpp"
-#include "duckdb/planner/operator/logical_delim_join.hpp"
+#include "duckdb/planner/operator/logical_comparison_join.hpp"
 #include "duckdb/planner/operator/logical_unnest.hpp"
 #include "duckdb/planner/operator/logical_projection.hpp"
 #include "duckdb/planner/operator/logical_window.hpp"
@@ -17,11 +17,9 @@ void UnnestRewriterPlanUpdater::VisitOperator(LogicalOperator &op) {
 }
 
 void UnnestRewriterPlanUpdater::VisitExpression(unique_ptr<Expression> *expression) {
-
 	auto &expr = *expression;
 
 	if (expr->expression_class == ExpressionClass::BOUND_COLUMN_REF) {
-
 		auto &bound_column_ref = expr->Cast<BoundColumnRefExpression>();
 		for (idx_t i = 0; i < replace_bindings.size(); i++) {
 			if (bound_column_ref.binding == replace_bindings[i].old_binding) {
@@ -76,7 +74,7 @@ void UnnestRewriter::FindCandidates(unique_ptr<LogicalOperator> *op_ptr,
 	}
 
 	// found a delim join
-	auto &delim_join = op->children[0]->Cast<LogicalDelimJoin>();
+	auto &delim_join = op->children[0]->Cast<LogicalComparisonJoin>();
 	// only support INNER delim joins
 	if (delim_join.join_type != JoinType::INNER) {
 		return;
@@ -295,7 +293,7 @@ void UnnestRewriter::UpdateBoundUnnestBindings(UnnestRewriterPlanUpdater &update
 void UnnestRewriter::GetDelimColumns(LogicalOperator &op) {
 
 	D_ASSERT(op.type == LogicalOperatorType::LOGICAL_DELIM_JOIN);
-	auto &delim_join = op.Cast<LogicalDelimJoin>();
+	auto &delim_join = op.Cast<LogicalComparisonJoin>();
 	for (idx_t i = 0; i < delim_join.duplicate_eliminated_columns.size(); i++) {
 		auto &expr = *delim_join.duplicate_eliminated_columns[i];
 		D_ASSERT(expr.type == ExpressionType::BOUND_COLUMN_REF);

package/src/duckdb/src/parallel/executor.cpp

@@ -3,6 +3,7 @@
 #include "duckdb/execution/execution_context.hpp"
 #include "duckdb/execution/operator/helper/physical_result_collector.hpp"
 #include "duckdb/execution/operator/set/physical_recursive_cte.hpp"
+#include "duckdb/execution/operator/set/physical_cte.hpp"
 #include "duckdb/execution/physical_operator.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/main/client_data.hpp"
@@ -106,7 +107,7 @@ void Executor::SchedulePipeline(const shared_ptr<MetaPipeline> &meta_pipeline, S
 			                                  group_stack.pipeline_finish_event, base_stack.pipeline_complete_event);
 
 			// dependencies: base_finish -> pipeline_event -> group_finish
-			pipeline_stack.pipeline_event.AddDependency(base_stack.pipeline_event);
+			pipeline_stack.pipeline_event.AddDependency(base_stack.pipeline_finish_event);
 			group_stack.pipeline_finish_event.AddDependency(pipeline_stack.pipeline_event);
 
 			// add pipeline stack to event map
@@ -267,6 +268,10 @@ void Executor::AddRecursiveCTE(PhysicalOperator &rec_cte) {
 	recursive_ctes.push_back(rec_cte);
 }
 
+void Executor::AddMaterializedCTE(PhysicalOperator &mat_cte) {
+	materialized_ctes.push_back(mat_cte);
+}
+
 void Executor::ReschedulePipelines(const vector<shared_ptr<MetaPipeline>> &pipelines_p,
                                    vector<shared_ptr<Event>> &events_p) {
 	ScheduleEventData event_data(pipelines_p, events_p, false);
@@ -344,6 +349,12 @@ void Executor::InitializeInternal(PhysicalOperator &plan) {
 			rec_cte.recursive_meta_pipeline->Ready();
 		}
 
+		// ready materialized cte pipelines too
+		for (auto &mat_cte_ref : materialized_ctes) {
+			auto &mat_cte = mat_cte_ref.get().Cast<PhysicalCTE>();
+			mat_cte.recursive_meta_pipeline->Ready();
+		}
+
 		// set root pipelines, i.e., all pipelines that end in the final sink
 		root_pipeline->GetPipelines(root_pipelines, false);
 		root_pipeline_idx = 0;
@@ -381,6 +392,10 @@ void Executor::CancelTasks() {
 			auto &rec_cte = rec_cte_ref.get().Cast<PhysicalRecursiveCTE>();
 			rec_cte.recursive_meta_pipeline.reset();
 		}
+		for (auto &mat_cte_ref : materialized_ctes) {
+			auto &mat_cte = mat_cte_ref.get().Cast<PhysicalCTE>();
+			mat_cte.recursive_meta_pipeline.reset();
+		}
 		pipelines.clear();
 		root_pipelines.clear();
 		to_be_rescheduled_tasks.clear();
@@ -443,6 +458,8 @@ bool Executor::ExecutionIsFinished() {
 }
 
 PendingExecutionResult Executor::ExecuteTask() {
+	// Only executor should return NO_TASKS_AVAILABLE
+	D_ASSERT(execution_result != PendingExecutionResult::NO_TASKS_AVAILABLE);
 	if (execution_result != PendingExecutionResult::RESULT_NOT_READY) {
 		return execution_result;
 	}
@@ -453,6 +470,10 @@ PendingExecutionResult Executor::ExecuteTask() {
 		if (!task) {
 			scheduler.GetTaskFromProducer(*producer, task);
 		}
+		if (!task && !HasError()) {
+			// there are no tasks to be scheduled and there are tasks blocked
+			return PendingExecutionResult::NO_TASKS_AVAILABLE;
+		}
 		if (task) {
 			// if we have a task, partially process it
 			auto result = task->Execute(TaskExecutionMode::PROCESS_PARTIAL);
@@ -574,6 +595,9 @@ bool Executor::GetPipelinesProgress(double &current_progress) { // LCOV_EXCL_STA
 		total_cardinality += child_cardinality;
 	}
 	current_progress = 0;
+	if (total_cardinality == 0) {
+		return true;
+	}
 	for (size_t i = 0; i < progress.size(); i++) {
 		current_progress += progress[i] * double(cardinality[i]) / double(total_cardinality);
 	}