duckdb 0.8.2-dev157.0 → 0.8.2-dev1573.0

package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp

@@ -5,6 +5,7 @@
 #include "duckdb/planner/expression/list.hpp"
 #include "duckdb/planner/expression_iterator.hpp"
 #include "duckdb/planner/operator/list.hpp"
+#include "duckdb/common/queue.hpp"
 
 #include <algorithm>
 #include <cmath>
@@ -323,6 +324,65 @@ void JoinOrderOptimizer::UpdateJoinNodesInFullPlan(JoinNode &node) {
 	}
 }
 
+static vector<unordered_set<idx_t>> AddSuperSets(const vector<unordered_set<idx_t>> &current,
+                                                 const vector<idx_t> &all_neighbors) {
+	vector<unordered_set<idx_t>> ret;
+
+	for (const auto &neighbor_set : current) {
+		auto max_val = std::max_element(neighbor_set.begin(), neighbor_set.end());
+		for (const auto &neighbor : all_neighbors) {
+			if (*max_val >= neighbor) {
+				continue;
+			}
+			if (neighbor_set.count(neighbor) == 0) {
+				unordered_set<idx_t> new_set;
+				for (auto &n : neighbor_set) {
+					new_set.insert(n);
+				}
+				new_set.insert(neighbor);
+				ret.push_back(new_set);
+			}
+		}
+	}
+
+	return ret;
+}
+
+// works by first creating all sets with cardinality 1
+// then iterates over each previously created group of subsets and will only add a neighbor if the neighbor
+// is greater than all relations in the set.
+static vector<unordered_set<idx_t>> GetAllNeighborSets(vector<idx_t> neighbors) {
+	vector<unordered_set<idx_t>> ret;
+	sort(neighbors.begin(), neighbors.end());
+	vector<unordered_set<idx_t>> added;
+	for (auto &neighbor : neighbors) {
+		added.push_back(unordered_set<idx_t>({neighbor}));
+		ret.push_back(unordered_set<idx_t>({neighbor}));
+	}
+	do {
+		added = AddSuperSets(added, neighbors);
+		for (auto &d : added) {
+			ret.push_back(d);
+		}
+	} while (!added.empty());
+#if DEBUG
+	// drive by test to make sure we have an accurate amount of
+	// subsets, and that each neighbor is in a correct amount
+	// of those subsets.
+	D_ASSERT(ret.size() == pow(2, neighbors.size()) - 1);
+	for (auto &n : neighbors) {
+		idx_t count = 0;
+		for (auto &set : ret) {
+			if (set.count(n) >= 1) {
+				count += 1;
+			}
+		}
+		D_ASSERT(count == pow(2, neighbors.size() - 1));
+	}
+#endif
+	return ret;
+}
+
 JoinNode &JoinOrderOptimizer::EmitPair(JoinRelationSet &left, JoinRelationSet &right,
                                        const vector<reference<NeighborInfo>> &info) {
 	// get the left and right join plans
@@ -405,8 +465,19 @@ bool JoinOrderOptimizer::EmitCSG(JoinRelationSet &node) {
 	//! Neighbors should be reversed when iterating over them.
 	std::sort(neighbors.begin(), neighbors.end(), std::greater_equal<idx_t>());
 	for (idx_t i = 0; i < neighbors.size() - 1; i++) {
-		D_ASSERT(neighbors[i] >= neighbors[i + 1]);
+		D_ASSERT(neighbors[i] > neighbors[i + 1]);
+	}
+
+	// Dphyp paper missiing this.
+	// Because we are traversing in reverse order, we need to add neighbors whose number is smaller than the current
+	// node to exclusion_set
+	// This avoids duplicated enumeration
+	unordered_set<idx_t> new_exclusion_set = exclusion_set;
+	for (idx_t i = 0; i < neighbors.size(); ++i) {
+		D_ASSERT(new_exclusion_set.find(neighbors[i]) == new_exclusion_set.end());
+		new_exclusion_set.insert(neighbors[i]);
 	}
+
 	for (auto neighbor : neighbors) {
 		// since the GetNeighbors only returns the smallest element in a list, the entry might not be connected to
 		// (only!) this neighbor,  hence we have to do a connectedness check before we can emit it
@@ -417,27 +488,35 @@ bool JoinOrderOptimizer::EmitCSG(JoinRelationSet &node) {
 				return false;
 			}
 		}
-		if (!EnumerateCmpRecursive(node, neighbor_relation, exclusion_set)) {
+
+		if (!EnumerateCmpRecursive(node, neighbor_relation, new_exclusion_set)) {
 			return false;
 		}
+
+		new_exclusion_set.erase(neighbor);
 	}
 	return true;
 }
 
 bool JoinOrderOptimizer::EnumerateCmpRecursive(JoinRelationSet &left, JoinRelationSet &right,
-                                               unordered_set<idx_t> exclusion_set) {
+                                               unordered_set<idx_t> &exclusion_set) {
 	// get the neighbors of the second relation under the exclusion set
 	auto neighbors = query_graph.GetNeighbors(right, exclusion_set);
 	if (neighbors.empty()) {
 		return true;
 	}
+
+	auto all_subset = GetAllNeighborSets(neighbors);
 	vector<reference<JoinRelationSet>> union_sets;
-	union_sets.reserve(neighbors.size());
-	for (idx_t i = 0; i < neighbors.size(); i++) {
-		auto &neighbor = set_manager.GetJoinRelation(neighbors[i]);
+	union_sets.reserve(all_subset.size());
+	for (const auto &rel_set : all_subset) {
+		auto &neighbor = set_manager.GetJoinRelation(rel_set);
 		// emit the combinations of this node and its neighbors
 		auto &combined_set = set_manager.Union(right, neighbor);
-		if (combined_set.count > right.count && plans.find(&combined_set) != plans.end()) {
+		// If combined_set.count == right.count, This means we found a neighbor that has been present before
+		// This means we didn't set exclusion_set correctly.
+		D_ASSERT(combined_set.count > right.count);
+		if (plans.find(&combined_set) != plans.end()) {
 			auto connections = query_graph.GetConnections(left, combined_set);
 			if (!connections.empty()) {
 				if (!TryEmitPair(left, combined_set, connections)) {
@@ -447,11 +526,15 @@ bool JoinOrderOptimizer::EnumerateCmpRecursive(JoinRelationSet &left, JoinRelati
 		}
 		union_sets.push_back(combined_set);
 	}
-	// recursively enumerate the sets
+
 	unordered_set<idx_t> new_exclusion_set = exclusion_set;
-	for (idx_t i = 0; i < neighbors.size(); i++) {
+	for (const auto &neighbor : neighbors) {
+		new_exclusion_set.insert(neighbor);
+	}
+
+	// recursively enumerate the sets
+	for (idx_t i = 0; i < union_sets.size(); i++) {
 		// updated the set of excluded entries with this neighbor
-		new_exclusion_set.insert(neighbors[i]);
 		if (!EnumerateCmpRecursive(left, union_sets[i], new_exclusion_set)) {
 			return false;
 		}
@@ -465,26 +548,30 @@ bool JoinOrderOptimizer::EnumerateCSGRecursive(JoinRelationSet &node, unordered_
 	if (neighbors.empty()) {
 		return true;
 	}
+
+	auto all_subset = GetAllNeighborSets(neighbors);
 	vector<reference<JoinRelationSet>> union_sets;
-	union_sets.reserve(neighbors.size());
-	for (idx_t i = 0; i < neighbors.size(); i++) {
-		auto &neighbor = set_manager.GetJoinRelation(neighbors[i]);
+	union_sets.reserve(all_subset.size());
+	for (const auto &rel_set : all_subset) {
+		auto &neighbor = set_manager.GetJoinRelation(rel_set);
 		// emit the combinations of this node and its neighbors
 		auto &new_set = set_manager.Union(node, neighbor);
-		if (new_set.count > node.count && plans.find(&new_set) != plans.end()) {
+		D_ASSERT(new_set.count > node.count);
+		if (plans.find(&new_set) != plans.end()) {
 			if (!EmitCSG(new_set)) {
 				return false;
 			}
 		}
 		union_sets.push_back(new_set);
 	}
-	// recursively enumerate the sets
+
 	unordered_set<idx_t> new_exclusion_set = exclusion_set;
-	for (idx_t i = 0; i < neighbors.size(); i++) {
-		// Reset the exclusion set so that the algorithm considers all combinations
-		// of the exclusion_set with a subset of neighbors.
-		new_exclusion_set = exclusion_set;
-		new_exclusion_set.insert(neighbors[i]);
+	for (const auto &neighbor : neighbors) {
+		new_exclusion_set.insert(neighbor);
+	}
+
+	// recursively enumerate the sets
+	for (idx_t i = 0; i < union_sets.size(); i++) {
 		// updated the set of excluded entries with this neighbor
 		if (!EnumerateCSGRecursive(union_sets[i], new_exclusion_set)) {
 			return false;
@@ -505,7 +592,7 @@ bool JoinOrderOptimizer::SolveJoinOrderExactly() {
 		}
 		// initialize the set of exclusion_set as all the nodes with a number below this
 		unordered_set<idx_t> exclusion_set;
-		for (idx_t j = 0; j < i - 1; j++) {
+		for (idx_t j = 0; j < i; j++) {
 			exclusion_set.insert(j);
 		}
 		// then we recursively search for neighbors that do not belong to the banned entries
@@ -516,63 +603,6 @@ bool JoinOrderOptimizer::SolveJoinOrderExactly() {
 	return true;
 }
 
-static vector<unordered_set<idx_t>> AddSuperSets(vector<unordered_set<idx_t>> current,
-                                                 const vector<idx_t> &all_neighbors) {
-	vector<unordered_set<idx_t>> ret;
-	for (auto &neighbor : all_neighbors) {
-		for (auto &neighbor_set : current) {
-			auto max_val = std::max_element(neighbor_set.begin(), neighbor_set.end());
-			if (*max_val >= neighbor) {
-				continue;
-			}
-			if (neighbor_set.count(neighbor) == 0) {
-				unordered_set<idx_t> new_set;
-				for (auto &n : neighbor_set) {
-					new_set.insert(n);
-				}
-				new_set.insert(neighbor);
-				ret.push_back(new_set);
-			}
-		}
-	}
-	return ret;
-}
-
-// works by first creating all sets with cardinality 1
-// then iterates over each previously created group of subsets and will only add a neighbor if the neighbor
-// is greater than all relations in the set.
-static vector<unordered_set<idx_t>> GetAllNeighborSets(unordered_set<idx_t> &exclusion_set, vector<idx_t> neighbors) {
-	vector<unordered_set<idx_t>> ret;
-	sort(neighbors.begin(), neighbors.end());
-	vector<unordered_set<idx_t>> added;
-	for (auto &neighbor : neighbors) {
-		added.push_back(unordered_set<idx_t>({neighbor}));
-		ret.push_back(unordered_set<idx_t>({neighbor}));
-	}
-	do {
-		added = AddSuperSets(added, neighbors);
-		for (auto &d : added) {
-			ret.push_back(d);
-		}
-	} while (!added.empty());
-#if DEBUG
-	// drive by test to make sure we have an accurate amount of
-	// subsets, and that each neighbor is in a correct amount
-	// of those subsets.
-	D_ASSERT(ret.size() == pow(2, neighbors.size()) - 1);
-	for (auto &n : neighbors) {
-		idx_t count = 0;
-		for (auto &set : ret) {
-			if (set.count(n) >= 1) {
-				count += 1;
-			}
-		}
-		D_ASSERT(count == pow(2, neighbors.size() - 1));
-	}
-#endif
-	return ret;
-}
-
 void JoinOrderOptimizer::UpdateDPTree(JoinNode &new_plan) {
 	if (!NodeInFullPlan(new_plan)) {
 		// if the new node is not in the full plan, feel free to return
@@ -586,8 +616,8 @@ void JoinOrderOptimizer::UpdateDPTree(JoinNode &new_plan) {
 		exclusion_set.insert(new_set.relations[i]);
 	}
 	auto neighbors = query_graph.GetNeighbors(new_set, exclusion_set);
-	auto all_neighbors = GetAllNeighborSets(exclusion_set, neighbors);
-	for (auto neighbor : all_neighbors) {
+	auto all_neighbors = GetAllNeighborSets(neighbors);
+	for (const auto &neighbor : all_neighbors) {
 		auto &neighbor_relation = set_manager.GetJoinRelation(neighbor);
 		auto &combined_set = set_manager.Union(new_set, neighbor_relation);
 
@@ -820,8 +850,9 @@ GenerateJoinRelation JoinOrderOptimizer::GenerateJoins(vector<unique_ptr<Logical
 		// FILTER on top of GET, add estimated properties to both
 		auto &filter_props = *result_operator->estimated_props;
 		auto &child_operator = *result_operator->children[0];
-		child_operator.estimated_props = make_uniq<EstimatedProperties>(
-		    filter_props.GetCardinality<double>() / CardinalityEstimator::DEFAULT_SELECTIVITY, filter_props.GetCost());
+		child_operator.estimated_props = make_uniq<EstimatedProperties>(filter_props.GetCardinality<double>() /
+		                                                                    CardinalityEstimator::DEFAULT_SELECTIVITY,
+		                                                                filter_props.GetCost<double>());
 		child_operator.estimated_cardinality = child_operator.estimated_props->GetCardinality<idx_t>();
 		child_operator.has_estimated_cardinality = true;
 	}

package/src/duckdb/src/optimizer/join_order/join_relation_set.cpp

@@ -65,7 +65,7 @@ JoinRelationSet &JoinRelationSetManager::GetJoinRelation(idx_t index) {
 	return GetJoinRelation(std::move(relations), count);
 }
 
-JoinRelationSet &JoinRelationSetManager::GetJoinRelation(unordered_set<idx_t> &bindings) {
+JoinRelationSet &JoinRelationSetManager::GetJoinRelation(const unordered_set<idx_t> &bindings) {
 	// create a sorted vector of the relations
 	unsafe_unique_array<idx_t> relations = bindings.empty() ? nullptr : make_unsafe_uniq_array<idx_t>(bindings.size());
 	idx_t count = 0;
@@ -94,16 +94,12 @@ JoinRelationSet &JoinRelationSetManager::Union(JoinRelationSet &left, JoinRelati
 				relations[count++] = left.relations[i];
 			}
 			break;
-		} else if (left.relations[i] == right.relations[j]) {
-			// equivalent, add only one of the two pairs
-			relations[count++] = left.relations[i];
-			i++;
-			j++;
 		} else if (left.relations[i] < right.relations[j]) {
 			// left is smaller, progress left and add it to the set
 			relations[count++] = left.relations[i];
 			i++;
 		} else {
+			D_ASSERT(left.relations[i] > right.relations[j]);
 			// right is smaller, progress right and add it to the set
 			relations[count++] = right.relations[j];
 			j++;

package/src/duckdb/src/optimizer/join_order/query_graph.cpp

@@ -76,22 +76,30 @@ void QueryGraph::CreateEdge(JoinRelationSet &left, JoinRelationSet &right, optio
 	info.neighbors.push_back(std::move(n));
 }
 
+void QueryGraph::EnumerateNeighborsDFS(JoinRelationSet &node, reference<QueryEdge> info, idx_t index,
+                                       const std::function<bool(NeighborInfo &)> &callback) {
+
+	for (auto &neighbor : info.get().neighbors) {
+		if (callback(*neighbor)) {
+			return;
+		}
+	}
+
+	for (idx_t node_index = index; node_index < node.count; ++node_index) {
+		auto iter = info.get().children.find(node.relations[node_index]);
+		if (iter != info.get().children.end()) {
+			reference<QueryEdge> new_info = *iter->second;
+			EnumerateNeighborsDFS(node, new_info, node_index + 1, callback);
+		}
+	}
+}
+
 void QueryGraph::EnumerateNeighbors(JoinRelationSet &node, const std::function<bool(NeighborInfo &)> &callback) {
 	for (idx_t j = 0; j < node.count; j++) {
-		reference<QueryEdge> info = root;
-		for (idx_t i = j; i < node.count; i++) {
-			auto entry = info.get().children.find(node.relations[i]);
-			if (entry == info.get().children.end()) {
-				// node not found
-				break;
-			}
-			// check if any subset of the other set is in this sets neighbors
-			info = *entry->second;
-			for (auto &neighbor : info.get().neighbors) {
-				if (callback(*neighbor)) {
-					return;
-				}
-			}
+		auto iter = root.children.find(node.relations[j]);
+		if (iter != root.children.end()) {
+			reference<QueryEdge> new_info = *iter->second;
+			EnumerateNeighborsDFS(node, new_info, j + 1, callback);
 		}
 	}
 }

package/src/duckdb/src/optimizer/optimizer.cpp

@@ -1,27 +1,28 @@
 #include "duckdb/optimizer/optimizer.hpp"
 
 #include "duckdb/execution/column_binding_resolver.hpp"
-#include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/main/query_profiler.hpp"
 #include "duckdb/optimizer/column_lifetime_optimizer.hpp"
 #include "duckdb/optimizer/common_aggregate_optimizer.hpp"
+#include "duckdb/optimizer/compressed_materialization.hpp"
 #include "duckdb/optimizer/cse_optimizer.hpp"
 #include "duckdb/optimizer/deliminator.hpp"
-#include "duckdb/optimizer/unnest_rewriter.hpp"
 #include "duckdb/optimizer/expression_heuristics.hpp"
 #include "duckdb/optimizer/filter_pullup.hpp"
 #include "duckdb/optimizer/filter_pushdown.hpp"
 #include "duckdb/optimizer/in_clause_rewriter.hpp"
 #include "duckdb/optimizer/join_order/join_order_optimizer.hpp"
 #include "duckdb/optimizer/regex_range_filter.hpp"
+#include "duckdb/optimizer/remove_duplicate_groups.hpp"
 #include "duckdb/optimizer/remove_unused_columns.hpp"
 #include "duckdb/optimizer/rule/equal_or_null_simplification.hpp"
 #include "duckdb/optimizer/rule/in_clause_simplification.hpp"
 #include "duckdb/optimizer/rule/list.hpp"
 #include "duckdb/optimizer/statistics_propagator.hpp"
 #include "duckdb/optimizer/topn_optimizer.hpp"
+#include "duckdb/optimizer/unnest_rewriter.hpp"
 #include "duckdb/planner/binder.hpp"
 #include "duckdb/planner/planner.hpp"
 
@@ -52,6 +53,10 @@ Optimizer::Optimizer(Binder &binder, ClientContext &context) : context(context),
 #endif
 }
 
+ClientContext &Optimizer::GetContext() {
+	return context;
+}
+
 void Optimizer::RunOptimizer(OptimizerType type, const std::function<void()> &callback) {
 	auto &config = DBConfig::GetConfig(context);
 	if (config.options.disabled_optimizers.find(type) != config.options.disabled_optimizers.end()) {
@@ -73,6 +78,16 @@ void Optimizer::Verify(LogicalOperator &op) {
 
 unique_ptr<LogicalOperator> Optimizer::Optimize(unique_ptr<LogicalOperator> plan_p) {
 	Verify(*plan_p);
+
+	switch (plan_p->type) {
+	case LogicalOperatorType::LOGICAL_TRANSACTION:
+	case LogicalOperatorType::LOGICAL_SET:
+	case LogicalOperatorType::LOGICAL_PRAGMA:
+		return plan_p;
+	default:
+		break;
+	}
+
 	this->plan = std::move(plan_p);
 	// first we perform expression rewrites using the ExpressionRewriter
 	// this does not change the logical plan structure, but only simplifies the expression trees
@@ -96,8 +111,14 @@ unique_ptr<LogicalOperator> Optimizer::Optimize(unique_ptr<LogicalOperator> plan
 	});
 
 	RunOptimizer(OptimizerType::IN_CLAUSE, [&]() {
-		InClauseRewriter rewriter(context, *this);
-		plan = rewriter.Rewrite(std::move(plan));
+		InClauseRewriter ic_rewriter(context, *this);
+		plan = ic_rewriter.Rewrite(std::move(plan));
+	});
+
+	// removes any redundant DelimGets/DelimJoins
+	RunOptimizer(OptimizerType::DELIMINATOR, [&]() {
+		Deliminator deliminator;
+		plan = deliminator.Optimize(std::move(plan));
 	});
 
 	// then we perform the join ordering optimization
@@ -107,12 +128,6 @@ unique_ptr<LogicalOperator> Optimizer::Optimize(unique_ptr<LogicalOperator> plan
 		plan = optimizer.Optimize(std::move(plan));
 	});
 
-	// removes any redundant DelimGets/DelimJoins
-	RunOptimizer(OptimizerType::DELIMINATOR, [&]() {
-		Deliminator deliminator(context);
-		plan = deliminator.Optimize(std::move(plan));
-	});
-
 	// rewrites UNNESTs in DelimJoins by moving them to the projection
 	RunOptimizer(OptimizerType::UNNEST_REWRITER, [&]() {
 		UnnestRewriter unnest_rewriter;
@@ -125,10 +140,10 @@ unique_ptr<LogicalOperator> Optimizer::Optimize(unique_ptr<LogicalOperator> plan
 		unused.VisitOperator(*plan);
 	});
 
-	// perform statistics propagation
-	RunOptimizer(OptimizerType::STATISTICS_PROPAGATION, [&]() {
-		StatisticsPropagator propagator(context);
-		propagator.PropagateStatistics(plan);
+	// Remove duplicate groups from aggregates
+	RunOptimizer(OptimizerType::DUPLICATE_GROUPS, [&]() {
+		RemoveDuplicateGroups remove;
+		remove.VisitOperator(*plan);
 	});
 
 	// then we extract common subexpressions inside the different operators
@@ -137,16 +152,38 @@ unique_ptr<LogicalOperator> Optimizer::Optimize(unique_ptr<LogicalOperator> plan
 		cse_optimizer.VisitOperator(*plan);
 	});
 
+	// creates projection maps so unused columns are projected out early
+	RunOptimizer(OptimizerType::COLUMN_LIFETIME, [&]() {
+		ColumnLifetimeAnalyzer column_lifetime(true);
+		column_lifetime.VisitOperator(*plan);
+	});
+
+	// perform statistics propagation
+	column_binding_map_t<unique_ptr<BaseStatistics>> statistics_map;
+	RunOptimizer(OptimizerType::STATISTICS_PROPAGATION, [&]() {
+		StatisticsPropagator propagator(*this);
+		propagator.PropagateStatistics(plan);
+		statistics_map = propagator.GetStatisticsMap();
+	});
+
+	// remove duplicate aggregates
 	RunOptimizer(OptimizerType::COMMON_AGGREGATE, [&]() {
 		CommonAggregateOptimizer common_aggregate;
 		common_aggregate.VisitOperator(*plan);
 	});
 
+	// creates projection maps so unused columns are projected out early
 	RunOptimizer(OptimizerType::COLUMN_LIFETIME, [&]() {
 		ColumnLifetimeAnalyzer column_lifetime(true);
 		column_lifetime.VisitOperator(*plan);
 	});
 
+	// compress data based on statistics for materializing operators
+	RunOptimizer(OptimizerType::COMPRESSED_MATERIALIZATION, [&]() {
+		CompressedMaterialization compressed_materialization(context, binder, std::move(statistics_map));
+		compressed_materialization.Compress(plan);
+	});
+
 	// transform ORDER BY + LIMIT to TopN
 	RunOptimizer(OptimizerType::TOP_N, [&]() {
 		TopN topn;

package/src/duckdb/src/optimizer/pushdown/pushdown_cross_product.cpp

@@ -42,13 +42,13 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownCrossProduct(unique_ptr<Logi
 		vector<JoinCondition> conditions;
 		vector<unique_ptr<Expression>> arbitrary_expressions;
 		auto join_type = JoinType::INNER;
-		LogicalComparisonJoin::ExtractJoinConditions(join_type, op->children[0], op->children[1], left_bindings,
-		                                             right_bindings, join_expressions, conditions,
+		LogicalComparisonJoin::ExtractJoinConditions(GetContext(), join_type, op->children[0], op->children[1],
+		                                             left_bindings, right_bindings, join_expressions, conditions,
 		                                             arbitrary_expressions);
 		// create the join from the join conditions
-		return LogicalComparisonJoin::CreateJoin(JoinType::INNER, JoinRefType::REGULAR, std::move(op->children[0]),
-		                                         std::move(op->children[1]), std::move(conditions),
-		                                         std::move(arbitrary_expressions));
+		return LogicalComparisonJoin::CreateJoin(GetContext(), JoinType::INNER, JoinRefType::REGULAR,
+		                                         std::move(op->children[0]), std::move(op->children[1]),
+		                                         std::move(conditions), std::move(arbitrary_expressions));
 	} else {
 		// no join conditions found: keep as cross product
 		return op;

package/src/duckdb/src/optimizer/pushdown/pushdown_get.cpp

@@ -4,7 +4,6 @@
 #include "duckdb/planner/expression/bound_parameter_expression.hpp"
 #include "duckdb/planner/operator/logical_filter.hpp"
 #include "duckdb/planner/operator/logical_get.hpp"
-#include "duckdb/storage/data_table.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/optimizer/remove_duplicate_groups.cpp

@@ -0,0 +1,127 @@
+#include "duckdb/optimizer/remove_duplicate_groups.hpp"
+
+#include "duckdb/common/pair.hpp"
+#include "duckdb/planner/expression/bound_columnref_expression.hpp"
+#include "duckdb/planner/operator/logical_aggregate.hpp"
+
+namespace duckdb {
+
+void RemoveDuplicateGroups::VisitOperator(LogicalOperator &op) {
+	switch (op.type) {
+	case LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY:
+		VisitAggregate(op.Cast<LogicalAggregate>());
+		break;
+	default:
+		break;
+	}
+	LogicalOperatorVisitor::VisitOperatorExpressions(op);
+	LogicalOperatorVisitor::VisitOperatorChildren(op);
+}
+
+void RemoveDuplicateGroups::VisitAggregate(LogicalAggregate &aggr) {
+	if (!aggr.grouping_functions.empty()) {
+		return;
+	}
+
+	auto &groups = aggr.groups;
+
+	column_binding_map_t<idx_t> duplicate_map;
+	vector<pair<idx_t, idx_t>> duplicates;
+	for (idx_t group_idx = 0; group_idx < groups.size(); group_idx++) {
+		const auto &group = groups[group_idx];
+		if (group->type != ExpressionType::BOUND_COLUMN_REF) {
+			continue;
+		}
+		const auto &colref = group->Cast<BoundColumnRefExpression>();
+		const auto &binding = colref.binding;
+		const auto it = duplicate_map.find(binding);
+		if (it == duplicate_map.end()) {
+			duplicate_map.emplace(binding, group_idx);
+		} else {
+			duplicates.emplace_back(it->second, group_idx);
+		}
+	}
+
+	if (duplicates.empty()) {
+		return;
+	}
+
+	// Sort duplicates by max duplicate group idx, because we want to remove groups from the back
+	sort(duplicates.begin(), duplicates.end(),
+	     [](const pair<idx_t, idx_t> &lhs, const pair<idx_t, idx_t> &rhs) { return lhs.second > rhs.second; });
+
+	// Now we want to remove the duplicates, but this alters the column bindings coming out of the aggregate,
+	// so we keep track of how they shift and do another round of column binding replacements
+	column_binding_map_t<ColumnBinding> group_binding_map;
+	for (idx_t group_idx = 0; group_idx < groups.size(); group_idx++) {
+		group_binding_map.emplace(ColumnBinding(aggr.group_index, group_idx),
+		                          ColumnBinding(aggr.group_index, group_idx));
+	}
+
+	for (idx_t duplicate_idx = 0; duplicate_idx < duplicates.size(); duplicate_idx++) {
+		const auto &duplicate = duplicates[duplicate_idx];
+		const auto &remaining_idx = duplicate.first;
+		const auto &removed_idx = duplicate.second;
+
+		// Store expression and remove it from groups
+		stored_expressions.emplace_back(std::move(groups[removed_idx]));
+		groups.erase(groups.begin() + removed_idx);
+
+		// This optimizer should run before statistics propagation, so this should be empty
+		// If it runs after, then group_stats should be updated too
+		D_ASSERT(aggr.group_stats.empty());
+
+		// Remove from grouping sets too
+		for (auto &grouping_set : aggr.grouping_sets) {
+			// Replace removed group with duplicate remaining group
+			if (grouping_set.erase(removed_idx) != 0) {
+				grouping_set.insert(remaining_idx);
+			}
+
+			// Indices shifted: Reinsert groups in the set with group_idx - 1
+			vector<idx_t> group_indices_to_reinsert;
+			for (auto &entry : grouping_set) {
+				if (entry > removed_idx) {
+					group_indices_to_reinsert.emplace_back(entry);
+				}
+			}
+			for (const auto group_idx : group_indices_to_reinsert) {
+				grouping_set.erase(group_idx);
+			}
+			for (const auto group_idx : group_indices_to_reinsert) {
+				grouping_set.insert(group_idx - 1);
+			}
+		}
+
+		// Update mapping
+		auto it = group_binding_map.find(ColumnBinding(aggr.group_index, removed_idx));
+		D_ASSERT(it != group_binding_map.end());
+		it->second.column_index = remaining_idx;
+
+		for (auto &map_entry : group_binding_map) {
+			auto &new_binding = map_entry.second;
+			if (new_binding.column_index > removed_idx) {
+				new_binding.column_index--;
+			}
+		}
+	}
+
+	// Replace all references to the old group binding with the new group binding
+	for (const auto &map_entry : group_binding_map) {
+		auto it = column_references.find(map_entry.first);
+		if (it != column_references.end()) {
+			for (auto expr : it->second) {
+				expr.get().binding = map_entry.second;
+			}
+		}
+	}
+}
+
+unique_ptr<Expression> RemoveDuplicateGroups::VisitReplace(BoundColumnRefExpression &expr,
+                                                           unique_ptr<Expression> *expr_ptr) {
+	// add a column reference
+	column_references[expr.binding].push_back(expr);
+	return nullptr;
+}
+
+} // namespace duckdb

package/src/duckdb/src/optimizer/remove_unused_columns.cpp

@@ -302,6 +302,10 @@ void RemoveUnusedColumns::VisitOperator(LogicalOperator &op) {
 		everything_referenced = true;
 		break;
 	}
+	case LogicalOperatorType::LOGICAL_MATERIALIZED_CTE: {
+		everything_referenced = true;
+		break;
+	}
 	case LogicalOperatorType::LOGICAL_CTE_REF: {
 		everything_referenced = true;
 		break;