duckdb 1.0.1-dev22.0 → 1.0.1-dev27.0

package/src/duckdb/src/optimizer/optimizer.cpp

@@ -4,24 +4,29 @@
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/main/query_profiler.hpp"
+#include "duckdb/optimizer/build_probe_side_optimizer.hpp"
 #include "duckdb/optimizer/column_lifetime_analyzer.hpp"
 #include "duckdb/optimizer/common_aggregate_optimizer.hpp"
 #include "duckdb/optimizer/cse_optimizer.hpp"
+#include "duckdb/optimizer/cte_filter_pusher.hpp"
 #include "duckdb/optimizer/deliminator.hpp"
 #include "duckdb/optimizer/expression_heuristics.hpp"
 #include "duckdb/optimizer/filter_pullup.hpp"
 #include "duckdb/optimizer/filter_pushdown.hpp"
 #include "duckdb/optimizer/in_clause_rewriter.hpp"
 #include "duckdb/optimizer/join_order/join_order_optimizer.hpp"
+#include "duckdb/optimizer/limit_pushdown.hpp"
 #include "duckdb/optimizer/regex_range_filter.hpp"
 #include "duckdb/optimizer/remove_duplicate_groups.hpp"
 #include "duckdb/optimizer/remove_unused_columns.hpp"
 #include "duckdb/optimizer/rule/equal_or_null_simplification.hpp"
 #include "duckdb/optimizer/rule/in_clause_simplification.hpp"
+#include "duckdb/optimizer/rule/join_dependent_filter.hpp"
 #include "duckdb/optimizer/rule/list.hpp"
 #include "duckdb/optimizer/statistics_propagator.hpp"
 #include "duckdb/optimizer/topn_optimizer.hpp"
 #include "duckdb/optimizer/unnest_rewriter.hpp"
+#include "duckdb/optimizer/join_filter_pushdown_optimizer.hpp"
 #include "duckdb/planner/binder.hpp"
 #include "duckdb/planner/planner.hpp"
 
@@ -43,6 +48,7 @@ Optimizer::Optimizer(Binder &binder, ClientContext &context) : context(context),
 	rewriter.rules.push_back(make_uniq<RegexOptimizationRule>(rewriter));
 	rewriter.rules.push_back(make_uniq<EmptyNeedleRemovalRule>(rewriter));
 	rewriter.rules.push_back(make_uniq<EnumComparisonRule>(rewriter));
+	rewriter.rules.push_back(make_uniq<JoinDependentFilterRule>(rewriter));
 	rewriter.rules.push_back(make_uniq<TimeStampComparison>(context, rewriter));
 
 #ifdef DEBUG
@@ -68,7 +74,7 @@ void Optimizer::RunOptimizer(OptimizerType type, const std::function<void()> &ca
 		return;
 	}
 	auto &profiler = QueryProfiler::Get(context);
-	profiler.StartPhase(OptimizerTypeToString(type));
+	profiler.StartPhase(MetricsUtils::GetOptimizerMetricByType(type));
 	callback();
 	profiler.EndPhase();
 	if (plan) {
@@ -80,17 +86,19 @@ void Optimizer::Verify(LogicalOperator &op) {
 	ColumnBindingResolver::Verify(op);
 }
 
-unique_ptr<LogicalOperator> Optimizer::Optimize(unique_ptr<LogicalOperator> plan_p) {
-	Verify(*plan_p);
-
-	switch (plan_p->type) {
+void Optimizer::RunBuiltInOptimizers() {
+	switch (plan->type) {
 	case LogicalOperatorType::LOGICAL_TRANSACTION:
-		return plan_p; // skip optimizing simple & often-occurring plans unaffected by rewrites
+	case LogicalOperatorType::LOGICAL_PRAGMA:
+	case LogicalOperatorType::LOGICAL_SET:
+	case LogicalOperatorType::LOGICAL_UPDATE_EXTENSIONS:
+	case LogicalOperatorType::LOGICAL_CREATE_SECRET:
+	case LogicalOperatorType::LOGICAL_EXTENSION_OPERATOR:
+		// skip optimizing simple & often-occurring plans unaffected by rewrites
+		return;
 	default:
 		break;
 	}
-
-	this->plan = std::move(plan_p);
 	// first we perform expression rewrites using the ExpressionRewriter
 	// this does not change the logical plan structure, but only simplifies the expression trees
 	RunOptimizer(OptimizerType::EXPRESSION_REWRITER, [&]() { rewriter.VisitOperator(*plan); });
@@ -104,9 +112,17 @@ unique_ptr<LogicalOperator> Optimizer::Optimize(unique_ptr<LogicalOperator> plan
 	// perform filter pushdown
 	RunOptimizer(OptimizerType::FILTER_PUSHDOWN, [&]() {
 		FilterPushdown filter_pushdown(*this);
+		unordered_set<idx_t> top_bindings;
+		filter_pushdown.CheckMarkToSemi(*plan, top_bindings);
 		plan = filter_pushdown.Rewrite(std::move(plan));
 	});
 
+	// derive and push filters into materialized CTEs
+	RunOptimizer(OptimizerType::CTE_FILTER_PUSHER, [&]() {
+		CTEFilterPusher cte_filter_pusher(*this);
+		plan = cte_filter_pusher.Optimize(std::move(plan));
+	});
+
 	RunOptimizer(OptimizerType::REGEX_RANGE, [&]() {
 		RegexRangeFilter regex_opt;
 		plan = regex_opt.Rewrite(std::move(plan));
@@ -154,6 +170,19 @@ unique_ptr<LogicalOperator> Optimizer::Optimize(unique_ptr<LogicalOperator> plan
 		cse_optimizer.VisitOperator(*plan);
 	});
 
+	// Once we know the column lifetime, we have more information regarding
+	// what relations should be the build side/probe side.
+	RunOptimizer(OptimizerType::BUILD_SIDE_PROBE_SIDE, [&]() {
+		BuildProbeSideOptimizer build_probe_side_optimizer(context, *plan);
+		build_probe_side_optimizer.VisitOperator(*plan);
+	});
+
+	// pushes LIMIT below PROJECTION
+	RunOptimizer(OptimizerType::LIMIT_PUSHDOWN, [&]() {
+		LimitPushdown limit_pushdown;
+		plan = limit_pushdown.Optimize(std::move(plan));
+	});
+
 	// transform ORDER BY + LIMIT to TopN
 	RunOptimizer(OptimizerType::TOP_N, [&]() {
 		TopN topn;
@@ -192,6 +221,20 @@ unique_ptr<LogicalOperator> Optimizer::Optimize(unique_ptr<LogicalOperator> plan
 		plan = expression_heuristics.Rewrite(std::move(plan));
 	});
 
+	// perform join filter pushdown after the dust has settled
+	RunOptimizer(OptimizerType::JOIN_FILTER_PUSHDOWN, [&]() {
+		JoinFilterPushdownOptimizer join_filter_pushdown(*this);
+		join_filter_pushdown.VisitOperator(*plan);
+	});
+}
+
+unique_ptr<LogicalOperator> Optimizer::Optimize(unique_ptr<LogicalOperator> plan_p) {
+	Verify(*plan_p);
+
+	this->plan = std::move(plan_p);
+
+	RunBuiltInOptimizers();
+
 	for (auto &optimizer_extension : DBConfig::GetConfig(context).optimizer_extensions) {
 		RunOptimizer(OptimizerType::EXTENSION, [&]() {
 			OptimizerExtensionInput input {GetContext(), *this, optimizer_extension.optimizer_info.get()};

package/src/duckdb/src/optimizer/pushdown/pushdown_aggregate.cpp

@@ -9,14 +9,6 @@ namespace duckdb {
 
 using Filter = FilterPushdown::Filter;
 
-static void ExtractFilterBindings(Expression &expr, vector<ColumnBinding> &bindings) {
-	if (expr.type == ExpressionType::BOUND_COLUMN_REF) {
-		auto &colref = expr.Cast<BoundColumnRefExpression>();
-		bindings.push_back(colref.binding);
-	}
-	ExpressionIterator::EnumerateChildren(expr, [&](Expression &child) { ExtractFilterBindings(child, bindings); });
-}
-
 static unique_ptr<Expression> ReplaceGroupBindings(LogicalAggregate &proj, unique_ptr<Expression> expr) {
 	if (expr->type == ExpressionType::BOUND_COLUMN_REF) {
 		auto &colref = expr->Cast<BoundColumnRefExpression>();
@@ -31,6 +23,14 @@ static unique_ptr<Expression> ReplaceGroupBindings(LogicalAggregate &proj, uniqu
 	return expr;
 }
 
+void FilterPushdown::ExtractFilterBindings(Expression &expr, vector<ColumnBinding> &bindings) {
+	if (expr.type == ExpressionType::BOUND_COLUMN_REF) {
+		auto &colref = expr.Cast<BoundColumnRefExpression>();
+		bindings.push_back(colref.binding);
+	}
+	ExpressionIterator::EnumerateChildren(expr, [&](Expression &child) { ExtractFilterBindings(child, bindings); });
+}
+
 unique_ptr<LogicalOperator> FilterPushdown::PushdownAggregate(unique_ptr<LogicalOperator> op) {
 	D_ASSERT(op->type == LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY);
 	auto &aggr = op->Cast<LogicalAggregate>();
@@ -50,21 +50,21 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownAggregate(unique_ptr<Logical
 		}
 		// no aggregate! we are filtering on a group
 		// we can only push this down if the filter is in all grouping sets
+		if (aggr.grouping_sets.empty()) {
+			// empty grouping set - we cannot pushdown the filter
+			continue;
+		}
+
 		vector<ColumnBinding> bindings;
 		ExtractFilterBindings(*f.filter, bindings);
+		if (bindings.empty()) {
+			// we can never push down empty grouping sets
+			continue;
+		}
 
 		bool can_pushdown_filter = true;
-		if (aggr.grouping_sets.empty()) {
-			// empty grouping set - we cannot pushdown the filter
-			can_pushdown_filter = false;
-		}
 		for (auto &grp : aggr.grouping_sets) {
 			// check for each of the grouping sets if they contain all groups
-			if (bindings.empty()) {
-				// we can never push down empty grouping sets
-				can_pushdown_filter = false;
-				break;
-			}
 			for (auto &binding : bindings) {
 				if (grp.find(binding.column_index) == grp.end()) {
 					can_pushdown_filter = false;

package/src/duckdb/src/optimizer/pushdown/pushdown_cross_product.cpp

@@ -34,7 +34,13 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownCrossProduct(unique_ptr<Logi
 				left_pushdown.filters.push_back(std::move(f));
 			} else if (side == JoinSide::RIGHT) {
 				// bindings match right side: push into right
-				right_pushdown.filters.push_back(std::move(f));
+				if (join_ref_type == JoinRefType::ASOF) {
+					// AsOf is really a table lookup, so we don't push filters
+					// down into the lookup (right) table
+					join_expressions.push_back(std::move(f->filter));
+				} else {
+					right_pushdown.filters.push_back(std::move(f));
+				}
 			} else {
 				D_ASSERT(side == JoinSide::BOTH || side == JoinSide::NONE);
 				// bindings match both: turn into join condition
@@ -56,9 +62,21 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownCrossProduct(unique_ptr<Logi
 		                                             op->children[1], left_bindings, right_bindings, join_expressions,
 		                                             conditions, arbitrary_expressions);
 		// create the join from the join conditions
-		return LogicalComparisonJoin::CreateJoin(GetContext(), join_type, join_ref_type, std::move(op->children[0]),
-		                                         std::move(op->children[1]), std::move(conditions),
-		                                         std::move(arbitrary_expressions));
+		auto new_op = LogicalComparisonJoin::CreateJoin(GetContext(), join_type, join_ref_type,
+		                                                std::move(op->children[0]), std::move(op->children[1]),
+		                                                std::move(conditions), std::move(arbitrary_expressions));
+
+		// possible cases are: AnyJoin, ComparisonJoin, or Filter + ComparisonJoin
+		if (op->has_estimated_cardinality) {
+			// set the estimated cardinality of the new operator
+			new_op->SetEstimatedCardinality(op->estimated_cardinality);
+			if (new_op->type == LogicalOperatorType::LOGICAL_FILTER) {
+				// if the new operators are Filter + ComparisonJoin, also set the estimated cardinality for the join
+				D_ASSERT(new_op->children[0]->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN);
+				new_op->children[0]->SetEstimatedCardinality(op->estimated_cardinality);
+			}
+		}
+		return new_op;
 	} else {
 		// no join conditions found: keep as cross product
 		D_ASSERT(op->type == LogicalOperatorType::LOGICAL_CROSS_PRODUCT);

package/src/duckdb/src/optimizer/pushdown/pushdown_get.cpp

@@ -6,7 +6,6 @@
 #include "duckdb/planner/operator/logical_get.hpp"
 
 namespace duckdb {
-
 unique_ptr<LogicalOperator> FilterPushdown::PushdownGet(unique_ptr<LogicalOperator> op) {
 	D_ASSERT(op->type == LogicalOperatorType::LOGICAL_GET);
 	auto &get = op->Cast<LogicalGet>();
@@ -53,23 +52,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownGet(unique_ptr<LogicalOperat
 
 	//! We generate the table filters that will be executed during the table scan
 	//! Right now this only executes simple AND filters
-	get.table_filters = combiner.GenerateTableScanFilters(get.column_ids);
-
-	// //! For more complex filters if all filters to a column are constants we generate a min max boundary used to
-	// check
-	// //! the zonemaps.
-	// auto zonemap_checks = combiner.GenerateZonemapChecks(get.column_ids, get.table_filters);
-
-	// for (auto &f : get.table_filters) {
-	// 	f.column_index = get.column_ids[f.column_index];
-	// }
-
-	// //! Use zonemap checks as table filters for pre-processing
-	// for (auto &zonemap_check : zonemap_checks) {
-	// 	if (zonemap_check.column_index != COLUMN_IDENTIFIER_ROW_ID) {
-	// 		get.table_filters.push_back(zonemap_check);
-	// 	}
-	// }
+	get.table_filters = combiner.GenerateTableScanFilters(get.GetColumnIds());
 
 	GenerateFilters();
 

package/src/duckdb/src/optimizer/pushdown/pushdown_inner_join.cpp

@@ -47,6 +47,12 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownInnerJoin(unique_ptr<Logical
 
 	// turn the inner join into a cross product
 	auto cross_product = make_uniq<LogicalCrossProduct>(std::move(op->children[0]), std::move(op->children[1]));
+
+	// preserve the estimated cardinality of the operator
+	if (op->has_estimated_cardinality) {
+		cross_product->SetEstimatedCardinality(op->estimated_cardinality);
+	}
+
 	// then push down cross product
 	return PushdownCrossProduct(std::move(cross_product));
 }

package/src/duckdb/src/optimizer/pushdown/pushdown_mark_join.cpp

@@ -9,6 +9,7 @@ using Filter = FilterPushdown::Filter;
 unique_ptr<LogicalOperator> FilterPushdown::PushdownMarkJoin(unique_ptr<LogicalOperator> op,
                                                              unordered_set<idx_t> &left_bindings,
                                                              unordered_set<idx_t> &right_bindings) {
+	auto op_bindings = op->GetColumnBindings();
 	auto &join = op->Cast<LogicalJoin>();
 	auto &comp_join = op->Cast<LogicalComparisonJoin>();
 	D_ASSERT(join.join_type == JoinType::MARK);
@@ -35,7 +36,8 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownMarkJoin(unique_ptr<LogicalO
 #endif
 			// this filter references the marker
 			// we can turn this into a SEMI join if the filter is on only the marker
-			if (filters[i]->filter->type == ExpressionType::BOUND_COLUMN_REF && convert_mark_joins) {
+			if (filters[i]->filter->type == ExpressionType::BOUND_COLUMN_REF && convert_mark_joins &&
+			    comp_join.convert_mark_to_semi) {
 				// filter just references the marker: turn into semi join
 #ifdef DEBUG
 				simplified_mark_join = true;
@@ -61,7 +63,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownMarkJoin(unique_ptr<LogicalO
 							break;
 						}
 					}
-					if (all_null_values_are_equal && convert_mark_joins) {
+					if (all_null_values_are_equal && convert_mark_joins && comp_join.convert_mark_to_semi) {
 #ifdef DEBUG
 						simplified_mark_join = true;
 #endif

package/src/duckdb/src/optimizer/pushdown/pushdown_window.cpp

@@ -0,0 +1,91 @@
+#include "duckdb/optimizer/filter_pushdown.hpp"
+#include "duckdb/parser/expression/window_expression.hpp"
+#include "duckdb/planner/expression/bound_columnref_expression.hpp"
+#include "duckdb/planner/expression/bound_function_expression.hpp"
+#include "duckdb/planner/expression/bound_window_expression.hpp"
+#include "duckdb/planner/expression_iterator.hpp"
+#include "duckdb/planner/operator/logical_window.hpp"
+
+namespace duckdb {
+
+using Filter = FilterPushdown::Filter;
+
+bool CanPushdownFilter(vector<column_binding_set_t> window_exprs_partition_bindings,
+                       const vector<ColumnBinding> &bindings) {
+	auto filter_on_all_partitions = true;
+	for (auto &partition_binding_set : window_exprs_partition_bindings) {
+		auto filter_on_binding_set = true;
+		for (auto &binding : bindings) {
+			if (partition_binding_set.find(binding) == partition_binding_set.end()) {
+				filter_on_binding_set = false;
+				break;
+			}
+		}
+		filter_on_all_partitions = filter_on_all_partitions && filter_on_binding_set;
+		if (!filter_on_all_partitions) {
+			break;
+		}
+	}
+	return filter_on_all_partitions;
+}
+
+unique_ptr<LogicalOperator> FilterPushdown::PushdownWindow(unique_ptr<LogicalOperator> op) {
+	D_ASSERT(op->type == LogicalOperatorType::LOGICAL_WINDOW);
+	auto &window = op->Cast<LogicalWindow>();
+	FilterPushdown pushdown(optimizer, convert_mark_joins);
+
+	// 1. Loop throguh the expressions, find the window expressions and investigate the partitions
+	// if a filter applies to a partition in each window expression then you can push the filter
+	// into the children.
+	vector<column_binding_set_t> window_exprs_partition_bindings;
+	for (auto &expr : window.expressions) {
+		if (expr->expression_class != ExpressionClass::BOUND_WINDOW) {
+			continue;
+		}
+		auto &window_expr = expr->Cast<BoundWindowExpression>();
+		auto &partitions = window_expr.partitions;
+		if (partitions.empty()) {
+			// If any window expression does not have partitions, we cannot push any filters.
+			// all window expressions need to be partitioned by the same column
+			// in order to push down the window.
+			return FinishPushdown(std::move(op));
+		}
+		column_binding_set_t partition_bindings;
+		// 2. Get the binding information of the partitions of the window expression
+		for (auto &partition_expr : partitions) {
+			switch (partition_expr->type) {
+			// TODO: Add expressions for function expressions like FLOOR, CEIL etc.
+			case ExpressionType::BOUND_COLUMN_REF: {
+				auto &partition_col = partition_expr->Cast<BoundColumnRefExpression>();
+				partition_bindings.insert(partition_col.binding);
+				break;
+			}
+			default:
+				break;
+			}
+		}
+		window_exprs_partition_bindings.push_back(partition_bindings);
+	}
+
+	if (window_exprs_partition_bindings.empty()) {
+		return FinishPushdown(std::move(op));
+	}
+
+	vector<unique_ptr<Filter>> leftover_filters;
+	// Loop through the filters. If a filter is on a partition in every window expression
+	// it can be pushed down.
+	for (idx_t i = 0; i < filters.size(); i++) {
+		// the filter must be on all partition bindings
+		vector<ColumnBinding> bindings;
+		ExtractFilterBindings(*filters.at(i)->filter, bindings);
+		if (CanPushdownFilter(window_exprs_partition_bindings, bindings)) {
+			pushdown.filters.push_back(std::move(filters.at(i)));
+		} else {
+			leftover_filters.push_back(std::move(filters.at(i)));
+		}
+	}
+	op->children[0] = pushdown.Rewrite(std::move(op->children[0]));
+	filters = std::move(leftover_filters);
+	return FinishPushdown(std::move(op));
+}
+} // namespace duckdb

package/src/duckdb/src/optimizer/remove_unused_columns.cpp

@@ -11,6 +11,7 @@
 #include "duckdb/planner/expression_iterator.hpp"
 #include "duckdb/planner/operator/logical_aggregate.hpp"
 #include "duckdb/planner/operator/logical_comparison_join.hpp"
+#include "duckdb/planner/operator/logical_distinct.hpp"
 #include "duckdb/planner/operator/logical_filter.hpp"
 #include "duckdb/planner/operator/logical_get.hpp"
 #include "duckdb/planner/operator/logical_order.hpp"
@@ -143,6 +144,9 @@ void RemoveUnusedColumns::VisitOperator(LogicalOperator &op) {
 					}
 					auto new_projection =
 					    make_uniq<LogicalProjection>(binder.GenerateTableIndex(), std::move(expressions));
+					if (child->has_estimated_cardinality) {
+						new_projection->SetEstimatedCardinality(child->estimated_cardinality);
+					}
 					new_projection->children.push_back(std::move(child));
 					op.children[child_idx] = std::move(new_projection);
 
@@ -158,30 +162,14 @@ void RemoveUnusedColumns::VisitOperator(LogicalOperator &op) {
 		return;
 	}
 	case LogicalOperatorType::LOGICAL_EXCEPT:
-	case LogicalOperatorType::LOGICAL_INTERSECT:
+	case LogicalOperatorType::LOGICAL_INTERSECT: {
 		// for INTERSECT/EXCEPT operations we can't remove anything, just recursively visit the children
 		for (auto &child : op.children) {
 			RemoveUnusedColumns remove(binder, context, true);
 			remove.VisitOperator(*child);
 		}
 		return;
-	case LogicalOperatorType::LOGICAL_ORDER_BY:
-		if (!everything_referenced) {
-			auto &order = op.Cast<LogicalOrder>();
-			D_ASSERT(order.projections.empty()); // should not yet be set
-			const auto all_bindings = order.GetColumnBindings();
-
-			for (idx_t col_idx = 0; col_idx < all_bindings.size(); col_idx++) {
-				if (column_references.find(all_bindings[col_idx]) != column_references.end()) {
-					order.projections.push_back(col_idx);
-				}
-			}
-		}
-		for (auto &child : op.children) {
-			RemoveUnusedColumns remove(binder, context, true);
-			remove.VisitOperator(*child);
-		}
-		return;
+	}
 	case LogicalOperatorType::LOGICAL_PROJECTION: {
 		if (!everything_referenced) {
 			auto &proj = op.Cast<LogicalProjection>();
@@ -221,9 +209,11 @@ void RemoveUnusedColumns::VisitOperator(LogicalOperator &op) {
 				return;
 			}
 
+			auto &final_column_ids = get.GetColumnIds();
+
 			// Create "selection vector" of all column ids
 			vector<idx_t> proj_sel;
-			for (idx_t col_idx = 0; col_idx < get.column_ids.size(); col_idx++) {
+			for (idx_t col_idx = 0; col_idx < final_column_ids.size(); col_idx++) {
 				proj_sel.push_back(col_idx);
 			}
 			// Create a copy that we can use to match ids later
@@ -235,8 +225,8 @@ void RemoveUnusedColumns::VisitOperator(LogicalOperator &op) {
 			// being projected out
 			for (auto &filter : get.table_filters.filters) {
 				optional_idx index;
-				for (idx_t i = 0; i < get.column_ids.size(); i++) {
-					if (get.column_ids[i] == filter.first) {
+				for (idx_t i = 0; i < final_column_ids.size(); i++) {
+					if (final_column_ids[i] == filter.first) {
 						index = i;
 						break;
 					}
@@ -257,9 +247,9 @@ void RemoveUnusedColumns::VisitOperator(LogicalOperator &op) {
 			vector<column_t> column_ids;
 			column_ids.reserve(col_sel.size());
 			for (auto col_sel_idx : col_sel) {
-				column_ids.push_back(get.column_ids[col_sel_idx]);
+				column_ids.push_back(final_column_ids[col_sel_idx]);
 			}
-			get.column_ids = std::move(column_ids);
+			get.SetColumnIds(std::move(column_ids));
 
 			if (get.function.filter_prune) {
 				// Now set the projection cols by matching the "selection vector" that excludes filter columns
@@ -275,11 +265,11 @@ void RemoveUnusedColumns::VisitOperator(LogicalOperator &op) {
 				}
 			}
 
-			if (get.column_ids.empty()) {
+			if (final_column_ids.empty()) {
 				// this generally means we are only interested in whether or not anything exists in the table (e.g.
 				// EXISTS(SELECT * FROM tbl)) in this case, we just scan the row identifier column as it means we do not
 				// need to read any of the columns
-				get.column_ids.push_back(COLUMN_IDENTIFIER_ROW_ID);
+				get.AddColumnId(COLUMN_IDENTIFIER_ROW_ID);
 			}
 		}
 		return;
@@ -293,6 +283,12 @@ void RemoveUnusedColumns::VisitOperator(LogicalOperator &op) {
 		break;
 	}
 	case LogicalOperatorType::LOGICAL_DISTINCT: {
+		auto &distinct = op.Cast<LogicalDistinct>();
+		if (distinct.distinct_type == DistinctType::DISTINCT_ON) {
+			// distinct type references columns that need to be distinct on, so no
+			// need to implicity reference everything.
+			break;
+		}
 		// distinct, all projected columns are used for the DISTINCT computation
 		// mark all columns as used and continue to the children
 		// FIXME: DISTINCT with expression list does not implicitly reference everything

package/src/duckdb/src/optimizer/rule/comparison_simplification.cpp

@@ -16,6 +16,7 @@ ComparisonSimplificationRule::ComparisonSimplificationRule(ExpressionRewriter &r
 
 unique_ptr<Expression> ComparisonSimplificationRule::Apply(LogicalOperator &op, vector<reference<Expression>> &bindings,
                                                            bool &changes_made, bool is_root) {
+
 	auto &expr = bindings[0].get().Cast<BoundComparisonExpression>();
 	auto &constant_expr = bindings[1].get();
 	bool column_ref_left = expr.left.get() != &constant_expr;

package/src/duckdb/src/optimizer/rule/empty_needle_removal.cpp

@@ -41,6 +41,9 @@ unique_ptr<Expression> EmptyNeedleRemovalRule::Apply(LogicalOperator &op, vector
 	}
 
 	D_ASSERT(prefix_value.type() == prefix_expr.return_type);
+	if (prefix_value.type().InternalType() != PhysicalType::VARCHAR) {
+		return nullptr;
+	}
 	auto &needle_string = StringValue::Get(prefix_value);
 
 	// PREFIX('xyz', '') is TRUE

package/src/duckdb/src/optimizer/rule/equal_or_null_simplification.cpp

@@ -18,7 +18,7 @@ EqualOrNullSimplification::EqualOrNullSimplification(ExpressionRewriter &rewrite
 	equal_child->policy = SetMatcher::Policy::SOME;
 	op->matchers.push_back(std::move(equal_child));
 
-	// AND conjuction on the other
+	// AND conjunction on the other
 	auto and_child = make_uniq<ConjunctionExpressionMatcher>();
 	and_child->expr_type = make_uniq<SpecificExpressionTypeMatcher>(ExpressionType::CONJUNCTION_AND);
 	and_child->policy = SetMatcher::Policy::SOME;
@@ -49,7 +49,7 @@ static unique_ptr<Expression> TryRewriteEqualOrIsNull(Expression &equal_expr, Ex
 		return nullptr;
 	}
 
-	// Make sure on the AND conjuction the relevant conditions appear
+	// Make sure on the AND conjunction the relevant conditions appear
 	auto &a_exp = *equal_cast.left;
 	auto &b_exp = *equal_cast.right;
 	bool a_is_null_found = false;

package/src/duckdb/src/optimizer/rule/in_clause_simplification_rule.cpp

@@ -23,9 +23,12 @@ unique_ptr<Expression> InClauseSimplificationRule::Apply(LogicalOperator &op, ve
 	if (cast_expression.child->expression_class != ExpressionClass::BOUND_COLUMN_REF) {
 		return nullptr;
 	}
-	//! Here we check if we can apply the expression on the constant side
+	//! The goal here is to remove the cast from the probe expression
+	//! and apply a cast to the constant expressions. We can only do this
+	//! if the semantics do not change, which only happens when BOTH casts
+	//! are invertible.
 	auto target_type = cast_expression.source_type();
-	if (!BoundCastExpression::CastIsInvertible(cast_expression.return_type, target_type)) {
+	if (!BoundCastExpression::CastIsInvertible(target_type, cast_expression.return_type)) {
 		return nullptr;
 	}
 	vector<unique_ptr<BoundConstantExpression>> cast_list;
@@ -36,6 +39,9 @@ unique_ptr<Expression> InClauseSimplificationRule::Apply(LogicalOperator &op, ve
 		}
 		D_ASSERT(expr.children[i]->IsFoldable());
 		auto constant_value = ExpressionExecutor::EvaluateScalar(GetContext(), *expr.children[i]);
+		if (!BoundCastExpression::CastIsInvertible(constant_value.type(), target_type)) {
+			return nullptr;
+		}
 		auto new_constant = constant_value.DefaultTryCastAs(target_type);
 		if (!new_constant) {
 			return nullptr;