duckdb 0.7.2-dev0.0 → 0.7.2-dev1034.0

package/src/duckdb/src/optimizer/join_order/cardinality_estimator.cpp

@@ -6,24 +6,18 @@
 #include "duckdb/planner/operator/logical_comparison_join.hpp"
 #include "duckdb/planner/operator/logical_get.hpp"
 #include "duckdb/storage/data_table.hpp"
-#include "duckdb/storage/statistics/numeric_statistics.hpp"
 #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
 
 namespace duckdb {
 
 static TableCatalogEntry *GetCatalogTableEntry(LogicalOperator *op) {
-	if (op->type == LogicalOperatorType::LOGICAL_GET) {
-		auto get = (LogicalGet *)op;
-		TableCatalogEntry *entry = get->GetTable();
-		return entry;
+	if (!op) {
+		return nullptr;
 	}
-	for (auto &child : op->children) {
-		TableCatalogEntry *entry = GetCatalogTableEntry(child.get());
-		if (entry != nullptr) {
-			return entry;
-		}
-	}
-	return nullptr;
+	D_ASSERT(op->type == LogicalOperatorType::LOGICAL_GET);
+	auto get = (LogicalGet *)op;
+	TableCatalogEntry *entry = get->GetTable();
+	return entry;
 }
 
 // The filter was made on top of a logical sample or other projection,
@@ -45,8 +39,7 @@ void CardinalityEstimator::AddRelationTdom(FilterInfo *filter_info) {
 		}
 	}
 	auto key = ColumnBinding(filter_info->left_binding.table_index, filter_info->left_binding.column_index);
-	column_binding_set_t tmp({key});
-	relations_to_tdoms.emplace_back(RelationsToTDom(tmp));
+	relations_to_tdoms.emplace_back(column_binding_set_t({key}));
 }
 
 bool CardinalityEstimator::SingleColumnFilter(FilterInfo *filter_info) {
@@ -99,7 +92,7 @@ void CardinalityEstimator::AddToEquivalenceSets(FilterInfo *filter_info, vector<
 		column_binding_set_t tmp;
 		tmp.insert(filter_info->left_binding);
 		tmp.insert(filter_info->right_binding);
-		relations_to_tdoms.emplace_back(RelationsToTDom(tmp));
+		relations_to_tdoms.emplace_back(tmp);
 		relations_to_tdoms.back().filters.push_back(filter_info);
 	}
 }
@@ -265,7 +258,7 @@ double CardinalityEstimator::EstimateCardinalityWithSet(JoinRelationSet *new_set
 			// connection to any subgraph in subgraphs. Add a new subgraph, and maybe later there will be
 			// a connection.
 			if (!found_match) {
-				subgraphs.emplace_back(Subgraph2Denominator());
+				subgraphs.emplace_back();
 				auto subgraph = &subgraphs.back();
 				subgraph->relations.insert(filter->left_binding.table_index);
 				subgraph->relations.insert(filter->right_binding.table_index);
@@ -304,23 +297,37 @@ static bool IsLogicalFilter(LogicalOperator *op) {
 	return op->type == LogicalOperatorType::LOGICAL_FILTER;
 }
 
-static LogicalGet *GetLogicalGet(LogicalOperator *op) {
+static LogicalGet *GetLogicalGet(LogicalOperator *op, idx_t table_index = DConstants::INVALID_INDEX) {
 	LogicalGet *get = nullptr;
 	switch (op->type) {
 	case LogicalOperatorType::LOGICAL_GET:
 		get = (LogicalGet *)op;
 		break;
 	case LogicalOperatorType::LOGICAL_FILTER:
-		get = GetLogicalGet(op->children.at(0).get());
+		get = GetLogicalGet(op->children.at(0).get(), table_index);
 		break;
 	case LogicalOperatorType::LOGICAL_PROJECTION:
-		get = GetLogicalGet(op->children.at(0).get());
+		get = GetLogicalGet(op->children.at(0).get(), table_index);
 		break;
+	case LogicalOperatorType::LOGICAL_ASOF_JOIN:
 	case LogicalOperatorType::LOGICAL_COMPARISON_JOIN: {
 		LogicalComparisonJoin *join = (LogicalComparisonJoin *)op;
+		// We should never be calling GetLogicalGet without a valid table_index.
+		// We are attempting to get the catalog table for a relation (for statistics/cardinality estimation)
+		// A logical join means there is a non-reorderable relation in the join plan. This means we need
+		// to know the exact table index to return.
+		D_ASSERT(table_index != DConstants::INVALID_INDEX);
 		if (join->join_type == JoinType::MARK || join->join_type == JoinType::LEFT) {
 			auto child = join->children.at(0).get();
-			get = GetLogicalGet(child);
+			get = GetLogicalGet(child, table_index);
+			if (get && get->table_index == table_index) {
+				return get;
+			}
+			child = join->children.at(1).get();
+			get = GetLogicalGet(child, table_index);
+			if (get && get->table_index == table_index) {
+				return get;
+			}
 		}
 		break;
 	}
@@ -370,16 +377,20 @@ void CardinalityEstimator::InitCardinalityEstimatorProps(vector<NodeOp> *node_op
 		if (op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN) {
 			auto &join = (LogicalComparisonJoin &)*op;
 			if (join.join_type == JoinType::LEFT) {
-				// TODO: inspect child operators to get a more accurate cost
-				// and cardinality estimation. If an base op is a Logical Comparison join
-				// it is probably a left join, so cost of the larger table is a fine
-				// estimate
-				// No need to update a mark join cost because I say so.
+				// If a base op is a Logical Comparison join it is probably a left join,
+				// so the cost of the larger table is a fine estimate.
+				// TODO: provide better estimates for cost of mark joins
+				// MARK joins are used for anti and semi joins, so the cost can conceivably be
+				// less than the base table cardinality.
 				join_node->SetCost(join_node->GetBaseTableCardinality());
 			}
+		} else if (op->type == LogicalOperatorType::LOGICAL_ASOF_JOIN) {
+			// AsOf joins have the cardinality of the LHS
+			join_node->SetCost(join_node->GetBaseTableCardinality());
 		}
-		// update cardinality with filters
+		// Total domains can be affected by filters. So we update base table cardinality first
 		EstimateBaseTableCardinality(join_node, op);
+		// Then update total domains.
 		UpdateTotalDomains(join_node, op);
 	}
 
@@ -390,74 +401,84 @@ void CardinalityEstimator::InitCardinalityEstimatorProps(vector<NodeOp> *node_op
 void CardinalityEstimator::UpdateTotalDomains(JoinNode *node, LogicalOperator *op) {
 	auto relation_id = node->set->relations[0];
 	relation_attributes[relation_id].cardinality = node->GetCardinality<double>();
+	//! Initialize the distinct count for all columns used in joins with the current relation.
+	idx_t distinct_count = node->GetBaseTableCardinality();
 	TableCatalogEntry *catalog_table = nullptr;
-	auto get = GetLogicalGet(op);
-	if (get) {
-		catalog_table = GetCatalogTableEntry(get);
-	}
-
-	//! Initialize the tdoms for all columns the relation uses in join conditions.
-	unordered_set<idx_t>::iterator ite;
-	idx_t count = node->GetBaseTableCardinality();
 
 	bool direct_filter = false;
+	LogicalGet *get = nullptr;
+	bool get_updated = true;
 	for (auto &column : relation_attributes[relation_id].columns) {
-		//! for every column in the relation, get the count via either HLL, or assume it to be
+		//! for every column used in a filter in the relation, get the distinct count via HLL, or assume it to be
 		//! the cardinality
 		ColumnBinding key = ColumnBinding(relation_id, column);
+		auto actual_binding = relation_column_to_original_column.find(key);
+		// each relation has columns that are either projected or used as filters
+		// In order to get column statistics we need to make sure the actual binding still
+		// refers to the same base table relation, as non-reorderable joins may involve 2+
+		// base table relations and therefore the columns may also refer to 2 different
+		// base table relations
+		if (actual_binding != relation_column_to_original_column.end() &&
+		    (!get || get->table_index != actual_binding->second.table_index)) {
+			get = GetLogicalGet(op, actual_binding->second.table_index);
+			get_updated = true;
+		} else {
+			get_updated = false;
+		}
 
-		if (catalog_table) {
-			relation_attributes[relation_id].original_name = catalog_table->name;
-			// Get HLL stats here
-			auto actual_binding = relation_column_to_original_column[key];
+		if (get_updated) {
+			catalog_table = GetCatalogTableEntry(get);
+		}
 
-			auto base_stats = catalog_table->GetStatistics(context, actual_binding.column_index);
+		if (catalog_table && actual_binding != relation_column_to_original_column.end()) {
+			// Get HLL stats here
+			auto base_stats = catalog_table->GetStatistics(context, actual_binding->second.column_index);
 			if (base_stats) {
-				count = base_stats->GetDistinctCount();
+				distinct_count = base_stats->GetDistinctCount();
 			}
 
-			// means you have a direct filter on a column. The count/total domain for the column
+			// means you have a direct filter on a column. The distinct_count/total domain for the column
 			// should be decreased to match the predicted total domain matching the filter.
 			// We decrease the total domain for all columns in the equivalence set because filter pushdown
 			// will mean all columns are affected.
 			if (direct_filter) {
-				count = node->GetCardinality<idx_t>();
+				distinct_count = node->GetCardinality<idx_t>();
 			}
 
-			// HLL has estimation error, count can't be greater than cardinality of the table before filters
-			if (count > node->GetBaseTableCardinality()) {
-				count = node->GetBaseTableCardinality();
+			// HLL has estimation error, distinct_count can't be greater than cardinality of the table before filters
+			if (distinct_count > node->GetBaseTableCardinality()) {
+				distinct_count = node->GetBaseTableCardinality();
 			}
 		} else {
-			// No HLL. So if we know there is a direct filter, reduce count to cardinality with filter
-			// otherwise assume the total domain is still the cardinality
+			// No HLL. So if we know there is a direct filter, reduce the distinct count to the cardinality
+			// with filter effects. Otherwise assume the distinct count is still the cardinality
 			if (direct_filter) {
-				count = node->GetCardinality<idx_t>();
+				distinct_count = node->GetCardinality<idx_t>();
 			} else {
-				count = node->GetBaseTableCardinality();
+				distinct_count = node->GetBaseTableCardinality();
 			}
 		}
-
+		// Update the relation_to_tdom set with the estimated distinct count (or tdom) calculated above
 		for (auto &relation_to_tdom : relations_to_tdoms) {
 			column_binding_set_t i_set = relation_to_tdom.equivalent_relations;
 			if (i_set.count(key) != 1) {
 				continue;
 			}
 			if (catalog_table) {
-				if (relation_to_tdom.tdom_hll < count) {
-					relation_to_tdom.tdom_hll = count;
+				if (relation_to_tdom.tdom_hll < distinct_count) {
+					relation_to_tdom.tdom_hll = distinct_count;
 					relation_to_tdom.has_tdom_hll = true;
 				}
-				if (relation_to_tdom.tdom_no_hll > count) {
-					relation_to_tdom.tdom_no_hll = count;
+				if (relation_to_tdom.tdom_no_hll > distinct_count) {
+					relation_to_tdom.tdom_no_hll = distinct_count;
 				}
 			} else {
 				// Here we don't have catalog statistics, and the following is how we determine
 				// the tdom
 				// 1. If there is any hll data in the equivalence set, use that
 				// 2. Otherwise, use the table with the smallest cardinality
-				if (relation_to_tdom.tdom_no_hll > count && !relation_to_tdom.has_tdom_hll) {
-					relation_to_tdom.tdom_no_hll = count;
+				if (relation_to_tdom.tdom_no_hll > distinct_count && !relation_to_tdom.has_tdom_hll) {
+					relation_to_tdom.tdom_no_hll = distinct_count;
 				}
 			}
 			break;
@@ -465,9 +486,8 @@ void CardinalityEstimator::UpdateTotalDomains(JoinNode *node, LogicalOperator *o
 	}
 }
 
-TableFilterSet *CardinalityEstimator::GetTableFilters(LogicalOperator *op) {
-	// First check table filters
-	auto get = GetLogicalGet(op);
+TableFilterSet *CardinalityEstimator::GetTableFilters(LogicalOperator *op, idx_t table_index) {
+	auto get = GetLogicalGet(op, table_index);
 	return get ? &get->table_filters : nullptr;
 }
 
@@ -529,9 +549,10 @@ idx_t CardinalityEstimator::InspectConjunctionOR(idx_t cardinality, idx_t column
 	return cardinality_after_filters;
 }
 
-idx_t CardinalityEstimator::InspectTableFilters(idx_t cardinality, LogicalOperator *op, TableFilterSet *table_filters) {
+idx_t CardinalityEstimator::InspectTableFilters(idx_t cardinality, LogicalOperator *op, TableFilterSet *table_filters,
+                                                idx_t table_index) {
 	idx_t cardinality_after_filters = cardinality;
-	auto get = GetLogicalGet(op);
+	auto get = GetLogicalGet(op, table_index);
 	unique_ptr<BaseStatistics> column_statistics;
 	for (auto &it : table_filters->filters) {
 		column_statistics = nullptr;
@@ -562,17 +583,30 @@ idx_t CardinalityEstimator::InspectTableFilters(idx_t cardinality, LogicalOperat
 
 void CardinalityEstimator::EstimateBaseTableCardinality(JoinNode *node, LogicalOperator *op) {
 	auto has_logical_filter = IsLogicalFilter(op);
-	auto table_filters = GetTableFilters(op);
+	D_ASSERT(node->set->count == 1);
+	auto relation_id = node->set->relations[0];
 
-	auto card_after_filters = node->GetBaseTableCardinality();
-	if (table_filters) {
-		double inspect_result = (double)InspectTableFilters(card_after_filters, op, table_filters);
-		card_after_filters = MinValue(inspect_result, (double)card_after_filters);
-	}
-	if (has_logical_filter) {
-		card_after_filters *= DEFAULT_SELECTIVITY;
+	double lowest_card_found = NumericLimits<double>::Maximum();
+	for (auto &column : relation_attributes[relation_id].columns) {
+		auto card_after_filters = node->GetBaseTableCardinality();
+		ColumnBinding key = ColumnBinding(relation_id, column);
+		TableFilterSet *table_filters = nullptr;
+		auto actual_binding = relation_column_to_original_column.find(key);
+		if (actual_binding != relation_column_to_original_column.end()) {
+			table_filters = GetTableFilters(op, actual_binding->second.table_index);
+		}
+
+		if (table_filters) {
+			double inspect_result =
+			    (double)InspectTableFilters(card_after_filters, op, table_filters, actual_binding->second.table_index);
+			card_after_filters = MinValue(inspect_result, (double)card_after_filters);
+		}
+		if (has_logical_filter) {
+			card_after_filters *= DEFAULT_SELECTIVITY;
+		}
+		lowest_card_found = MinValue(card_after_filters, lowest_card_found);
 	}
-	node->SetEstimatedCardinality(card_after_filters);
+	node->SetEstimatedCardinality(lowest_card_found);
 }
 
 } // namespace duckdb

package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp

@@ -113,7 +113,7 @@ bool JoinOrderOptimizer::ExtractJoinRelations(LogicalOperator &input_op, vector<
 	bool non_reorderable_operation = false;
 	if (op->type == LogicalOperatorType::LOGICAL_UNION || op->type == LogicalOperatorType::LOGICAL_EXCEPT ||
 	    op->type == LogicalOperatorType::LOGICAL_INTERSECT || op->type == LogicalOperatorType::LOGICAL_DELIM_JOIN ||
-	    op->type == LogicalOperatorType::LOGICAL_ANY_JOIN) {
+	    op->type == LogicalOperatorType::LOGICAL_ANY_JOIN || op->type == LogicalOperatorType::LOGICAL_ASOF_JOIN) {
 		// set operation, optimize separately in children
 		non_reorderable_operation = true;
 	}
@@ -138,7 +138,7 @@ bool JoinOrderOptimizer::ExtractJoinRelations(LogicalOperator &input_op, vector<
 					std::swap(join.children[0], join.children[1]);
 					for (auto &cond : join.conditions) {
 						std::swap(cond.left, cond.right);
-						cond.comparison = FlipComparisionExpression(cond.comparison);
+						cond.comparison = FlipComparisonExpression(cond.comparison);
 					}
 				}
 			}
@@ -152,11 +152,11 @@ bool JoinOrderOptimizer::ExtractJoinRelations(LogicalOperator &input_op, vector<
 		// new NULL values in the right side, so pushing this condition through the join leads to incorrect results
 		// for this reason, we just start a new JoinOptimizer pass in each of the children of the join
 
-		// Keep track of all of the filter bindings the new join order optimizer makes
+		// Keep track of all filter bindings the new join order optimizer makes
 		vector<column_binding_map_t<ColumnBinding>> child_binding_maps;
 		idx_t child_bindings_it = 0;
 		for (auto &child : op->children) {
-			child_binding_maps.emplace_back(column_binding_map_t<ColumnBinding>());
+			child_binding_maps.emplace_back();
 			JoinOrderOptimizer optimizer(context);
 			child = optimizer.Optimize(std::move(child));
 			// save the relation bindings from the optimized child. These later all get added to the
@@ -184,6 +184,7 @@ bool JoinOrderOptimizer::ExtractJoinRelations(LogicalOperator &input_op, vector<
 	}
 
 	switch (op->type) {
+	case LogicalOperatorType::LOGICAL_ASOF_JOIN:
 	case LogicalOperatorType::LOGICAL_COMPARISON_JOIN:
 	case LogicalOperatorType::LOGICAL_CROSS_PRODUCT: {
 		// inner join or cross product
@@ -222,12 +223,25 @@ bool JoinOrderOptimizer::ExtractJoinRelations(LogicalOperator &input_op, vector<
 	}
 	case LogicalOperatorType::LOGICAL_PROJECTION: {
 		auto proj = (LogicalProjection *)op;
-		// we run the join order optimizer witin the subquery as well
+		// we run the join order optimizer within the subquery as well
 		JoinOrderOptimizer optimizer(context);
 		op->children[0] = optimizer.Optimize(std::move(op->children[0]));
 		// projection, add to the set of relations
 		auto relation = make_unique<SingleJoinRelation>(&input_op, parent);
-		relation_mapping[proj->table_index] = relations.size();
+		auto relation_id = relations.size();
+		// push one child column binding map back.
+		vector<column_binding_map_t<ColumnBinding>> child_binding_maps;
+		child_binding_maps.emplace_back();
+		optimizer.cardinality_estimator.CopyRelationMap(child_binding_maps.at(0));
+		// This logical projection may sit on top of a logical comparison join that has been pushed down
+		// we want to copy the binding info of both tables
+		relation_mapping[proj->table_index] = relation_id;
+		for (auto &binding_info : child_binding_maps.at(0)) {
+			cardinality_estimator.AddRelationToColumnMapping(
+			    ColumnBinding(proj->table_index, binding_info.first.column_index), binding_info.second);
+			cardinality_estimator.AddColumnToRelationMap(binding_info.second.table_index,
+			                                             binding_info.second.column_index);
+		}
 		relations.push_back(std::move(relation));
 		return true;
 	}
@@ -756,7 +770,7 @@ JoinOrderOptimizer::GenerateJoins(vector<unique_ptr<LogicalOperator>> &extracted
 
 				if (invert) {
 					// reverse comparison expression if we reverse the order of the children
-					cond.comparison = FlipComparisionExpression(cond.comparison);
+					cond.comparison = FlipComparisonExpression(cond.comparison);
 				}
 				join->conditions.push_back(std::move(cond));
 			}
@@ -833,7 +847,7 @@ JoinOrderOptimizer::GenerateJoins(vector<unique_ptr<LogicalOperator>> &extracted
 				cond.comparison = comparison.type;
 				if (invert) {
 					// reverse comparison expression if we reverse the order of the children
-					cond.comparison = FlipComparisionExpression(comparison.type);
+					cond.comparison = FlipComparisonExpression(comparison.type);
 				}
 				// now find the join to push it into
 				auto node = result_operator.get();
@@ -853,7 +867,8 @@ JoinOrderOptimizer::GenerateJoins(vector<unique_ptr<LogicalOperator>> &extracted
 						result_operator->children[0] = std::move(comp_join);
 					}
 				} else {
-					D_ASSERT(node->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN);
+					D_ASSERT(node->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN ||
+					         node->type == LogicalOperatorType::LOGICAL_ASOF_JOIN);
 					auto &comp_join = (LogicalComparisonJoin &)*node;
 					comp_join.conditions.push_back(std::move(cond));
 				}
@@ -869,9 +884,11 @@ unique_ptr<LogicalOperator> JoinOrderOptimizer::RewritePlan(unique_ptr<LogicalOp
 
 	// first we will extract all relations from the main plan
 	vector<unique_ptr<LogicalOperator>> extracted_relations;
+	extracted_relations.reserve(relations.size());
 	for (auto &relation : relations) {
 		extracted_relations.push_back(ExtractJoinRelation(*relation));
 	}
+
 	// now we generate the actual joins
 	auto join_tree = GenerateJoins(extracted_relations, node);
 	// perform the final pushdown of remaining filters
@@ -893,7 +910,8 @@ unique_ptr<LogicalOperator> JoinOrderOptimizer::RewritePlan(unique_ptr<LogicalOp
 	auto op = plan.get();
 	auto parent = plan.get();
 	while (op->type != LogicalOperatorType::LOGICAL_CROSS_PRODUCT &&
-	       op->type != LogicalOperatorType::LOGICAL_COMPARISON_JOIN) {
+	       op->type != LogicalOperatorType::LOGICAL_COMPARISON_JOIN &&
+	       op->type != LogicalOperatorType::LOGICAL_ASOF_JOIN) {
 		D_ASSERT(op->children.size() == 1);
 		parent = op;
 		op = op->children[0].get();
@@ -928,7 +946,8 @@ unique_ptr<LogicalOperator> JoinOrderOptimizer::Optimize(unique_ptr<LogicalOpera
 	// filters in the process
 	expression_set_t filter_set;
 	for (auto &f_op : filter_operators) {
-		if (f_op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN) {
+		if (f_op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN ||
+		    f_op->type == LogicalOperatorType::LOGICAL_ASOF_JOIN) {
 			auto &join = (LogicalComparisonJoin &)*f_op;
 			D_ASSERT(join.join_type == JoinType::INNER);
 			D_ASSERT(join.expressions.empty());
@@ -999,7 +1018,7 @@ unique_ptr<LogicalOperator> JoinOrderOptimizer::Optimize(unique_ptr<LogicalOpera
 	for (idx_t i = 0; i < relations.size(); i++) {
 		auto &rel = *relations[i];
 		auto node = set_manager.GetJoinRelation(i);
-		nodes_ops.emplace_back(NodeOp(make_unique<JoinNode>(node, 0), rel.op));
+		nodes_ops.emplace_back(make_unique<JoinNode>(node, 0), rel.op);
 	}
 
 	cardinality_estimator.InitCardinalityEstimatorProps(&nodes_ops, &filter_infos);

package/src/duckdb/src/optimizer/optimizer.cpp

@@ -39,6 +39,7 @@ Optimizer::Optimizer(Binder &binder, ClientContext &context) : context(context),
 	rewriter.rules.push_back(make_unique<EqualOrNullSimplification>(rewriter));
 	rewriter.rules.push_back(make_unique<MoveConstantsRule>(rewriter));
 	rewriter.rules.push_back(make_unique<LikeOptimizationRule>(rewriter));
+	rewriter.rules.push_back(make_unique<OrderedAggregateOptimizer>(rewriter));
 	rewriter.rules.push_back(make_unique<RegexOptimizationRule>(rewriter));
 	rewriter.rules.push_back(make_unique<EmptyNeedleRemovalRule>(rewriter));
 	rewriter.rules.push_back(make_unique<EnumComparisonRule>(rewriter));

package/src/duckdb/src/optimizer/pullup/pullup_from_left.cpp

@@ -6,8 +6,8 @@ namespace duckdb {
 
 unique_ptr<LogicalOperator> FilterPullup::PullupFromLeft(unique_ptr<LogicalOperator> op) {
 	D_ASSERT(op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN ||
-	         op->type == LogicalOperatorType::LOGICAL_ANY_JOIN || op->type == LogicalOperatorType::LOGICAL_EXCEPT ||
-	         op->type == LogicalOperatorType::LOGICAL_DELIM_JOIN);
+	         op->type == LogicalOperatorType::LOGICAL_ASOF_JOIN || op->type == LogicalOperatorType::LOGICAL_ANY_JOIN ||
+	         op->type == LogicalOperatorType::LOGICAL_EXCEPT || op->type == LogicalOperatorType::LOGICAL_DELIM_JOIN);
 
 	FilterPullup left_pullup(true, can_add_column);
 	FilterPullup right_pullup(false, can_add_column);

package/src/duckdb/src/optimizer/pushdown/pushdown_aggregate.cpp

@@ -9,6 +9,14 @@ namespace duckdb {
 
 using Filter = FilterPushdown::Filter;
 
+static void ExtractFilterBindings(Expression &expr, vector<ColumnBinding> &bindings) {
+	if (expr.type == ExpressionType::BOUND_COLUMN_REF) {
+		auto &colref = (BoundColumnRefExpression &)expr;
+		bindings.push_back(colref.binding);
+	}
+	ExpressionIterator::EnumerateChildren(expr, [&](Expression &child) { ExtractFilterBindings(child, bindings); });
+}
+
 static unique_ptr<Expression> ReplaceGroupBindings(LogicalAggregate &proj, unique_ptr<Expression> expr) {
 	if (expr->type == ExpressionType::BOUND_COLUMN_REF) {
 		auto &colref = (BoundColumnRefExpression &)*expr;
@@ -40,14 +48,34 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownAggregate(unique_ptr<Logical
 			// filter on GROUPINGS function: cannot pushdown
 			continue;
 		}
-		// if there are any empty grouping sets, we cannot push down filters
-		bool has_empty_grouping_sets = false;
+		// no aggregate! we are filtering on a group
+		// we can only push this down if the filter is in all grouping sets
+		vector<ColumnBinding> bindings;
+		ExtractFilterBindings(*f.filter, bindings);
+
+		bool can_pushdown_filter = true;
+		if (aggr.grouping_sets.empty()) {
+			// empty grouping set - we cannot pushdown the filter
+			can_pushdown_filter = false;
+		}
 		for (auto &grp : aggr.grouping_sets) {
-			if (grp.empty()) {
-				has_empty_grouping_sets = true;
+			// check for each of the grouping sets if they contain all groups
+			if (bindings.empty()) {
+				// we can never push down empty grouping sets
+				can_pushdown_filter = false;
+				break;
+			}
+			for (auto &binding : bindings) {
+				if (grp.find(binding.column_index) == grp.end()) {
+					can_pushdown_filter = false;
+					break;
+				}
+			}
+			if (!can_pushdown_filter) {
+				break;
 			}
 		}
-		if (has_empty_grouping_sets) {
+		if (!can_pushdown_filter) {
 			continue;
 		}
 		// no aggregate! we can push this down

package/src/duckdb/src/optimizer/pushdown/pushdown_cross_product.cpp

@@ -46,7 +46,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownCrossProduct(unique_ptr<Logi
 		                                             right_bindings, join_expressions, conditions,
 		                                             arbitrary_expressions);
 		// create the join from the join conditions
-		return LogicalComparisonJoin::CreateJoin(JoinType::INNER, std::move(op->children[0]),
+		return LogicalComparisonJoin::CreateJoin(JoinType::INNER, JoinRefType::REGULAR, std::move(op->children[0]),
 		                                         std::move(op->children[1]), std::move(conditions),
 		                                         std::move(arbitrary_expressions));
 	} else {

package/src/duckdb/src/optimizer/pushdown/pushdown_inner_join.cpp

@@ -24,6 +24,9 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownInnerJoin(unique_ptr<Logical
 			// filter statically evaluates to false, strip tree
 			return make_unique<LogicalEmptyResult>(std::move(op));
 		}
+	} else if (op->type == LogicalOperatorType::LOGICAL_ASOF_JOIN) {
+		// Don't mess with non-standard condition interpretations
+		return FinishPushdown(std::move(op));
 	} else {
 		// comparison join
 		D_ASSERT(op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN);

package/src/duckdb/src/optimizer/pushdown/pushdown_left_join.cpp

@@ -68,7 +68,9 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownLeftJoin(unique_ptr<LogicalO
 	// for a comparison join we create a FilterCombiner that checks if we can push conditions on LHS join conditions
 	// into the RHS of the join
 	FilterCombiner filter_combiner(optimizer);
-	if (op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN) {
+	const auto isComparison = (op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN ||
+	                           op->type == LogicalOperatorType::LOGICAL_ASOF_JOIN);
+	if (isComparison) {
 		// add all comparison conditions
 		auto &comparison_join = (LogicalComparisonJoin &)*op;
 		for (auto &cond : comparison_join.conditions) {
@@ -82,7 +84,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownLeftJoin(unique_ptr<LogicalO
 		if (side == JoinSide::LEFT) {
 			// bindings match left side
 			// we can push the filter into the left side
-			if (op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN) {
+			if (isComparison) {
 				// we MIGHT be able to push it down the RHS as well, but only if it is a comparison that matches the
 				// join predicates we use the FilterCombiner to figure this out add the expression to the FilterCombiner
 				filter_combiner.AddFilter(filters[i]->filter->Copy());
@@ -122,16 +124,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownLeftJoin(unique_ptr<LogicalO
 	right_pushdown.GenerateFilters();
 	op->children[0] = left_pushdown.Rewrite(std::move(op->children[0]));
 	op->children[1] = right_pushdown.Rewrite(std::move(op->children[1]));
-	if (filters.empty()) {
-		// no filters to push
-		return op;
-	}
-	auto filter = make_unique<LogicalFilter>();
-	for (auto &f : filters) {
-		filter->expressions.push_back(std::move(f->filter));
-	}
-	filter->children.push_back(std::move(op));
-	return std::move(filter);
+	return PushFinalFilters(std::move(op));
 }
 
 } // namespace duckdb

package/src/duckdb/src/optimizer/pushdown/pushdown_mark_join.cpp

@@ -13,7 +13,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownMarkJoin(unique_ptr<LogicalO
 	auto &comp_join = (LogicalComparisonJoin &)*op;
 	D_ASSERT(join.join_type == JoinType::MARK);
 	D_ASSERT(op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN ||
-	         op->type == LogicalOperatorType::LOGICAL_DELIM_JOIN);
+	         op->type == LogicalOperatorType::LOGICAL_DELIM_JOIN || op->type == LogicalOperatorType::LOGICAL_ASOF_JOIN);
 
 	right_bindings.insert(comp_join.mark_index);
 	FilterPushdown left_pushdown(optimizer), right_pushdown(optimizer);
@@ -77,7 +77,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownMarkJoin(unique_ptr<LogicalO
 	}
 	op->children[0] = left_pushdown.Rewrite(std::move(op->children[0]));
 	op->children[1] = right_pushdown.Rewrite(std::move(op->children[1]));
-	return FinishPushdown(std::move(op));
+	return PushFinalFilters(std::move(op));
 }
 
 } // namespace duckdb

package/src/duckdb/src/optimizer/pushdown/pushdown_single_join.cpp

@@ -23,7 +23,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownSingleJoin(unique_ptr<Logica
 	}
 	op->children[0] = left_pushdown.Rewrite(std::move(op->children[0]));
 	op->children[1] = right_pushdown.Rewrite(std::move(op->children[1]));
-	return FinishPushdown(std::move(op));
+	return PushFinalFilters(std::move(op));
 }
 
 } // namespace duckdb

package/src/duckdb/src/optimizer/remove_unused_columns.cpp

@@ -71,6 +71,7 @@ void RemoveUnusedColumns::VisitOperator(LogicalOperator &op) {
 		remove.VisitOperator(*op.children[0]);
 		return;
 	}
+	case LogicalOperatorType::LOGICAL_ASOF_JOIN:
 	case LogicalOperatorType::LOGICAL_DELIM_JOIN:
 	case LogicalOperatorType::LOGICAL_COMPARISON_JOIN: {
 		if (!everything_referenced) {

package/src/duckdb/src/optimizer/rule/move_constants.cpp

@@ -73,7 +73,10 @@ unique_ptr<Expression> MoveConstantsRule::Apply(LogicalOperator &op, vector<Expr
 			}
 			auto result_value = Value::HUGEINT(outer_value);
 			if (!result_value.DefaultTryCastAs(constant_type)) {
-				// if the cast is not possible then the comparison is not possible
+				// if the cast is not possible then an equality comparison is not possible
+				if (comparison->type != ExpressionType::COMPARE_EQUAL) {
+					return nullptr;
+				}
 				return ExpressionRewriter::ConstantOrNull(std::move(arithmetic->children[arithmetic_child_index]),
 				                                          Value::BOOLEAN(false));
 			}
@@ -86,14 +89,17 @@ unique_ptr<Expression> MoveConstantsRule::Apply(LogicalOperator &op, vector<Expr
 			}
 			auto result_value = Value::HUGEINT(inner_value);
 			if (!result_value.DefaultTryCastAs(constant_type)) {
-				// if the cast is not possible then the comparison is not possible
+				// if the cast is not possible then an equality comparison is not possible
+				if (comparison->type != ExpressionType::COMPARE_EQUAL) {
+					return nullptr;
+				}
 				return ExpressionRewriter::ConstantOrNull(std::move(arithmetic->children[arithmetic_child_index]),
 				                                          Value::BOOLEAN(false));
 			}
 			outer_constant->value = std::move(result_value);
 			// in this case, we should also flip the comparison
 			// e.g. if we have [4 - x < 2] then we should have [x > 2]
-			comparison->type = FlipComparisionExpression(comparison->type);
+			comparison->type = FlipComparisonExpression(comparison->type);
 		}
 	} else {
 		D_ASSERT(op_type == "*");
@@ -123,7 +129,7 @@ unique_ptr<Expression> MoveConstantsRule::Apply(LogicalOperator &op, vector<Expr
 		}
 		if (inner_value < 0) {
 			// multiply by negative value, need to flip expression
-			comparison->type = FlipComparisionExpression(comparison->type);
+			comparison->type = FlipComparisonExpression(comparison->type);
 		}
 		// else divide the RHS by the LHS
 		// we need to do a range check on the cast even though we do a division