duckdb 0.7.2-dev0.0 → 0.7.2-dev1138.0

package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp

@@ -7,6 +7,7 @@
 #include "duckdb/planner/operator/list.hpp"
 
 #include <algorithm>
+#include <cmath>
 
 namespace std {
 
@@ -113,7 +114,7 @@ bool JoinOrderOptimizer::ExtractJoinRelations(LogicalOperator &input_op, vector<
 	bool non_reorderable_operation = false;
 	if (op->type == LogicalOperatorType::LOGICAL_UNION || op->type == LogicalOperatorType::LOGICAL_EXCEPT ||
 	    op->type == LogicalOperatorType::LOGICAL_INTERSECT || op->type == LogicalOperatorType::LOGICAL_DELIM_JOIN ||
-	    op->type == LogicalOperatorType::LOGICAL_ANY_JOIN) {
+	    op->type == LogicalOperatorType::LOGICAL_ANY_JOIN || op->type == LogicalOperatorType::LOGICAL_ASOF_JOIN) {
 		// set operation, optimize separately in children
 		non_reorderable_operation = true;
 	}
@@ -138,7 +139,7 @@ bool JoinOrderOptimizer::ExtractJoinRelations(LogicalOperator &input_op, vector<
 					std::swap(join.children[0], join.children[1]);
 					for (auto &cond : join.conditions) {
 						std::swap(cond.left, cond.right);
-						cond.comparison = FlipComparisionExpression(cond.comparison);
+						cond.comparison = FlipComparisonExpression(cond.comparison);
 					}
 				}
 			}
@@ -152,11 +153,11 @@ bool JoinOrderOptimizer::ExtractJoinRelations(LogicalOperator &input_op, vector<
 		// new NULL values in the right side, so pushing this condition through the join leads to incorrect results
 		// for this reason, we just start a new JoinOptimizer pass in each of the children of the join
 
-		// Keep track of all of the filter bindings the new join order optimizer makes
+		// Keep track of all filter bindings the new join order optimizer makes
 		vector<column_binding_map_t<ColumnBinding>> child_binding_maps;
 		idx_t child_bindings_it = 0;
 		for (auto &child : op->children) {
-			child_binding_maps.emplace_back(column_binding_map_t<ColumnBinding>());
+			child_binding_maps.emplace_back();
 			JoinOrderOptimizer optimizer(context);
 			child = optimizer.Optimize(std::move(child));
 			// save the relation bindings from the optimized child. These later all get added to the
@@ -184,6 +185,7 @@ bool JoinOrderOptimizer::ExtractJoinRelations(LogicalOperator &input_op, vector<
 	}
 
 	switch (op->type) {
+	case LogicalOperatorType::LOGICAL_ASOF_JOIN:
 	case LogicalOperatorType::LOGICAL_COMPARISON_JOIN:
 	case LogicalOperatorType::LOGICAL_CROSS_PRODUCT: {
 		// inner join or cross product
@@ -222,12 +224,25 @@ bool JoinOrderOptimizer::ExtractJoinRelations(LogicalOperator &input_op, vector<
 	}
 	case LogicalOperatorType::LOGICAL_PROJECTION: {
 		auto proj = (LogicalProjection *)op;
-		// we run the join order optimizer witin the subquery as well
+		// we run the join order optimizer within the subquery as well
 		JoinOrderOptimizer optimizer(context);
 		op->children[0] = optimizer.Optimize(std::move(op->children[0]));
 		// projection, add to the set of relations
 		auto relation = make_unique<SingleJoinRelation>(&input_op, parent);
-		relation_mapping[proj->table_index] = relations.size();
+		auto relation_id = relations.size();
+		// push one child column binding map back.
+		vector<column_binding_map_t<ColumnBinding>> child_binding_maps;
+		child_binding_maps.emplace_back();
+		optimizer.cardinality_estimator.CopyRelationMap(child_binding_maps.at(0));
+		// This logical projection may sit on top of a logical comparison join that has been pushed down
+		// we want to copy the binding info of both tables
+		relation_mapping[proj->table_index] = relation_id;
+		for (auto &binding_info : child_binding_maps.at(0)) {
+			cardinality_estimator.AddRelationToColumnMapping(
+			    ColumnBinding(proj->table_index, binding_info.first.column_index), binding_info.second);
+			cardinality_estimator.AddColumnToRelationMap(binding_info.second.table_index,
+			                                             binding_info.second.column_index);
+		}
 		relations.push_back(std::move(relation));
 		return true;
 	}
@@ -756,7 +771,7 @@ JoinOrderOptimizer::GenerateJoins(vector<unique_ptr<LogicalOperator>> &extracted
 
 				if (invert) {
 					// reverse comparison expression if we reverse the order of the children
-					cond.comparison = FlipComparisionExpression(cond.comparison);
+					cond.comparison = FlipComparisonExpression(cond.comparison);
 				}
 				join->conditions.push_back(std::move(cond));
 			}
@@ -833,7 +848,7 @@ JoinOrderOptimizer::GenerateJoins(vector<unique_ptr<LogicalOperator>> &extracted
 				cond.comparison = comparison.type;
 				if (invert) {
 					// reverse comparison expression if we reverse the order of the children
-					cond.comparison = FlipComparisionExpression(comparison.type);
+					cond.comparison = FlipComparisonExpression(comparison.type);
 				}
 				// now find the join to push it into
 				auto node = result_operator.get();
@@ -853,7 +868,8 @@ JoinOrderOptimizer::GenerateJoins(vector<unique_ptr<LogicalOperator>> &extracted
 						result_operator->children[0] = std::move(comp_join);
 					}
 				} else {
-					D_ASSERT(node->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN);
+					D_ASSERT(node->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN ||
+					         node->type == LogicalOperatorType::LOGICAL_ASOF_JOIN);
 					auto &comp_join = (LogicalComparisonJoin &)*node;
 					comp_join.conditions.push_back(std::move(cond));
 				}
@@ -869,9 +885,11 @@ unique_ptr<LogicalOperator> JoinOrderOptimizer::RewritePlan(unique_ptr<LogicalOp
 
 	// first we will extract all relations from the main plan
 	vector<unique_ptr<LogicalOperator>> extracted_relations;
+	extracted_relations.reserve(relations.size());
 	for (auto &relation : relations) {
 		extracted_relations.push_back(ExtractJoinRelation(*relation));
 	}
+
 	// now we generate the actual joins
 	auto join_tree = GenerateJoins(extracted_relations, node);
 	// perform the final pushdown of remaining filters
@@ -893,7 +911,8 @@ unique_ptr<LogicalOperator> JoinOrderOptimizer::RewritePlan(unique_ptr<LogicalOp
 	auto op = plan.get();
 	auto parent = plan.get();
 	while (op->type != LogicalOperatorType::LOGICAL_CROSS_PRODUCT &&
-	       op->type != LogicalOperatorType::LOGICAL_COMPARISON_JOIN) {
+	       op->type != LogicalOperatorType::LOGICAL_COMPARISON_JOIN &&
+	       op->type != LogicalOperatorType::LOGICAL_ASOF_JOIN) {
 		D_ASSERT(op->children.size() == 1);
 		parent = op;
 		op = op->children[0].get();
@@ -928,7 +947,8 @@ unique_ptr<LogicalOperator> JoinOrderOptimizer::Optimize(unique_ptr<LogicalOpera
 	// filters in the process
 	expression_set_t filter_set;
 	for (auto &f_op : filter_operators) {
-		if (f_op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN) {
+		if (f_op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN ||
+		    f_op->type == LogicalOperatorType::LOGICAL_ASOF_JOIN) {
 			auto &join = (LogicalComparisonJoin &)*f_op;
 			D_ASSERT(join.join_type == JoinType::INNER);
 			D_ASSERT(join.expressions.empty());
@@ -999,7 +1019,7 @@ unique_ptr<LogicalOperator> JoinOrderOptimizer::Optimize(unique_ptr<LogicalOpera
 	for (idx_t i = 0; i < relations.size(); i++) {
 		auto &rel = *relations[i];
 		auto node = set_manager.GetJoinRelation(i);
-		nodes_ops.emplace_back(NodeOp(make_unique<JoinNode>(node, 0), rel.op));
+		nodes_ops.emplace_back(make_unique<JoinNode>(node, 0), rel.op);
 	}
 
 	cardinality_estimator.InitCardinalityEstimatorProps(&nodes_ops, &filter_infos);

package/src/duckdb/src/optimizer/optimizer.cpp

@@ -39,6 +39,7 @@ Optimizer::Optimizer(Binder &binder, ClientContext &context) : context(context),
 	rewriter.rules.push_back(make_unique<EqualOrNullSimplification>(rewriter));
 	rewriter.rules.push_back(make_unique<MoveConstantsRule>(rewriter));
 	rewriter.rules.push_back(make_unique<LikeOptimizationRule>(rewriter));
+	rewriter.rules.push_back(make_unique<OrderedAggregateOptimizer>(rewriter));
 	rewriter.rules.push_back(make_unique<RegexOptimizationRule>(rewriter));
 	rewriter.rules.push_back(make_unique<EmptyNeedleRemovalRule>(rewriter));
 	rewriter.rules.push_back(make_unique<EnumComparisonRule>(rewriter));

package/src/duckdb/src/optimizer/pullup/pullup_from_left.cpp

@@ -6,8 +6,8 @@ namespace duckdb {
 
 unique_ptr<LogicalOperator> FilterPullup::PullupFromLeft(unique_ptr<LogicalOperator> op) {
 	D_ASSERT(op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN ||
-	         op->type == LogicalOperatorType::LOGICAL_ANY_JOIN || op->type == LogicalOperatorType::LOGICAL_EXCEPT ||
-	         op->type == LogicalOperatorType::LOGICAL_DELIM_JOIN);
+	         op->type == LogicalOperatorType::LOGICAL_ASOF_JOIN || op->type == LogicalOperatorType::LOGICAL_ANY_JOIN ||
+	         op->type == LogicalOperatorType::LOGICAL_EXCEPT || op->type == LogicalOperatorType::LOGICAL_DELIM_JOIN);
 
 	FilterPullup left_pullup(true, can_add_column);
 	FilterPullup right_pullup(false, can_add_column);

package/src/duckdb/src/optimizer/pushdown/pushdown_aggregate.cpp

@@ -9,6 +9,14 @@ namespace duckdb {
 
 using Filter = FilterPushdown::Filter;
 
+static void ExtractFilterBindings(Expression &expr, vector<ColumnBinding> &bindings) {
+	if (expr.type == ExpressionType::BOUND_COLUMN_REF) {
+		auto &colref = (BoundColumnRefExpression &)expr;
+		bindings.push_back(colref.binding);
+	}
+	ExpressionIterator::EnumerateChildren(expr, [&](Expression &child) { ExtractFilterBindings(child, bindings); });
+}
+
 static unique_ptr<Expression> ReplaceGroupBindings(LogicalAggregate &proj, unique_ptr<Expression> expr) {
 	if (expr->type == ExpressionType::BOUND_COLUMN_REF) {
 		auto &colref = (BoundColumnRefExpression &)*expr;
@@ -40,14 +48,34 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownAggregate(unique_ptr<Logical
 			// filter on GROUPINGS function: cannot pushdown
 			continue;
 		}
-		// if there are any empty grouping sets, we cannot push down filters
-		bool has_empty_grouping_sets = false;
+		// no aggregate! we are filtering on a group
+		// we can only push this down if the filter is in all grouping sets
+		vector<ColumnBinding> bindings;
+		ExtractFilterBindings(*f.filter, bindings);
+
+		bool can_pushdown_filter = true;
+		if (aggr.grouping_sets.empty()) {
+			// empty grouping set - we cannot pushdown the filter
+			can_pushdown_filter = false;
+		}
 		for (auto &grp : aggr.grouping_sets) {
-			if (grp.empty()) {
-				has_empty_grouping_sets = true;
+			// check for each of the grouping sets if they contain all groups
+			if (bindings.empty()) {
+				// we can never push down empty grouping sets
+				can_pushdown_filter = false;
+				break;
+			}
+			for (auto &binding : bindings) {
+				if (grp.find(binding.column_index) == grp.end()) {
+					can_pushdown_filter = false;
+					break;
+				}
+			}
+			if (!can_pushdown_filter) {
+				break;
 			}
 		}
-		if (has_empty_grouping_sets) {
+		if (!can_pushdown_filter) {
 			continue;
 		}
 		// no aggregate! we can push this down

package/src/duckdb/src/optimizer/pushdown/pushdown_cross_product.cpp

@@ -46,7 +46,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownCrossProduct(unique_ptr<Logi
 		                                             right_bindings, join_expressions, conditions,
 		                                             arbitrary_expressions);
 		// create the join from the join conditions
-		return LogicalComparisonJoin::CreateJoin(JoinType::INNER, std::move(op->children[0]),
+		return LogicalComparisonJoin::CreateJoin(JoinType::INNER, JoinRefType::REGULAR, std::move(op->children[0]),
 		                                         std::move(op->children[1]), std::move(conditions),
 		                                         std::move(arbitrary_expressions));
 	} else {

package/src/duckdb/src/optimizer/pushdown/pushdown_inner_join.cpp

@@ -24,6 +24,9 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownInnerJoin(unique_ptr<Logical
 			// filter statically evaluates to false, strip tree
 			return make_unique<LogicalEmptyResult>(std::move(op));
 		}
+	} else if (op->type == LogicalOperatorType::LOGICAL_ASOF_JOIN) {
+		// Don't mess with non-standard condition interpretations
+		return FinishPushdown(std::move(op));
 	} else {
 		// comparison join
 		D_ASSERT(op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN);

package/src/duckdb/src/optimizer/pushdown/pushdown_left_join.cpp

@@ -68,7 +68,9 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownLeftJoin(unique_ptr<LogicalO
 	// for a comparison join we create a FilterCombiner that checks if we can push conditions on LHS join conditions
 	// into the RHS of the join
 	FilterCombiner filter_combiner(optimizer);
-	if (op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN) {
+	const auto isComparison = (op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN ||
+	                           op->type == LogicalOperatorType::LOGICAL_ASOF_JOIN);
+	if (isComparison) {
 		// add all comparison conditions
 		auto &comparison_join = (LogicalComparisonJoin &)*op;
 		for (auto &cond : comparison_join.conditions) {
@@ -82,7 +84,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownLeftJoin(unique_ptr<LogicalO
 		if (side == JoinSide::LEFT) {
 			// bindings match left side
 			// we can push the filter into the left side
-			if (op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN) {
+			if (isComparison) {
 				// we MIGHT be able to push it down the RHS as well, but only if it is a comparison that matches the
 				// join predicates we use the FilterCombiner to figure this out add the expression to the FilterCombiner
 				filter_combiner.AddFilter(filters[i]->filter->Copy());
@@ -122,16 +124,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownLeftJoin(unique_ptr<LogicalO
 	right_pushdown.GenerateFilters();
 	op->children[0] = left_pushdown.Rewrite(std::move(op->children[0]));
 	op->children[1] = right_pushdown.Rewrite(std::move(op->children[1]));
-	if (filters.empty()) {
-		// no filters to push
-		return op;
-	}
-	auto filter = make_unique<LogicalFilter>();
-	for (auto &f : filters) {
-		filter->expressions.push_back(std::move(f->filter));
-	}
-	filter->children.push_back(std::move(op));
-	return std::move(filter);
+	return PushFinalFilters(std::move(op));
 }
 
 } // namespace duckdb

package/src/duckdb/src/optimizer/pushdown/pushdown_mark_join.cpp

@@ -13,7 +13,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownMarkJoin(unique_ptr<LogicalO
 	auto &comp_join = (LogicalComparisonJoin &)*op;
 	D_ASSERT(join.join_type == JoinType::MARK);
 	D_ASSERT(op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN ||
-	         op->type == LogicalOperatorType::LOGICAL_DELIM_JOIN);
+	         op->type == LogicalOperatorType::LOGICAL_DELIM_JOIN || op->type == LogicalOperatorType::LOGICAL_ASOF_JOIN);
 
 	right_bindings.insert(comp_join.mark_index);
 	FilterPushdown left_pushdown(optimizer), right_pushdown(optimizer);
@@ -77,7 +77,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownMarkJoin(unique_ptr<LogicalO
 	}
 	op->children[0] = left_pushdown.Rewrite(std::move(op->children[0]));
 	op->children[1] = right_pushdown.Rewrite(std::move(op->children[1]));
-	return FinishPushdown(std::move(op));
+	return PushFinalFilters(std::move(op));
 }
 
 } // namespace duckdb

package/src/duckdb/src/optimizer/pushdown/pushdown_single_join.cpp

@@ -23,7 +23,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownSingleJoin(unique_ptr<Logica
 	}
 	op->children[0] = left_pushdown.Rewrite(std::move(op->children[0]));
 	op->children[1] = right_pushdown.Rewrite(std::move(op->children[1]));
-	return FinishPushdown(std::move(op));
+	return PushFinalFilters(std::move(op));
 }
 
 } // namespace duckdb

package/src/duckdb/src/optimizer/remove_unused_columns.cpp

@@ -71,6 +71,7 @@ void RemoveUnusedColumns::VisitOperator(LogicalOperator &op) {
 		remove.VisitOperator(*op.children[0]);
 		return;
 	}
+	case LogicalOperatorType::LOGICAL_ASOF_JOIN:
 	case LogicalOperatorType::LOGICAL_DELIM_JOIN:
 	case LogicalOperatorType::LOGICAL_COMPARISON_JOIN: {
 		if (!everything_referenced) {

package/src/duckdb/src/optimizer/rule/move_constants.cpp

@@ -73,7 +73,10 @@ unique_ptr<Expression> MoveConstantsRule::Apply(LogicalOperator &op, vector<Expr
 			}
 			auto result_value = Value::HUGEINT(outer_value);
 			if (!result_value.DefaultTryCastAs(constant_type)) {
-				// if the cast is not possible then the comparison is not possible
+				// if the cast is not possible then an equality comparison is not possible
+				if (comparison->type != ExpressionType::COMPARE_EQUAL) {
+					return nullptr;
+				}
 				return ExpressionRewriter::ConstantOrNull(std::move(arithmetic->children[arithmetic_child_index]),
 				                                          Value::BOOLEAN(false));
 			}
@@ -86,14 +89,17 @@ unique_ptr<Expression> MoveConstantsRule::Apply(LogicalOperator &op, vector<Expr
 			}
 			auto result_value = Value::HUGEINT(inner_value);
 			if (!result_value.DefaultTryCastAs(constant_type)) {
-				// if the cast is not possible then the comparison is not possible
+				// if the cast is not possible then an equality comparison is not possible
+				if (comparison->type != ExpressionType::COMPARE_EQUAL) {
+					return nullptr;
+				}
 				return ExpressionRewriter::ConstantOrNull(std::move(arithmetic->children[arithmetic_child_index]),
 				                                          Value::BOOLEAN(false));
 			}
 			outer_constant->value = std::move(result_value);
 			// in this case, we should also flip the comparison
 			// e.g. if we have [4 - x < 2] then we should have [x > 2]
-			comparison->type = FlipComparisionExpression(comparison->type);
+			comparison->type = FlipComparisonExpression(comparison->type);
 		}
 	} else {
 		D_ASSERT(op_type == "*");
@@ -123,7 +129,7 @@ unique_ptr<Expression> MoveConstantsRule::Apply(LogicalOperator &op, vector<Expr
 		}
 		if (inner_value < 0) {
 			// multiply by negative value, need to flip expression
-			comparison->type = FlipComparisionExpression(comparison->type);
+			comparison->type = FlipComparisonExpression(comparison->type);
 		}
 		// else divide the RHS by the LHS
 		// we need to do a range check on the cast even though we do a division

package/src/duckdb/src/optimizer/rule/ordered_aggregate_optimizer.cpp

@@ -0,0 +1,30 @@
+#include "duckdb/optimizer/rule/ordered_aggregate_optimizer.hpp"
+
+#include "duckdb/optimizer/matcher/expression_matcher.hpp"
+#include "duckdb/planner/expression/bound_aggregate_expression.hpp"
+
+namespace duckdb {
+
+OrderedAggregateOptimizer::OrderedAggregateOptimizer(ExpressionRewriter &rewriter) : Rule(rewriter) {
+	// we match on an OR expression within a LogicalFilter node
+	root = make_unique<ExpressionMatcher>();
+	root->expr_class = ExpressionClass::BOUND_AGGREGATE;
+}
+
+unique_ptr<Expression> OrderedAggregateOptimizer::Apply(LogicalOperator &op, vector<Expression *> &bindings,
+                                                        bool &changes_made, bool is_root) {
+	auto aggr = (BoundAggregateExpression *)bindings[0];
+	if (!aggr->order_bys) {
+		// no ORDER BYs defined
+		return nullptr;
+	}
+	if (aggr->function.order_dependent == AggregateOrderDependent::NOT_ORDER_DEPENDENT) {
+		// not an order dependent aggregate but we have an ORDER BY clause - remove it
+		aggr->order_bys.reset();
+		changes_made = true;
+		return nullptr;
+	}
+	return nullptr;
+}
+
+} // namespace duckdb

package/src/duckdb/src/optimizer/rule/regex_optimizations.cpp

@@ -35,7 +35,7 @@ unique_ptr<Expression> RegexOptimizationRule::Apply(LogicalOperator &op, vector<
 
 	auto constant_value = ExpressionExecutor::EvaluateScalar(GetContext(), *constant_expr);
 	D_ASSERT(constant_value.type() == constant_expr->return_type);
-	auto &patt_str = StringValue::Get(constant_value);
+	auto patt_str = StringValue::Get(constant_value);
 
 	duckdb_re2::RE2 pattern(patt_str);
 	if (!pattern.ok()) {
@@ -47,7 +47,14 @@ unique_ptr<Expression> RegexOptimizationRule::Apply(LogicalOperator &op, vector<
 		auto contains = make_unique<BoundFunctionExpression>(root->return_type, ContainsFun::GetFunction(),
 		                                                     std::move(root->children), nullptr);
 
-		contains->children[1] = make_unique<BoundConstantExpression>(Value(patt_str));
+		string min;
+		string max;
+		pattern.PossibleMatchRange(&min, &max, patt_str.size());
+		if (min == max) {
+			contains->children[1] = make_unique<BoundConstantExpression>(Value(std::move(min)));
+		} else {
+			contains->children[1] = make_unique<BoundConstantExpression>(Value(std::move(patt_str)));
+		}
 		return std::move(contains);
 	}
 	return nullptr;

package/src/duckdb/src/optimizer/statistics/expression/propagate_aggregate.cpp

@@ -5,15 +5,21 @@ namespace duckdb {
 
 unique_ptr<BaseStatistics> StatisticsPropagator::PropagateExpression(BoundAggregateExpression &aggr,
                                                                      unique_ptr<Expression> *expr_ptr) {
-	vector<unique_ptr<BaseStatistics>> stats;
+	vector<BaseStatistics> stats;
 	stats.reserve(aggr.children.size());
 	for (auto &child : aggr.children) {
-		stats.push_back(PropagateExpression(child));
+		auto stat = PropagateExpression(child);
+		if (!stat) {
+			stats.push_back(BaseStatistics::CreateUnknown(child->return_type));
+		} else {
+			stats.push_back(stat->Copy());
+		}
 	}
 	if (!aggr.function.statistics) {
 		return nullptr;
 	}
-	return aggr.function.statistics(context, aggr, aggr.bind_info.get(), stats, node_stats.get());
+	AggregateStatisticsInput input(aggr.bind_info.get(), stats, node_stats.get());
+	return aggr.function.statistics(context, aggr, input);
 }
 
 } // namespace duckdb

package/src/duckdb/src/optimizer/statistics/expression/propagate_and_compress.cpp

@@ -5,7 +5,6 @@
 #include "duckdb/planner/expression/bound_constant_expression.hpp"
 #include "duckdb/planner/expression/bound_function_expression.hpp"
 #include "duckdb/storage/statistics/base_statistics.hpp"
-#include "duckdb/storage/statistics/numeric_statistics.hpp"
 #include "duckdb/common/operator/subtract.hpp"
 
 namespace duckdb {
@@ -44,14 +43,14 @@ bool GetCastType(hugeint_t range, LogicalType &cast_type) {
 }
 
 template <class T>
-unique_ptr<Expression> TemplatedCastToSmallestType(unique_ptr<Expression> expr, NumericStatistics &num_stats) {
+unique_ptr<Expression> TemplatedCastToSmallestType(unique_ptr<Expression> expr, BaseStatistics &stats) {
 	// Compute range
-	if (num_stats.min.IsNull() || num_stats.max.IsNull()) {
+	if (!NumericStats::HasMinMax(stats)) {
 		return expr;
 	}
 
-	auto signed_min_val = num_stats.min.GetValue<T>();
-	auto signed_max_val = num_stats.max.GetValue<T>();
+	auto signed_min_val = NumericStats::Min(stats).GetValue<T>();
+	auto signed_max_val = NumericStats::Max(stats).GetValue<T>();
 	if (signed_max_val < signed_min_val) {
 		return expr;
 	}
@@ -82,7 +81,7 @@ unique_ptr<Expression> TemplatedCastToSmallestType(unique_ptr<Expression> expr,
 	return BoundCastExpression::AddDefaultCastToType(std::move(minus_expr), cast_type);
 }
 
-unique_ptr<Expression> CastToSmallestType(unique_ptr<Expression> expr, NumericStatistics &num_stats) {
+unique_ptr<Expression> CastToSmallestType(unique_ptr<Expression> expr, BaseStatistics &num_stats) {
 	auto physical_type = expr->return_type.InternalType();
 	switch (physical_type) {
 	case PhysicalType::UINT8:
@@ -111,7 +110,7 @@ void StatisticsPropagator::PropagateAndCompress(unique_ptr<Expression> &expr, un
 	stats = PropagateExpression(expr);
 	if (stats) {
 		if (expr->return_type.IsIntegral()) {
-			expr = CastToSmallestType(std::move(expr), (NumericStatistics &)*stats);
+			expr = CastToSmallestType(std::move(expr), *stats);
 		}
 	}
 }

package/src/duckdb/src/optimizer/statistics/expression/propagate_cast.cpp

@@ -1,24 +1,27 @@
 #include "duckdb/optimizer/statistics_propagator.hpp"
 #include "duckdb/planner/expression/bound_cast_expression.hpp"
-#include "duckdb/storage/statistics/numeric_statistics.hpp"
 
 namespace duckdb {
 
-static unique_ptr<BaseStatistics> StatisticsOperationsNumericNumericCast(const BaseStatistics *input_p,
+static unique_ptr<BaseStatistics> StatisticsOperationsNumericNumericCast(const BaseStatistics &input,
                                                                          const LogicalType &target) {
-	auto &input = (NumericStatistics &)*input_p;
-
-	Value min = input.min, max = input.max;
+	if (!NumericStats::HasMinMax(input)) {
+		return nullptr;
+	}
+	Value min = NumericStats::Min(input);
+	Value max = NumericStats::Max(input);
 	if (!min.DefaultTryCastAs(target) || !max.DefaultTryCastAs(target)) {
 		// overflow in cast: bailout
 		return nullptr;
 	}
-	auto stats = make_unique<NumericStatistics>(target, std::move(min), std::move(max), input.stats_type);
-	stats->CopyBase(*input_p);
-	return std::move(stats);
+	auto result = NumericStats::CreateEmpty(target);
+	result.CopyBase(input);
+	NumericStats::SetMin(result, min);
+	NumericStats::SetMax(result, max);
+	return result.ToUnique();
 }
 
-static unique_ptr<BaseStatistics> StatisticsNumericCastSwitch(const BaseStatistics *input, const LogicalType &target) {
+static unique_ptr<BaseStatistics> StatisticsNumericCastSwitch(const BaseStatistics &input, const LogicalType &target) {
 	switch (target.InternalType()) {
 	case PhysicalType::INT8:
 	case PhysicalType::INT16:
@@ -48,13 +51,13 @@ unique_ptr<BaseStatistics> StatisticsPropagator::PropagateExpression(BoundCastEx
 	case PhysicalType::INT128:
 	case PhysicalType::FLOAT:
 	case PhysicalType::DOUBLE:
-		result_stats = StatisticsNumericCastSwitch(child_stats.get(), cast.return_type);
+		result_stats = StatisticsNumericCastSwitch(*child_stats, cast.return_type);
 		break;
 	default:
 		return nullptr;
 	}
 	if (cast.try_cast && result_stats) {
-		result_stats->validity_stats = make_unique<ValidityStatistics>(true, true);
+		result_stats->Set(StatsInfo::CAN_HAVE_NULL_VALUES);
 	}
 	return result_stats;
 }

package/src/duckdb/src/optimizer/statistics/expression/propagate_columnref.cpp

@@ -9,7 +9,7 @@ unique_ptr<BaseStatistics> StatisticsPropagator::PropagateExpression(BoundColumn
 	if (stats == statistics_map.end()) {
 		return nullptr;
 	}
-	return stats->second->Copy();
+	return stats->second->ToUnique();
 }
 
 } // namespace duckdb

package/src/duckdb/src/optimizer/statistics/expression/propagate_comparison.cpp

@@ -1,15 +1,14 @@
 #include "duckdb/optimizer/statistics_propagator.hpp"
 #include "duckdb/planner/expression/bound_comparison_expression.hpp"
 #include "duckdb/planner/expression/bound_constant_expression.hpp"
-#include "duckdb/storage/statistics/numeric_statistics.hpp"
 #include "duckdb/optimizer/expression_rewriter.hpp"
 
 namespace duckdb {
 
-FilterPropagateResult StatisticsPropagator::PropagateComparison(BaseStatistics &left, BaseStatistics &right,
+FilterPropagateResult StatisticsPropagator::PropagateComparison(BaseStatistics &lstats, BaseStatistics &rstats,
                                                                 ExpressionType comparison) {
 	// only handle numerics for now
-	switch (left.type.InternalType()) {
+	switch (lstats.GetType().InternalType()) {
 	case PhysicalType::BOOL:
 	case PhysicalType::UINT8:
 	case PhysicalType::UINT16:
@@ -26,9 +25,7 @@ FilterPropagateResult StatisticsPropagator::PropagateComparison(BaseStatistics &
 	default:
 		return FilterPropagateResult::NO_PRUNING_POSSIBLE;
 	}
-	auto &lstats = (NumericStatistics &)left;
-	auto &rstats = (NumericStatistics &)right;
-	if (lstats.min.IsNull() || lstats.max.IsNull() || rstats.min.IsNull() || rstats.max.IsNull()) {
+	if (!NumericStats::HasMinMax(lstats) || !NumericStats::HasMinMax(rstats)) {
 		// no stats available: nothing to prune
 		return FilterPropagateResult::NO_PRUNING_POSSIBLE;
 	}
@@ -38,52 +35,53 @@ FilterPropagateResult StatisticsPropagator::PropagateComparison(BaseStatistics &
 	switch (comparison) {
 	case ExpressionType::COMPARE_EQUAL:
 		// l = r, if l.min > r.max or r.min > l.max equality is not possible
-		if (lstats.min > rstats.max || rstats.min > lstats.max) {
+		if (NumericStats::Min(lstats) > NumericStats::Max(rstats) ||
+		    NumericStats::Min(rstats) > NumericStats::Max(lstats)) {
 			return has_null ? FilterPropagateResult::FILTER_FALSE_OR_NULL : FilterPropagateResult::FILTER_ALWAYS_FALSE;
 		} else {
 			return FilterPropagateResult::NO_PRUNING_POSSIBLE;
 		}
 	case ExpressionType::COMPARE_GREATERTHAN:
 		// l > r
-		if (lstats.min > rstats.max) {
+		if (NumericStats::Min(lstats) > NumericStats::Max(rstats)) {
 			// if l.min > r.max, it is always true ONLY if neither side contains nulls
 			return has_null ? FilterPropagateResult::FILTER_TRUE_OR_NULL : FilterPropagateResult::FILTER_ALWAYS_TRUE;
 		}
 		// if r.min is bigger or equal to l.max, the filter is always false
-		if (rstats.min >= lstats.max) {
+		if (NumericStats::Min(rstats) >= NumericStats::Max(lstats)) {
 			return has_null ? FilterPropagateResult::FILTER_FALSE_OR_NULL : FilterPropagateResult::FILTER_ALWAYS_FALSE;
 		}
 		return FilterPropagateResult::NO_PRUNING_POSSIBLE;
 	case ExpressionType::COMPARE_GREATERTHANOREQUALTO:
 		// l >= r
-		if (lstats.min >= rstats.max) {
+		if (NumericStats::Min(lstats) >= NumericStats::Max(rstats)) {
 			// if l.min >= r.max, it is always true ONLY if neither side contains nulls
 			return has_null ? FilterPropagateResult::FILTER_TRUE_OR_NULL : FilterPropagateResult::FILTER_ALWAYS_TRUE;
 		}
 		// if r.min > l.max, the filter is always false
-		if (rstats.min > lstats.max) {
+		if (NumericStats::Min(rstats) > NumericStats::Max(lstats)) {
 			return has_null ? FilterPropagateResult::FILTER_FALSE_OR_NULL : FilterPropagateResult::FILTER_ALWAYS_FALSE;
 		}
 		return FilterPropagateResult::NO_PRUNING_POSSIBLE;
 	case ExpressionType::COMPARE_LESSTHAN:
 		// l < r
-		if (lstats.max < rstats.min) {
+		if (NumericStats::Max(lstats) < NumericStats::Min(rstats)) {
 			// if l.max < r.min, it is always true ONLY if neither side contains nulls
 			return has_null ? FilterPropagateResult::FILTER_TRUE_OR_NULL : FilterPropagateResult::FILTER_ALWAYS_TRUE;
 		}
 		// if l.min >= rstats.max, the filter is always false
-		if (lstats.min >= rstats.max) {
+		if (NumericStats::Min(lstats) >= NumericStats::Max(rstats)) {
 			return has_null ? FilterPropagateResult::FILTER_FALSE_OR_NULL : FilterPropagateResult::FILTER_ALWAYS_FALSE;
 		}
 		return FilterPropagateResult::NO_PRUNING_POSSIBLE;
 	case ExpressionType::COMPARE_LESSTHANOREQUALTO:
 		// l <= r
-		if (lstats.max <= rstats.min) {
+		if (NumericStats::Max(lstats) <= NumericStats::Min(rstats)) {
 			// if l.max <= r.min, it is always true ONLY if neither side contains nulls
 			return has_null ? FilterPropagateResult::FILTER_TRUE_OR_NULL : FilterPropagateResult::FILTER_ALWAYS_TRUE;
 		}
 		// if l.min > rstats.max, the filter is always false
-		if (lstats.min > rstats.max) {
+		if (NumericStats::Min(lstats) > NumericStats::Max(rstats)) {
 			return has_null ? FilterPropagateResult::FILTER_FALSE_OR_NULL : FilterPropagateResult::FILTER_ALWAYS_FALSE;
 		}
 		return FilterPropagateResult::NO_PRUNING_POSSIBLE;

package/src/duckdb/src/optimizer/statistics/expression/propagate_conjunction.cpp

@@ -2,7 +2,6 @@
 #include "duckdb/optimizer/statistics_propagator.hpp"
 #include "duckdb/planner/expression/bound_conjunction_expression.hpp"
 #include "duckdb/planner/expression/bound_constant_expression.hpp"
-#include "duckdb/storage/statistics/numeric_statistics.hpp"
 #include "duckdb/optimizer/expression_rewriter.hpp"
 #include "duckdb/execution/expression_executor.hpp"