duckdb 0.8.2-dev33.0 → 0.8.2-dev3334.0

package/src/duckdb/src/optimizer/join_order/relation_statistics_helper.cpp

@@ -0,0 +1,351 @@
+#include "duckdb/optimizer/join_order/relation_statistics_helper.hpp"
+#include "duckdb/planner/expression/list.hpp"
+#include "duckdb/planner/operator/list.hpp"
+#include "duckdb/planner/filter/conjunction_filter.hpp"
+#include "duckdb/planner/expression_iterator.hpp"
+#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
+#include "duckdb/function/table/table_scan.hpp"
+#include "duckdb/planner/operator/logical_get.hpp"
+#include "duckdb/storage/data_table.hpp"
+#include "duckdb/planner/filter/constant_filter.hpp"
+
+namespace duckdb {
+
+static ExpressionBinding GetChildColumnBinding(Expression &expr) {
+	auto ret = ExpressionBinding();
+	switch (expr.expression_class) {
+	case ExpressionClass::BOUND_FUNCTION: {
+		// TODO: Other expression classes that can have 0 children?
+		auto &func = expr.Cast<BoundFunctionExpression>();
+		// no children some sort of gen_random_uuid() or equivalent.
+		if (func.children.empty()) {
+			ret.found_expression = true;
+			ret.expression_is_constant = true;
+			return ret;
+		}
+		break;
+	}
+	case ExpressionClass::BOUND_COLUMN_REF: {
+		ret.found_expression = true;
+		auto &new_col_ref = expr.Cast<BoundColumnRefExpression>();
+		ret.child_binding = ColumnBinding(new_col_ref.binding.table_index, new_col_ref.binding.column_index);
+		return ret;
+	}
+	case ExpressionClass::BOUND_LAMBDA_REF:
+	case ExpressionClass::BOUND_CONSTANT:
+	case ExpressionClass::BOUND_DEFAULT:
+	case ExpressionClass::BOUND_PARAMETER:
+	case ExpressionClass::BOUND_REF:
+		ret.found_expression = true;
+		ret.expression_is_constant = true;
+		return ret;
+	default:
+		break;
+	}
+	ExpressionIterator::EnumerateChildren(expr, [&](unique_ptr<Expression> &child) {
+		auto recursive_result = GetChildColumnBinding(*child);
+		if (recursive_result.found_expression) {
+			ret = recursive_result;
+		}
+	});
+	// we didn't find a Bound Column Ref
+	return ret;
+}
+
+RelationStats RelationStatisticsHelper::ExtractGetStats(LogicalGet &get, ClientContext &context) {
+	auto return_stats = RelationStats();
+
+	auto base_table_cardinality = get.EstimateCardinality(context);
+	auto cardinality_after_filters = base_table_cardinality;
+	unique_ptr<BaseStatistics> column_statistics;
+
+	auto table_thing = get.GetTable();
+	auto name = string("some table");
+	if (table_thing) {
+		name = table_thing->name;
+		return_stats.table_name = name;
+	}
+
+	// if we can get the catalog table, then our column statistics will be accurate
+	// parquet readers etc. will still return statistics, but they initialize distinct column
+	// counts to 0.
+	// TODO: fix this, some file formats can encode distinct counts, we don't want to rely on
+	//  getting a catalog table to know that we can use statistics.
+	bool have_catalog_table_statistics = false;
+	if (get.GetTable()) {
+		have_catalog_table_statistics = true;
+	}
+
+	// first push back basic distinct counts for each column (if we have them).
+	for (idx_t i = 0; i < get.column_ids.size(); i++) {
+		bool have_distinct_count_stats = false;
+		if (get.function.statistics) {
+			column_statistics = get.function.statistics(context, get.bind_data.get(), get.column_ids[i]);
+			if (column_statistics && have_catalog_table_statistics) {
+				auto column_distinct_count = DistinctCount({column_statistics->GetDistinctCount(), true});
+				return_stats.column_distinct_count.push_back(column_distinct_count);
+				return_stats.column_names.push_back(name + "." + get.names.at(get.column_ids.at(i)));
+				have_distinct_count_stats = true;
+			}
+		}
+		if (!have_distinct_count_stats) {
+			// currently treating the cardinality as the distinct count.
+			// the cardinality estimator will update these distinct counts based
+			// on the extra columns that are joined on.
+			auto column_distinct_count = DistinctCount({cardinality_after_filters, false});
+			return_stats.column_distinct_count.push_back(column_distinct_count);
+			auto column_name = string("column");
+			if (get.column_ids.at(i) < get.names.size()) {
+				column_name = get.names.at(get.column_ids.at(i));
+			}
+			return_stats.column_names.push_back(get.GetName() + "." + column_name);
+		}
+	}
+
+	if (!get.table_filters.filters.empty()) {
+		column_statistics = nullptr;
+		for (auto &it : get.table_filters.filters) {
+			if (get.bind_data && get.function.name.compare("seq_scan") == 0) {
+				auto &table_scan_bind_data = get.bind_data->Cast<TableScanBindData>();
+				column_statistics = get.function.statistics(context, &table_scan_bind_data, it.first);
+			}
+
+			if (column_statistics && it.second->filter_type == TableFilterType::CONJUNCTION_AND) {
+				auto &filter = it.second->Cast<ConjunctionAndFilter>();
+				idx_t cardinality_with_and_filter = RelationStatisticsHelper::InspectConjunctionAND(
+				    base_table_cardinality, it.first, filter, *column_statistics);
+				cardinality_after_filters = MinValue(cardinality_after_filters, cardinality_with_and_filter);
+			}
+		}
+		// if the above code didn't find an equality filter (i.e country_code = "[us]")
+		// and there are other table filters (i.e cost > 50), use default selectivity.
+		bool has_equality_filter = (cardinality_after_filters != base_table_cardinality);
+		if (!has_equality_filter && !get.table_filters.filters.empty()) {
+			cardinality_after_filters =
+			    MaxValue<idx_t>(base_table_cardinality * RelationStatisticsHelper::DEFAULT_SELECTIVITY, 1);
+		}
+		if (base_table_cardinality == 0) {
+			cardinality_after_filters = 0;
+		}
+	}
+	return_stats.cardinality = cardinality_after_filters;
+	// update the estimated cardinality of the get as well.
+	// This is not updated during plan reconstruction.
+	get.estimated_cardinality = cardinality_after_filters;
+	get.has_estimated_cardinality = true;
+	D_ASSERT(base_table_cardinality >= cardinality_after_filters);
+	return_stats.stats_initialized = true;
+	return return_stats;
+}
+
+RelationStats RelationStatisticsHelper::ExtractDelimGetStats(LogicalDelimGet &delim_get, ClientContext &context) {
+	RelationStats stats;
+	stats.table_name = delim_get.GetName();
+	idx_t card = delim_get.EstimateCardinality(context);
+	stats.cardinality = card;
+	stats.stats_initialized = true;
+	for (auto &binding : delim_get.GetColumnBindings()) {
+		stats.column_distinct_count.push_back(DistinctCount({1, false}));
+		stats.column_names.push_back("column" + to_string(binding.column_index));
+	}
+	return stats;
+}
+
+RelationStats RelationStatisticsHelper::ExtractProjectionStats(LogicalProjection &proj, RelationStats &child_stats) {
+	auto proj_stats = RelationStats();
+	proj_stats.cardinality = child_stats.cardinality;
+	proj_stats.table_name = proj.GetName();
+	for (auto &expr : proj.expressions) {
+		proj_stats.column_names.push_back(expr->GetName());
+		auto res = GetChildColumnBinding(*expr);
+		D_ASSERT(res.found_expression);
+		if (res.expression_is_constant) {
+			proj_stats.column_distinct_count.push_back(DistinctCount({1, true}));
+		} else {
+			auto column_index = res.child_binding.column_index;
+			if (column_index >= child_stats.column_distinct_count.size() && expr->ToString() == "count_star()") {
+				// only one value for a count star
+				proj_stats.column_distinct_count.push_back(DistinctCount({1, true}));
+			} else {
+				// TODO: add this back in
+				//	D_ASSERT(column_index < stats.column_distinct_count.size());
+				if (column_index < child_stats.column_distinct_count.size()) {
+					proj_stats.column_distinct_count.push_back(child_stats.column_distinct_count.at(column_index));
+				} else {
+					proj_stats.column_distinct_count.push_back(DistinctCount({proj_stats.cardinality, false}));
+				}
+			}
+		}
+	}
+	proj_stats.stats_initialized = true;
+	return proj_stats;
+}
+
+RelationStats RelationStatisticsHelper::ExtractDummyScanStats(LogicalDummyScan &dummy_scan, ClientContext &context) {
+	auto stats = RelationStats();
+	idx_t card = dummy_scan.EstimateCardinality(context);
+	stats.cardinality = card;
+	for (idx_t i = 0; i < dummy_scan.GetColumnBindings().size(); i++) {
+		stats.column_distinct_count.push_back(DistinctCount({card, false}));
+		stats.column_names.push_back("dummy_scan_column");
+	}
+	stats.stats_initialized = true;
+	stats.table_name = "dummy scan";
+	return stats;
+}
+
+void RelationStatisticsHelper::CopyRelationStats(RelationStats &to, const RelationStats &from) {
+	to.column_distinct_count = from.column_distinct_count;
+	to.column_names = from.column_names;
+	to.cardinality = from.cardinality;
+	to.table_name = from.table_name;
+	to.stats_initialized = from.stats_initialized;
+}
+
+RelationStats RelationStatisticsHelper::CombineStatsOfReorderableOperator(vector<ColumnBinding> &bindings,
+                                                                          vector<RelationStats> relation_stats) {
+	RelationStats stats;
+	idx_t max_card = 0;
+	for (auto &child_stats : relation_stats) {
+		for (idx_t i = 0; i < child_stats.column_distinct_count.size(); i++) {
+			stats.column_distinct_count.push_back(child_stats.column_distinct_count.at(i));
+			stats.column_names.push_back(child_stats.column_names.at(i));
+		}
+		stats.table_name += "joined with " + child_stats.table_name;
+		max_card = MaxValue(max_card, child_stats.cardinality);
+	}
+	stats.stats_initialized = true;
+	stats.cardinality = max_card;
+	return stats;
+}
+
+RelationStats RelationStatisticsHelper::CombineStatsOfNonReorderableOperator(LogicalOperator &op,
+                                                                             vector<RelationStats> child_stats) {
+	D_ASSERT(child_stats.size() == 2);
+	RelationStats ret;
+	idx_t child_1_card = child_stats[0].stats_initialized ? child_stats[0].cardinality : 0;
+	idx_t child_2_card = child_stats[1].stats_initialized ? child_stats[1].cardinality : 0;
+	ret.cardinality = MaxValue(child_1_card, child_2_card);
+	ret.stats_initialized = true;
+	ret.filter_strength = 1;
+	ret.table_name = child_stats[0].table_name + " joined with " + child_stats[1].table_name;
+	for (auto &stats : child_stats) {
+		// MARK joins are nonreorderable. They won't return initialized stats
+		// continue in this case.
+		if (!stats.stats_initialized) {
+			continue;
+		}
+		for (auto &distinct_count : stats.column_distinct_count) {
+			ret.column_distinct_count.push_back(distinct_count);
+		}
+		for (auto &column_name : stats.column_names) {
+			ret.column_names.push_back(column_name);
+		}
+	}
+	return ret;
+}
+
+RelationStats RelationStatisticsHelper::ExtractExpressionGetStats(LogicalExpressionGet &expression_get,
+                                                                  ClientContext &context) {
+	auto stats = RelationStats();
+	idx_t card = expression_get.EstimateCardinality(context);
+	stats.cardinality = card;
+	for (idx_t i = 0; i < expression_get.GetColumnBindings().size(); i++) {
+		stats.column_distinct_count.push_back(DistinctCount({card, false}));
+		stats.column_names.push_back("expression_get_column");
+	}
+	stats.stats_initialized = true;
+	stats.table_name = "expression_get";
+	return stats;
+}
+
+RelationStats RelationStatisticsHelper::ExtractWindowStats(LogicalWindow &window, RelationStats &child_stats) {
+	RelationStats stats;
+	stats.cardinality = child_stats.cardinality;
+	stats.column_distinct_count = child_stats.column_distinct_count;
+	stats.column_names = child_stats.column_names;
+	stats.stats_initialized = true;
+	auto num_child_columns = window.GetColumnBindings().size();
+
+	for (idx_t column_index = child_stats.column_distinct_count.size(); column_index < num_child_columns;
+	     column_index++) {
+		stats.column_distinct_count.push_back(DistinctCount({child_stats.cardinality, false}));
+		stats.column_names.push_back("window");
+	}
+	return stats;
+}
+
+RelationStats RelationStatisticsHelper::ExtractAggregationStats(LogicalAggregate &aggr, RelationStats &child_stats) {
+	RelationStats stats;
+	// TODO: look at child distinct count to better estimate cardinality.
+	stats.cardinality = child_stats.cardinality;
+	stats.column_distinct_count = child_stats.column_distinct_count;
+	stats.column_names = child_stats.column_names;
+	stats.stats_initialized = true;
+	auto num_child_columns = aggr.GetColumnBindings().size();
+
+	for (idx_t column_index = child_stats.column_distinct_count.size(); column_index < num_child_columns;
+	     column_index++) {
+		stats.column_distinct_count.push_back(DistinctCount({child_stats.cardinality, false}));
+		stats.column_names.push_back("aggregate");
+	}
+	return stats;
+}
+
+idx_t RelationStatisticsHelper::InspectConjunctionAND(idx_t cardinality, idx_t column_index,
+                                                      ConjunctionAndFilter &filter, BaseStatistics &base_stats) {
+	auto cardinality_after_filters = cardinality;
+	for (auto &child_filter : filter.child_filters) {
+		if (child_filter->filter_type != TableFilterType::CONSTANT_COMPARISON) {
+			continue;
+		}
+		auto &comparison_filter = child_filter->Cast<ConstantFilter>();
+		if (comparison_filter.comparison_type != ExpressionType::COMPARE_EQUAL) {
+			continue;
+		}
+		auto column_count = base_stats.GetDistinctCount();
+		auto filtered_card = cardinality;
+		// column_count = 0 when there is no column count (i.e parquet scans)
+		if (column_count > 0) {
+			// we want the ceil of cardinality/column_count. We also want to avoid compiler errors
+			filtered_card = (cardinality + column_count - 1) / column_count;
+			cardinality_after_filters = filtered_card;
+		}
+	}
+	return cardinality_after_filters;
+}
+
+// TODO: Currently only simple AND filters are pushed into table scans.
+//  When OR filters are pushed this function can be added
+// idx_t RelationStatisticsHelper::InspectConjunctionOR(idx_t cardinality, idx_t column_index, ConjunctionOrFilter
+// &filter,
+//                                                     BaseStatistics &base_stats) {
+//	auto has_equality_filter = false;
+//	auto cardinality_after_filters = cardinality;
+//	for (auto &child_filter : filter.child_filters) {
+//		if (child_filter->filter_type != TableFilterType::CONSTANT_COMPARISON) {
+//			continue;
+//		}
+//		auto &comparison_filter = child_filter->Cast<ConstantFilter>();
+//		if (comparison_filter.comparison_type == ExpressionType::COMPARE_EQUAL) {
+//			auto column_count = base_stats.GetDistinctCount();
+//			auto increment = MaxValue<idx_t>(((cardinality + column_count - 1) / column_count), 1);
+//			if (has_equality_filter) {
+//				cardinality_after_filters += increment;
+//			} else {
+//				cardinality_after_filters = increment;
+//			}
+//			has_equality_filter = true;
+//		}
+//		if (child_filter->filter_type == TableFilterType::CONJUNCTION_AND) {
+//			auto &and_filter = child_filter->Cast<ConjunctionAndFilter>();
+//			cardinality_after_filters = RelationStatisticsHelper::InspectConjunctionAND(
+//			    cardinality_after_filters, column_index, and_filter, base_stats);
+//			continue;
+//		}
+//	}
+//	D_ASSERT(cardinality_after_filters > 0);
+//	return cardinality_after_filters;
+//}
+
+} // namespace duckdb

package/src/duckdb/src/optimizer/optimizer.cpp

@@ -1,27 +1,28 @@
 #include "duckdb/optimizer/optimizer.hpp"
 
 #include "duckdb/execution/column_binding_resolver.hpp"
-#include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/main/query_profiler.hpp"
 #include "duckdb/optimizer/column_lifetime_optimizer.hpp"
 #include "duckdb/optimizer/common_aggregate_optimizer.hpp"
+#include "duckdb/optimizer/compressed_materialization.hpp"
 #include "duckdb/optimizer/cse_optimizer.hpp"
 #include "duckdb/optimizer/deliminator.hpp"
-#include "duckdb/optimizer/unnest_rewriter.hpp"
 #include "duckdb/optimizer/expression_heuristics.hpp"
 #include "duckdb/optimizer/filter_pullup.hpp"
 #include "duckdb/optimizer/filter_pushdown.hpp"
 #include "duckdb/optimizer/in_clause_rewriter.hpp"
 #include "duckdb/optimizer/join_order/join_order_optimizer.hpp"
 #include "duckdb/optimizer/regex_range_filter.hpp"
+#include "duckdb/optimizer/remove_duplicate_groups.hpp"
 #include "duckdb/optimizer/remove_unused_columns.hpp"
 #include "duckdb/optimizer/rule/equal_or_null_simplification.hpp"
 #include "duckdb/optimizer/rule/in_clause_simplification.hpp"
 #include "duckdb/optimizer/rule/list.hpp"
 #include "duckdb/optimizer/statistics_propagator.hpp"
 #include "duckdb/optimizer/topn_optimizer.hpp"
+#include "duckdb/optimizer/unnest_rewriter.hpp"
 #include "duckdb/planner/binder.hpp"
 #include "duckdb/planner/planner.hpp"
 
@@ -52,6 +53,10 @@ Optimizer::Optimizer(Binder &binder, ClientContext &context) : context(context),
 #endif
 }
 
+ClientContext &Optimizer::GetContext() {
+	return context;
+}
+
 void Optimizer::RunOptimizer(OptimizerType type, const std::function<void()> &callback) {
 	auto &config = DBConfig::GetConfig(context);
 	if (config.options.disabled_optimizers.find(type) != config.options.disabled_optimizers.end()) {
@@ -73,6 +78,14 @@ void Optimizer::Verify(LogicalOperator &op) {
 
 unique_ptr<LogicalOperator> Optimizer::Optimize(unique_ptr<LogicalOperator> plan_p) {
 	Verify(*plan_p);
+
+	switch (plan_p->type) {
+	case LogicalOperatorType::LOGICAL_TRANSACTION:
+		return plan_p; // skip optimizing simple & often-occurring plans unaffected by rewrites
+	default:
+		break;
+	}
+
 	this->plan = std::move(plan_p);
 	// first we perform expression rewrites using the ExpressionRewriter
 	// this does not change the logical plan structure, but only simplifies the expression trees
@@ -96,8 +109,14 @@ unique_ptr<LogicalOperator> Optimizer::Optimize(unique_ptr<LogicalOperator> plan
 	});
 
 	RunOptimizer(OptimizerType::IN_CLAUSE, [&]() {
-		InClauseRewriter rewriter(context, *this);
-		plan = rewriter.Rewrite(std::move(plan));
+		InClauseRewriter ic_rewriter(context, *this);
+		plan = ic_rewriter.Rewrite(std::move(plan));
+	});
+
+	// removes any redundant DelimGets/DelimJoins
+	RunOptimizer(OptimizerType::DELIMINATOR, [&]() {
+		Deliminator deliminator;
+		plan = deliminator.Optimize(std::move(plan));
 	});
 
 	// then we perform the join ordering optimization
@@ -107,12 +126,6 @@ unique_ptr<LogicalOperator> Optimizer::Optimize(unique_ptr<LogicalOperator> plan
 		plan = optimizer.Optimize(std::move(plan));
 	});
 
-	// removes any redundant DelimGets/DelimJoins
-	RunOptimizer(OptimizerType::DELIMINATOR, [&]() {
-		Deliminator deliminator(context);
-		plan = deliminator.Optimize(std::move(plan));
-	});
-
 	// rewrites UNNESTs in DelimJoins by moving them to the projection
 	RunOptimizer(OptimizerType::UNNEST_REWRITER, [&]() {
 		UnnestRewriter unnest_rewriter;
@@ -125,10 +138,10 @@ unique_ptr<LogicalOperator> Optimizer::Optimize(unique_ptr<LogicalOperator> plan
 		unused.VisitOperator(*plan);
 	});
 
-	// perform statistics propagation
-	RunOptimizer(OptimizerType::STATISTICS_PROPAGATION, [&]() {
-		StatisticsPropagator propagator(context);
-		propagator.PropagateStatistics(plan);
+	// Remove duplicate groups from aggregates
+	RunOptimizer(OptimizerType::DUPLICATE_GROUPS, [&]() {
+		RemoveDuplicateGroups remove;
+		remove.VisitOperator(*plan);
 	});
 
 	// then we extract common subexpressions inside the different operators
@@ -137,16 +150,38 @@ unique_ptr<LogicalOperator> Optimizer::Optimize(unique_ptr<LogicalOperator> plan
 		cse_optimizer.VisitOperator(*plan);
 	});
 
+	// creates projection maps so unused columns are projected out early
+	RunOptimizer(OptimizerType::COLUMN_LIFETIME, [&]() {
+		ColumnLifetimeAnalyzer column_lifetime(true);
+		column_lifetime.VisitOperator(*plan);
+	});
+
+	// perform statistics propagation
+	column_binding_map_t<unique_ptr<BaseStatistics>> statistics_map;
+	RunOptimizer(OptimizerType::STATISTICS_PROPAGATION, [&]() {
+		StatisticsPropagator propagator(*this);
+		propagator.PropagateStatistics(plan);
+		statistics_map = propagator.GetStatisticsMap();
+	});
+
+	// remove duplicate aggregates
 	RunOptimizer(OptimizerType::COMMON_AGGREGATE, [&]() {
 		CommonAggregateOptimizer common_aggregate;
 		common_aggregate.VisitOperator(*plan);
 	});
 
+	// creates projection maps so unused columns are projected out early
 	RunOptimizer(OptimizerType::COLUMN_LIFETIME, [&]() {
 		ColumnLifetimeAnalyzer column_lifetime(true);
 		column_lifetime.VisitOperator(*plan);
 	});
 
+	// compress data based on statistics for materializing operators
+	RunOptimizer(OptimizerType::COMPRESSED_MATERIALIZATION, [&]() {
+		CompressedMaterialization compressed_materialization(context, binder, std::move(statistics_map));
+		compressed_materialization.Compress(plan);
+	});
+
 	// transform ORDER BY + LIMIT to TopN
 	RunOptimizer(OptimizerType::TOP_N, [&]() {
 		TopN topn;

package/src/duckdb/src/optimizer/pushdown/pushdown_cross_product.cpp

@@ -42,13 +42,13 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownCrossProduct(unique_ptr<Logi
 		vector<JoinCondition> conditions;
 		vector<unique_ptr<Expression>> arbitrary_expressions;
 		auto join_type = JoinType::INNER;
-		LogicalComparisonJoin::ExtractJoinConditions(join_type, op->children[0], op->children[1], left_bindings,
-		                                             right_bindings, join_expressions, conditions,
+		LogicalComparisonJoin::ExtractJoinConditions(GetContext(), join_type, op->children[0], op->children[1],
+		                                             left_bindings, right_bindings, join_expressions, conditions,
 		                                             arbitrary_expressions);
 		// create the join from the join conditions
-		return LogicalComparisonJoin::CreateJoin(JoinType::INNER, JoinRefType::REGULAR, std::move(op->children[0]),
-		                                         std::move(op->children[1]), std::move(conditions),
-		                                         std::move(arbitrary_expressions));
+		return LogicalComparisonJoin::CreateJoin(GetContext(), JoinType::INNER, JoinRefType::REGULAR,
+		                                         std::move(op->children[0]), std::move(op->children[1]),
+		                                         std::move(conditions), std::move(arbitrary_expressions));
 	} else {
 		// no join conditions found: keep as cross product
 		return op;

package/src/duckdb/src/optimizer/pushdown/pushdown_get.cpp

@@ -4,7 +4,6 @@
 #include "duckdb/planner/expression/bound_parameter_expression.hpp"
 #include "duckdb/planner/operator/logical_filter.hpp"
 #include "duckdb/planner/operator/logical_get.hpp"
-#include "duckdb/storage/data_table.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/optimizer/pushdown/pushdown_projection.cpp

@@ -6,6 +6,23 @@
 
 namespace duckdb {
 
+static bool HasSideEffects(LogicalProjection &proj, const unique_ptr<Expression> &expr) {
+	if (expr->type == ExpressionType::BOUND_COLUMN_REF) {
+		auto &colref = expr->Cast<BoundColumnRefExpression>();
+		D_ASSERT(colref.binding.table_index == proj.table_index);
+		D_ASSERT(colref.binding.column_index < proj.expressions.size());
+		D_ASSERT(colref.depth == 0);
+		if (proj.expressions[colref.binding.column_index]->HasSideEffects()) {
+			return true;
+		}
+		return false;
+	}
+	bool has_side_effects = false;
+	ExpressionIterator::EnumerateChildren(
+	    *expr, [&](unique_ptr<Expression> &child) { has_side_effects |= HasSideEffects(proj, child); });
+	return has_side_effects;
+}
+
 static unique_ptr<Expression> ReplaceProjectionBindings(LogicalProjection &proj, unique_ptr<Expression> expr) {
 	if (expr->type == ExpressionType::BOUND_COLUMN_REF) {
 		auto &colref = expr->Cast<BoundColumnRefExpression>();
@@ -27,15 +44,25 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownProjection(unique_ptr<Logica
 	// all the BoundColumnRefExpressions in the filter should refer to the LogicalProjection
 	// we can rewrite them by replacing those references with the expression of the LogicalProjection node
 	FilterPushdown child_pushdown(optimizer);
+	// There are some expressions can not be pushed down. We should keep them
+	// and add an extra filter operator.
+	vector<unique_ptr<Expression>> remain_expressions;
 	for (auto &filter : filters) {
 		auto &f = *filter;
 		D_ASSERT(f.bindings.size() <= 1);
-		// rewrite the bindings within this subquery
-		f.filter = ReplaceProjectionBindings(proj, std::move(f.filter));
-		// add the filter to the child pushdown
-		if (child_pushdown.AddFilter(std::move(f.filter)) == FilterResult::UNSATISFIABLE) {
-			// filter statically evaluates to false, strip tree
-			return make_uniq<LogicalEmptyResult>(std::move(op));
+		bool has_side_effects = HasSideEffects(proj, f.filter);
+		if (has_side_effects) {
+			// We can't push down related expressions if the column in the
+			// expression is generated by the functions which have side effects
+			remain_expressions.push_back(std::move(f.filter));
+		} else {
+			// rewrite the bindings within this subquery
+			f.filter = ReplaceProjectionBindings(proj, std::move(f.filter));
+			// add the filter to the child pushdown
+			if (child_pushdown.AddFilter(std::move(f.filter)) == FilterResult::UNSATISFIABLE) {
+				// filter statically evaluates to false, strip tree
+				return make_uniq<LogicalEmptyResult>(std::move(op));
+			}
 		}
 	}
 	child_pushdown.GenerateFilters();
@@ -45,7 +72,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownProjection(unique_ptr<Logica
 		// child returns an empty result: generate an empty result here too
 		return make_uniq<LogicalEmptyResult>(std::move(op));
 	}
-	return op;
+	return AddLogicalFilter(std::move(op), std::move(remain_expressions));
 }
 
 } // namespace duckdb