duckdb 1.3.2 → 1.3.3-dev4.0

package/src/duckdb/src/planner/binder.cpp

@@ -29,36 +29,31 @@
 namespace duckdb {
 
 Binder &Binder::GetRootBinder() {
-	reference<Binder> root = *this;
-	while (root.get().parent) {
-		root = *root.get().parent;
-	}
-	return root.get();
+	return root_binder;
 }
 
 idx_t Binder::GetBinderDepth() const {
-	const_reference<Binder> root = *this;
-	idx_t depth = 1;
-	while (root.get().parent) {
-		depth++;
-		root = *root.get().parent;
-	}
 	return depth;
 }
 
-shared_ptr<Binder> Binder::CreateBinder(ClientContext &context, optional_ptr<Binder> parent, BinderType binder_type) {
-	auto depth = parent ? parent->GetBinderDepth() : 0;
+void Binder::IncreaseDepth() {
+	depth++;
 	if (depth > context.config.max_expression_depth) {
 		throw BinderException("Max expression depth limit of %lld exceeded. Use \"SET max_expression_depth TO x\" to "
 		                      "increase the maximum expression depth.",
 		                      context.config.max_expression_depth);
 	}
+}
+
+shared_ptr<Binder> Binder::CreateBinder(ClientContext &context, optional_ptr<Binder> parent, BinderType binder_type) {
 	return shared_ptr<Binder>(new Binder(context, parent ? parent->shared_from_this() : nullptr, binder_type));
 }
 
 Binder::Binder(ClientContext &context, shared_ptr<Binder> parent_p, BinderType binder_type)
     : context(context), bind_context(*this), parent(std::move(parent_p)), bound_tables(0), binder_type(binder_type),
-      entry_retriever(context) {
+      entry_retriever(context), root_binder(parent ? parent->GetRootBinder() : *this),
+      depth(parent ? parent->GetBinderDepth() : 1) {
+	IncreaseDepth();
 	if (parent) {
 		entry_retriever.Inherit(parent->entry_retriever);
 
@@ -80,13 +75,12 @@ unique_ptr<BoundCTENode> Binder::BindMaterializedCTE(CommonTableExpressionMap &c
 	vector<unique_ptr<CTENode>> materialized_ctes;
 	for (auto &cte : cte_map.map) {
 		auto &cte_entry = cte.second;
-		if (cte_entry->materialized == CTEMaterialize::CTE_MATERIALIZE_ALWAYS) {
-			auto mat_cte = make_uniq<CTENode>();
-			mat_cte->ctename = cte.first;
-			mat_cte->query = cte_entry->query->node->Copy();
-			mat_cte->aliases = cte_entry->aliases;
-			materialized_ctes.push_back(std::move(mat_cte));
-		}
+		auto mat_cte = make_uniq<CTENode>();
+		mat_cte->ctename = cte.first;
+		mat_cte->query = cte_entry->query->node->Copy();
+		mat_cte->aliases = cte_entry->aliases;
+		mat_cte->materialized = cte_entry->materialized;
+		materialized_ctes.push_back(std::move(mat_cte));
 	}
 
 	if (materialized_ctes.empty()) {
@@ -196,6 +190,8 @@ BoundStatement Binder::Bind(SQLStatement &statement) {
 		return Bind(statement.Cast<CopyDatabaseStatement>());
 	case StatementType::UPDATE_EXTENSIONS_STATEMENT:
 		return Bind(statement.Cast<UpdateExtensionsStatement>());
+	case StatementType::MERGE_INTO_STATEMENT:
+		return BindWithCTE(statement.Cast<MergeIntoStatement>());
 	default: // LCOV_EXCL_START
 		throw NotImplementedException("Unimplemented statement type \"%s\" for Bind",
 		                              StatementTypeToString(statement.type));
@@ -208,118 +204,6 @@ void Binder::AddCTEMap(CommonTableExpressionMap &cte_map) {
 	}
 }
 
-static void GetTableRefCountsNode(case_insensitive_map_t<idx_t> &cte_ref_counts, QueryNode &node);
-
-static void GetTableRefCountsExpr(case_insensitive_map_t<idx_t> &cte_ref_counts, ParsedExpression &expr) {
-	if (expr.GetExpressionType() == ExpressionType::SUBQUERY) {
-		auto &subquery = expr.Cast<SubqueryExpression>();
-		GetTableRefCountsNode(cte_ref_counts, *subquery.subquery->node);
-	} else {
-		ParsedExpressionIterator::EnumerateChildren(
-		    expr, [&](ParsedExpression &expr) { GetTableRefCountsExpr(cte_ref_counts, expr); });
-	}
-}
-
-static void GetTableRefCountsNode(case_insensitive_map_t<idx_t> &cte_ref_counts, QueryNode &node) {
-	ParsedExpressionIterator::EnumerateQueryNodeChildren(
-	    node, [&](unique_ptr<ParsedExpression> &child) { GetTableRefCountsExpr(cte_ref_counts, *child); },
-	    [&](TableRef &ref) {
-		    if (ref.type != TableReferenceType::BASE_TABLE) {
-			    return;
-		    }
-		    auto cte_ref_counts_it = cte_ref_counts.find(ref.Cast<BaseTableRef>().table_name);
-		    if (cte_ref_counts_it != cte_ref_counts.end()) {
-			    cte_ref_counts_it->second++;
-		    }
-	    });
-}
-
-static bool ParsedExpressionIsAggregate(Binder &binder, const ParsedExpression &expr) {
-	if (expr.GetExpressionClass() == ExpressionClass::FUNCTION) {
-		auto &function = expr.Cast<FunctionExpression>();
-		QueryErrorContext error_context;
-
-		EntryLookupInfo lookup_info(CatalogType::AGGREGATE_FUNCTION_ENTRY, function.function_name, error_context);
-		auto entry =
-		    binder.GetCatalogEntry(function.catalog, function.schema, lookup_info, OnEntryNotFound::RETURN_NULL);
-		if (entry && entry->type == CatalogType::AGGREGATE_FUNCTION_ENTRY) {
-			return true;
-		}
-	}
-	bool is_aggregate = false;
-	ParsedExpressionIterator::EnumerateChildren(
-	    expr, [&](const ParsedExpression &child) { is_aggregate |= ParsedExpressionIsAggregate(binder, child); });
-	return is_aggregate;
-}
-
-bool Binder::OptimizeCTEs(QueryNode &node) {
-	D_ASSERT(context.config.enable_optimizer);
-
-	// only applies to nodes that have at least one CTE
-	auto &cte_map = node.cte_map.map;
-	if (cte_map.empty()) {
-		return false;
-	}
-
-	// initialize counts with the CTE names
-	case_insensitive_map_t<idx_t> cte_ref_counts;
-	for (auto &cte : cte_map) {
-		cte_ref_counts[cte.first];
-	}
-
-	// count the references of each CTE
-	GetTableRefCountsNode(cte_ref_counts, node);
-
-	// determine for each CTE whether it should be materialized
-	bool result = false;
-	for (auto &cte : cte_map) {
-		if (cte.second->materialized != CTEMaterialize::CTE_MATERIALIZE_DEFAULT) {
-			continue; // only triggers when nothing is specified
-		}
-		if (bind_context.GetCTEBinding(cte.first)) {
-			continue; // there's a CTE in the bind context with an overlapping name, we can't also materialize this
-		}
-
-		auto cte_ref_counts_it = cte_ref_counts.find(cte.first);
-		D_ASSERT(cte_ref_counts_it != cte_ref_counts.end());
-
-		// only applies to CTEs that are referenced more than once
-		if (cte_ref_counts_it->second <= 1) {
-			continue;
-		}
-
-		// if the cte is a SELECT node
-		if (cte.second->query->node->type != QueryNodeType::SELECT_NODE) {
-			continue;
-		}
-
-		// we materialize if the CTE ends in an aggregation
-		auto &cte_node = cte.second->query->node->Cast<SelectNode>();
-		bool materialize = !cte_node.groups.group_expressions.empty() || !cte_node.groups.grouping_sets.empty();
-		// or has a distinct modifier
-		for (auto &modifier : cte_node.modifiers) {
-			if (materialize) {
-				break;
-			}
-			if (modifier->type == ResultModifierType::DISTINCT_MODIFIER) {
-				materialize = true;
-			}
-		}
-		for (auto &sel : cte_node.select_list) {
-			if (materialize) {
-				break;
-			}
-			materialize |= ParsedExpressionIsAggregate(*this, *sel);
-		}
-
-		if (materialize) {
-			cte.second->materialized = CTEMaterialize::CTE_MATERIALIZE_ALWAYS;
-			result = true;
-		}
-	}
-	return result;
-}
-
 unique_ptr<BoundQueryNode> Binder::BindNode(QueryNode &node) {
 	// first we visit the set of CTEs and add them to the bind context
 	AddCTEMap(node.cte_map);
@@ -345,33 +229,13 @@ unique_ptr<BoundQueryNode> Binder::BindNode(QueryNode &node) {
 
 BoundStatement Binder::Bind(QueryNode &node) {
 	BoundStatement result;
-	if (node.type != QueryNodeType::CTE_NODE && // Issue #13850 - Don't auto-materialize if users materialize (for now)
-	    !Optimizer::OptimizerDisabled(context, OptimizerType::MATERIALIZED_CTE) && context.config.enable_optimizer &&
-	    OptimizeCTEs(node)) {
-		switch (node.type) {
-		case QueryNodeType::SELECT_NODE:
-			result = BindWithCTE(node.Cast<SelectNode>());
-			break;
-		case QueryNodeType::RECURSIVE_CTE_NODE:
-			result = BindWithCTE(node.Cast<RecursiveCTENode>());
-			break;
-		case QueryNodeType::CTE_NODE:
-			result = BindWithCTE(node.Cast<CTENode>());
-			break;
-		default:
-			D_ASSERT(node.type == QueryNodeType::SET_OPERATION_NODE);
-			result = BindWithCTE(node.Cast<SetOperationNode>());
-			break;
-		}
-	} else {
-		auto bound_node = BindNode(node);
+	auto bound_node = BindNode(node);
 
-		result.names = bound_node->names;
-		result.types = bound_node->types;
+	result.names = bound_node->names;
+	result.types = bound_node->types;
 
-		// and plan it
-		result.plan = CreatePlan(*bound_node);
-	}
+	// and plan it
+	result.plan = CreatePlan(*bound_node);
 	return result;
 }
 
@@ -423,6 +287,9 @@ unique_ptr<BoundTableRef> Binder::Bind(TableRef &ref) {
 	case TableReferenceType::DELIM_GET:
 		result = Bind(ref.Cast<DelimGetRef>());
 		break;
+	case TableReferenceType::BOUND_TABLE_REF:
+		result = Bind(ref.Cast<BoundRefWrapper>());
+		break;
 	case TableReferenceType::CTE:
 	case TableReferenceType::INVALID:
 	default:
@@ -659,28 +526,26 @@ case_insensitive_map_t<unique_ptr<TableRef>> &Binder::GetReplacementScans() {
 }
 
 // FIXME: this is extremely naive
-void VerifyNotExcluded(ParsedExpression &expr) {
-	if (expr.GetExpressionType() == ExpressionType::COLUMN_REF) {
-		auto &column_ref = expr.Cast<ColumnRefExpression>();
-		if (!column_ref.IsQualified()) {
-			return;
-		}
-		auto &table_name = column_ref.GetTableName();
-		if (table_name == "excluded") {
-			throw NotImplementedException("'excluded' qualified columns are not supported in the RETURNING clause yet");
-		}
-		return;
-	}
-	ParsedExpressionIterator::EnumerateChildren(
-	    expr, [&](const ParsedExpression &child) { VerifyNotExcluded((ParsedExpression &)child); });
+void VerifyNotExcluded(const ParsedExpression &root_expr) {
+	ParsedExpressionIterator::VisitExpression<ColumnRefExpression>(
+	    root_expr, [&](const ColumnRefExpression &column_ref) {
+		    if (!column_ref.IsQualified()) {
+			    return;
+		    }
+		    auto &table_name = column_ref.GetTableName();
+		    if (table_name == "excluded") {
+			    throw NotImplementedException(
+			        "'excluded' qualified columns are not supported in the RETURNING clause yet");
+		    }
+	    });
 }
 
 BoundStatement Binder::BindReturning(vector<unique_ptr<ParsedExpression>> returning_list, TableCatalogEntry &table,
                                      const string &alias, idx_t update_table_index,
-                                     unique_ptr<LogicalOperator> child_operator, BoundStatement result) {
+                                     unique_ptr<LogicalOperator> child_operator, virtual_column_map_t virtual_columns) {
 
 	vector<LogicalType> types;
-	vector<std::string> names;
+	vector<string> names;
 
 	auto binder = Binder::CreateBinder(context);
 
@@ -695,12 +560,14 @@ BoundStatement Binder::BindReturning(vector<unique_ptr<ParsedExpression>> return
 		column_count++;
 	}
 
-	binder->bind_context.AddBaseTable(update_table_index, alias, names, types, bound_columns, table, false);
+	binder->bind_context.AddBaseTable(update_table_index, alias, names, types, bound_columns, table,
+	                                  std::move(virtual_columns));
 	ReturningBinder returning_binder(*binder, context);
 
 	vector<unique_ptr<Expression>> projection_expressions;
 	LogicalType result_type;
 	vector<unique_ptr<ParsedExpression>> new_returning_list;
+	BoundStatement result;
 	binder->ExpandStarExpressions(returning_list, new_returning_list);
 	for (auto &returning_expr : new_returning_list) {
 		VerifyNotExcluded(*returning_expr);

package/src/duckdb/src/planner/bound_result_modifier.cpp

@@ -36,6 +36,14 @@ bool BoundOrderByNode::Equals(const BoundOrderByNode &other) const {
 	return true;
 }
 
+string BoundOrderByNode::GetOrderModifier() const {
+	string modifier;
+	modifier += (type == OrderType::ASCENDING) ? "ASC" : "DESC";
+	modifier += " NULLS";
+	modifier += (null_order == OrderByNullType::NULLS_FIRST) ? " FIRST" : " LAST";
+	return modifier;
+}
+
 string BoundOrderByNode::ToString() const {
 	auto str = expression->ToString();
 	switch (type) {

package/src/duckdb/src/planner/collation_binding.cpp

@@ -3,6 +3,7 @@
 #include "duckdb/catalog/catalog_entry/scalar_function_catalog_entry.hpp"
 #include "duckdb/planner/expression/bound_function_expression.hpp"
 #include "duckdb/main/config.hpp"
+#include "duckdb/main/settings.hpp"
 #include "duckdb/catalog/catalog.hpp"
 #include "duckdb/function/function_binder.hpp"
 
@@ -18,7 +19,7 @@ bool PushVarcharCollation(ClientContext &context, unique_ptr<Expression> &source
 	auto str_collation = StringType::GetCollation(sql_type);
 	string collation;
 	if (str_collation.empty()) {
-		collation = DBConfig::GetConfig(context).options.collation;
+		collation = DBConfig::GetSetting<DefaultCollationSetting>(context);
 	} else {
 		collation = str_collation;
 	}

package/src/duckdb/src/planner/expression.cpp

@@ -68,7 +68,7 @@ bool Expression::PropagatesNullValues() const {
 	if (type == ExpressionType::OPERATOR_IS_NULL || type == ExpressionType::OPERATOR_IS_NOT_NULL ||
 	    type == ExpressionType::COMPARE_NOT_DISTINCT_FROM || type == ExpressionType::COMPARE_DISTINCT_FROM ||
 	    type == ExpressionType::CONJUNCTION_OR || type == ExpressionType::CONJUNCTION_AND ||
-	    type == ExpressionType::OPERATOR_COALESCE) {
+	    type == ExpressionType::OPERATOR_COALESCE || type == ExpressionType::CASE_EXPR) {
 		return false;
 	}
 	bool propagate_null_values = true;

package/src/duckdb/src/planner/expression_binder/order_binder.cpp

@@ -14,6 +14,7 @@
 #include "duckdb/planner/expression_binder/select_bind_state.hpp"
 #include "duckdb/main/client_config.hpp"
 #include "duckdb/common/pair.hpp"
+#include "duckdb/main/settings.hpp"
 
 namespace duckdb {
 
@@ -59,8 +60,9 @@ optional_idx OrderBinder::TryGetProjectionReference(ParsedExpression &expr) cons
 			// non-integral expression
 			// ORDER BY <constant> has no effect
 			// this is disabled by default (matching Postgres) - but we can control this with a setting
-			auto &config = ClientConfig::GetConfig(binders[0].get().context);
-			if (!config.order_by_non_integer_literal) {
+			auto order_by_non_integer_literal =
+			    DBConfig::GetSetting<OrderByNonIntegerLiteralSetting>(binders[0].get().context);
+			if (!order_by_non_integer_literal) {
 				throw BinderException(expr,
 				                      "%s non-integer literal has no effect.\n* SET "
 				                      "order_by_non_integer_literal=true to allow this behavior.",

package/src/duckdb/src/planner/expression_binder/projection_binder.cpp

@@ -0,0 +1,46 @@
+#include "duckdb/planner/expression_binder/projection_binder.hpp"
+#include "duckdb/planner/expression/bound_columnref_expression.hpp"
+
+namespace duckdb {
+
+ProjectionBinder::ProjectionBinder(Binder &binder, ClientContext &context, idx_t proj_index_p,
+                                   vector<unique_ptr<Expression>> &proj_expressions_p, string clause_p)
+    : ExpressionBinder(binder, context), proj_index(proj_index_p), proj_expressions(proj_expressions_p),
+      clause(std::move(clause_p)) {
+}
+
+BindResult ProjectionBinder::BindColumnRef(unique_ptr<ParsedExpression> &expr_ptr, idx_t depth, bool root_expression) {
+	auto result = ExpressionBinder::BindExpression(expr_ptr, depth);
+	if (result.HasError()) {
+		return result;
+	}
+	if (result.expression->GetExpressionClass() == ExpressionClass::BOUND_LAMBDA_REF) {
+		return result;
+	}
+	// we have successfully bound a column - push it into the projection and emit a reference
+	auto proj_ref = make_uniq<BoundColumnRefExpression>(result.expression->return_type,
+	                                                    ColumnBinding(proj_index, proj_expressions.size()));
+	proj_ref->alias = result.expression->GetName();
+	proj_expressions.push_back(std::move(result.expression));
+	return BindResult(std::move(proj_ref));
+}
+
+BindResult ProjectionBinder::BindExpression(unique_ptr<ParsedExpression> &expr_ptr, idx_t depth, bool root_expression) {
+	auto &expr = *expr_ptr;
+	switch (expr.GetExpressionClass()) {
+	case ExpressionClass::DEFAULT:
+		return BindUnsupportedExpression(expr, depth, clause + " cannot contain DEFAULT clause");
+	case ExpressionClass::WINDOW:
+		return BindUnsupportedExpression(expr, depth, clause + " cannot contain window functions!");
+	case ExpressionClass::COLUMN_REF:
+		return BindColumnRef(expr_ptr, depth, root_expression);
+	default:
+		return ExpressionBinder::BindExpression(expr_ptr, depth);
+	}
+}
+
+string ProjectionBinder::UnsupportedAggregateMessage() {
+	return clause + " cannot contain aggregate functions";
+}
+
+} // namespace duckdb

package/src/duckdb/src/planner/expression_binder/table_function_binder.cpp

@@ -6,8 +6,10 @@
 
 namespace duckdb {
 
-TableFunctionBinder::TableFunctionBinder(Binder &binder, ClientContext &context, string table_function_name_p)
-    : ExpressionBinder(binder, context), table_function_name(std::move(table_function_name_p)) {
+TableFunctionBinder::TableFunctionBinder(Binder &binder, ClientContext &context, string table_function_name_p,
+                                         string clause_p)
+    : ExpressionBinder(binder, context), table_function_name(std::move(table_function_name_p)),
+      clause(std::move(clause_p)) {
 }
 
 BindResult TableFunctionBinder::BindLambdaReference(LambdaRefExpression &expr, idx_t depth) {
@@ -67,18 +69,18 @@ BindResult TableFunctionBinder::BindExpression(unique_ptr<ParsedExpression> &exp
 	case ExpressionClass::COLUMN_REF:
 		return BindColumnReference(expr_ptr, depth, root_expression);
 	case ExpressionClass::SUBQUERY:
-		throw BinderException("Table function cannot contain subqueries");
+		throw BinderException(clause + " cannot contain subqueries");
 	case ExpressionClass::DEFAULT:
-		return BindResult("Table function cannot contain DEFAULT clause");
+		return BindResult(clause + " cannot contain DEFAULT clause");
 	case ExpressionClass::WINDOW:
-		return BindResult("Table function cannot contain window functions!");
+		return BindResult(clause + " cannot contain window functions!");
 	default:
 		return ExpressionBinder::BindExpression(expr_ptr, depth);
 	}
 }
 
 string TableFunctionBinder::UnsupportedAggregateMessage() {
-	return "Table function cannot contain aggregates!";
+	return clause + " cannot contain aggregates!";
 }
 
 } // namespace duckdb

package/src/duckdb/src/planner/expression_iterator.cpp

@@ -152,6 +152,37 @@ void ExpressionIterator::EnumerateExpression(unique_ptr<Expression> &expr,
 	                                      [&](unique_ptr<Expression> &child) { EnumerateExpression(child, callback); });
 }
 
+void ExpressionIterator::EnumerateExpression(unique_ptr<Expression> &expr,
+                                             const std::function<void(unique_ptr<Expression> &child)> &callback) {
+	if (!expr) {
+		return;
+	}
+	callback(expr);
+	ExpressionIterator::EnumerateChildren(*expr,
+	                                      [&](unique_ptr<Expression> &child) { EnumerateExpression(child, callback); });
+}
+
+void ExpressionIterator::VisitExpressionClass(const Expression &expr, ExpressionClass expr_class,
+                                              const std::function<void(const Expression &child)> &callback) {
+	if (expr.GetExpressionClass() == expr_class) {
+		callback(expr);
+		return;
+	}
+	ExpressionIterator::EnumerateChildren(
+	    expr, [&](const Expression &child) { VisitExpressionClass(child, expr_class, callback); });
+}
+
+void ExpressionIterator::VisitExpressionClassMutable(
+    unique_ptr<Expression> &expr, ExpressionClass expr_class,
+    const std::function<void(unique_ptr<Expression> &child)> &callback) {
+	if (expr->GetExpressionClass() == expr_class) {
+		callback(expr);
+		return;
+	}
+	ExpressionIterator::EnumerateChildren(
+	    *expr, [&](unique_ptr<Expression> &child) { VisitExpressionClassMutable(child, expr_class, callback); });
+}
+
 void BoundNodeVisitor::VisitExpression(unique_ptr<Expression> &expression) {
 	VisitExpressionChildren(*expression);
 }