duckdb 1.0.1-dev22.0 → 1.0.1-dev27.0

package/src/duckdb/src/planner/expression_binder.cpp

@@ -6,6 +6,8 @@
 #include "duckdb/planner/binder.hpp"
 #include "duckdb/planner/expression/list.hpp"
 #include "duckdb/planner/expression_iterator.hpp"
+#include "duckdb/common/operator/cast_operators.hpp"
+#include "duckdb/main/client_config.hpp"
 
 namespace duckdb {
 
@@ -35,18 +37,21 @@ ExpressionBinder::~ExpressionBinder() {
 }
 
 void ExpressionBinder::InitializeStackCheck() {
+	static constexpr idx_t INITIAL_DEPTH = 5;
 	if (binder.HasActiveBinder()) {
-		stack_depth = binder.GetActiveBinder().stack_depth;
+		stack_depth = binder.GetActiveBinder().stack_depth + INITIAL_DEPTH;
 	} else {
-		stack_depth = 0;
+		stack_depth = INITIAL_DEPTH;
 	}
 }
 
 StackChecker<ExpressionBinder> ExpressionBinder::StackCheck(const ParsedExpression &expr, idx_t extra_stack) {
 	D_ASSERT(stack_depth != DConstants::INVALID_INDEX);
-	if (stack_depth + extra_stack >= MAXIMUM_STACK_DEPTH) {
-		throw BinderException("Maximum recursion depth exceeded (Maximum: %llu) while binding \"%s\"",
-		                      MAXIMUM_STACK_DEPTH, expr.ToString());
+	auto &options = ClientConfig::GetConfig(context);
+	if (stack_depth + extra_stack >= options.max_expression_depth) {
+		throw BinderException("Max expression depth limit of %lld exceeded. Use \"SET max_expression_depth TO x\" to "
+		                      "increase the maximum expression depth.",
+		                      options.max_expression_depth);
 	}
 	return StackChecker<ExpressionBinder>(*this, extra_stack);
 }
@@ -65,7 +70,7 @@ BindResult ExpressionBinder::BindExpression(unique_ptr<ParsedExpression> &expr,
 	case ExpressionClass::COLLATE:
 		return BindExpression(expr_ref.Cast<CollateExpression>(), depth);
 	case ExpressionClass::COLUMN_REF:
-		return BindExpression(expr_ref.Cast<ColumnRefExpression>(), depth);
+		return BindExpression(expr_ref.Cast<ColumnRefExpression>(), depth, root_expression);
 	case ExpressionClass::LAMBDA_REF:
 		return BindExpression(expr_ref.Cast<LambdaRefExpression>(), depth);
 	case ExpressionClass::COMPARISON:
@@ -91,16 +96,103 @@ BindResult ExpressionBinder::BindExpression(unique_ptr<ParsedExpression> &expr,
 		return BindExpression(expr_ref.Cast<SubqueryExpression>(), depth);
 	case ExpressionClass::PARAMETER:
 		return BindExpression(expr_ref.Cast<ParameterExpression>(), depth);
-	case ExpressionClass::POSITIONAL_REFERENCE: {
+	case ExpressionClass::POSITIONAL_REFERENCE:
 		return BindPositionalReference(expr, depth, root_expression);
-	}
 	case ExpressionClass::STAR:
-		return BindResult(BinderException(expr_ref, "STAR expression is not supported here"));
+		return BindResult(BinderException::Unsupported(expr_ref, "STAR expression is not supported here"));
 	default:
 		throw NotImplementedException("Unimplemented expression class");
 	}
 }
 
+static bool CombineMissingColumns(ErrorData &current, ErrorData new_error) {
+	auto &current_info = current.ExtraInfo();
+	auto &new_info = new_error.ExtraInfo();
+	auto current_entry = current_info.find("error_subtype");
+	auto new_entry = new_info.find("error_subtype");
+	if (current_entry == current_info.end() || new_entry == new_info.end()) {
+		// no subtype info in either expression
+		return false;
+	}
+	if (current_entry->second != "COLUMN_NOT_FOUND" || new_entry->second != "COLUMN_NOT_FOUND") {
+		// either info is not a `COLUMN_NOT_FOUND`
+		return false;
+	}
+	current_entry = current_info.find("name");
+	new_entry = new_info.find("name");
+	if (current_entry == current_info.end() || new_entry == new_info.end()) {
+		// no candidate info in either column
+		return false;
+	}
+	if (current_entry->second != new_entry->second) {
+		// error does not concern the same name/column
+		return false;
+	}
+	auto column_name = current_entry->second;
+	current_entry = current_info.find("candidates");
+	new_entry = new_info.find("candidates");
+	if (current_entry == current_info.end()) {
+		// no current candidates - use new candidates
+		current = std::move(new_error);
+		return true;
+	}
+	if (new_entry == new_info.end()) {
+		// no new candidates - use current candidates
+		return true;
+	}
+	// both errors have candidates - combine the candidates
+	auto current_candidates = StringUtil::Split(current_entry->second, ",");
+	auto new_candidates = StringUtil::Split(new_entry->second, ",");
+	current_candidates.insert(current_candidates.end(), new_candidates.begin(), new_candidates.end());
+
+	// run the similarity ranking on both sets of candidates
+	unordered_set<string> candidates;
+	vector<pair<string, double>> scores;
+	for (auto &candidate : current_candidates) {
+		// split by "." since the candidates might be in the form "table.column"
+		auto column_splits = StringUtil::Split(candidate, ".");
+		if (column_splits.empty()) {
+			continue;
+		}
+		auto &candidate_column = column_splits.back();
+		auto entry = candidates.find(candidate);
+		if (entry != candidates.end()) {
+			// already found
+			continue;
+		}
+		auto score = StringUtil::SimilarityRating(candidate_column, column_name);
+		candidates.insert(candidate);
+		scores.emplace_back(make_pair(std::move(candidate), score));
+	}
+	// get a new top-n
+	auto top_candidates = StringUtil::TopNStrings(scores);
+	// get query location
+	QueryErrorContext context;
+	current_entry = current_info.find("position");
+	new_entry = current_info.find("position");
+	uint64_t position;
+	if (current_entry != current_info.end() &&
+	    TryCast::Operation<string_t, uint64_t>(current_entry->second, position)) {
+		context = QueryErrorContext(position);
+	} else if (new_entry != new_info.end() && TryCast::Operation<string_t, uint64_t>(new_entry->second, position)) {
+		context = QueryErrorContext(position);
+	}
+	// generate a new (combined) error
+	current = BinderException::ColumnNotFound(column_name, top_candidates, context);
+	return true;
+}
+
+static void CombineErrors(ErrorData &current, ErrorData new_error) {
+	// try to combine missing column exceptions in order to pick the most relevant one
+	if (CombineMissingColumns(current, new_error)) {
+		// keep the old info
+		return;
+	}
+
+	// override the error with the new one
+	current = std::move(new_error);
+}
+
 BindResult ExpressionBinder::BindCorrelatedColumns(unique_ptr<ParsedExpression> &expr, ErrorData error_message) {
 	// try to bind in one of the outer queries, if the binding error occurred in a subquery
 	auto &active_binders = binder.GetActiveBinders();
@@ -113,10 +205,12 @@ BindResult ExpressionBinder::BindCorrelatedColumns(unique_ptr<ParsedExpression>
 	while (!active_binders.empty()) {
 		auto &next_binder = active_binders.back().get();
 		ExpressionBinder::QualifyColumnNames(next_binder.binder, expr);
-		bind_error = next_binder.Bind(expr, depth);
-		if (!bind_error.HasError()) {
+		auto next_error = next_binder.Bind(expr, depth);
+		if (!next_error.HasError()) {
+			bind_error = std::move(next_error);
 			break;
 		}
+		CombineErrors(bind_error, std::move(next_error));
 		depth++;
 		active_binders.pop_back();
 	}
@@ -228,7 +322,8 @@ unique_ptr<Expression> ExpressionBinder::Bind(unique_ptr<ParsedExpression> &expr
 		// aggregate with constant input must be bound to a root node.
 		auto result = BindCorrelatedColumns(expr, error_msg);
 		if (result.HasError()) {
-			result.error.Throw();
+			CombineErrors(error_msg, std::move(result.error));
+			error_msg.Throw();
 		}
 		auto &bound_expr = expr->Cast<BoundExpression>();
 		ExtractCorrelatedExpressions(binder, *bound_expr.expr);
@@ -282,19 +377,24 @@ ErrorData ExpressionBinder::Bind(unique_ptr<ParsedExpression> &expr, idx_t depth
 	return ErrorData();
 }
 
+BindResult ExpressionBinder::BindUnsupportedExpression(ParsedExpression &expr, idx_t depth, const string &message) {
+	// we always prefer to throw an error if it occurs in a child expression
+	// since that error might be more descriptive
+	// bind all children
+	ErrorData result;
+	ParsedExpressionIterator::EnumerateChildren(
+	    expr, [&](unique_ptr<ParsedExpression> &child) { BindChild(child, depth, result); });
+	if (result.HasError()) {
+		return BindResult(std::move(result));
+	}
+	return BindResult(BinderException::Unsupported(expr, message));
+}
+
 bool ExpressionBinder::IsUnnestFunction(const string &function_name) {
 	return function_name == "unnest" || function_name == "unlist";
 }
 
-bool ExpressionBinder::IsLambdaFunction(const FunctionExpression &function) {
-	// check for lambda parameters, ignore ->> operator (JSON extension)
-	if (function.function_name != "->>") {
-		for (auto &child : function.children) {
-			if (child->expression_class == ExpressionClass::LAMBDA) {
-				return true;
-			}
-		}
-	}
+bool ExpressionBinder::TryBindAlias(ColumnRefExpression &colref, bool root_expression, BindResult &result) {
 	return false;
 }
 

package/src/duckdb/src/planner/expression_iterator.cpp

@@ -239,6 +239,21 @@ void BoundNodeVisitor::VisitBoundQueryNode(BoundQueryNode &node) {
 	}
 }
 
+class LogicalBoundNodeVisitor : public LogicalOperatorVisitor {
+public:
+	explicit LogicalBoundNodeVisitor(BoundNodeVisitor &parent) : parent(parent) {
+	}
+
+	void VisitExpression(unique_ptr<Expression> *expression) override {
+		auto &expr = **expression;
+		parent.VisitExpression(*expression);
+		VisitExpressionChildren(expr);
+	}
+
+protected:
+	BoundNodeVisitor &parent;
+};
+
 void BoundNodeVisitor::VisitBoundTableRef(BoundTableRef &ref) {
 	switch (ref.type) {
 	case TableReferenceType::EXPRESSION_LIST: {
@@ -264,7 +279,17 @@ void BoundNodeVisitor::VisitBoundTableRef(BoundTableRef &ref) {
 		VisitBoundQueryNode(*bound_subquery.subquery);
 		break;
 	}
-	case TableReferenceType::TABLE_FUNCTION:
+	case TableReferenceType::TABLE_FUNCTION: {
+		auto &bound_table_function = ref.Cast<BoundTableFunction>();
+		LogicalBoundNodeVisitor node_visitor(*this);
+		if (bound_table_function.get) {
+			node_visitor.VisitOperator(*bound_table_function.get);
+		}
+		if (bound_table_function.subquery) {
+			VisitBoundTableRef(*bound_table_function.subquery);
+		}
+		break;
+	}
 	case TableReferenceType::EMPTY_FROM:
 	case TableReferenceType::BASE_TABLE:
 	case TableReferenceType::CTE:

package/src/duckdb/src/planner/filter/conjunction_filter.cpp

@@ -1,4 +1,5 @@
 #include "duckdb/planner/filter/conjunction_filter.hpp"
+#include "duckdb/planner/expression/bound_conjunction_expression.hpp"
 
 namespace duckdb {
 
@@ -46,6 +47,22 @@ bool ConjunctionOrFilter::Equals(const TableFilter &other_p) const {
 	return true;
 }
 
+unique_ptr<TableFilter> ConjunctionOrFilter::Copy() const {
+	auto result = make_uniq<ConjunctionOrFilter>();
+	for (auto &filter : child_filters) {
+		result->child_filters.push_back(filter->Copy());
+	}
+	return std::move(result);
+}
+
+unique_ptr<Expression> ConjunctionOrFilter::ToExpression(const Expression &column) const {
+	auto conjunction = make_uniq<BoundConjunctionExpression>(ExpressionType::CONJUNCTION_OR);
+	for (auto &filter : child_filters) {
+		conjunction->children.push_back(filter->ToExpression(column));
+	}
+	return std::move(conjunction);
+}
+
 ConjunctionAndFilter::ConjunctionAndFilter() : ConjunctionFilter(TableFilterType::CONJUNCTION_AND) {
 }
 
@@ -91,4 +108,20 @@ bool ConjunctionAndFilter::Equals(const TableFilter &other_p) const {
 	return true;
 }
 
+unique_ptr<TableFilter> ConjunctionAndFilter::Copy() const {
+	auto result = make_uniq<ConjunctionAndFilter>();
+	for (auto &filter : child_filters) {
+		result->child_filters.push_back(filter->Copy());
+	}
+	return std::move(result);
+}
+
+unique_ptr<Expression> ConjunctionAndFilter::ToExpression(const Expression &column) const {
+	auto conjunction = make_uniq<BoundConjunctionExpression>(ExpressionType::CONJUNCTION_AND);
+	for (auto &filter : child_filters) {
+		conjunction->children.push_back(filter->ToExpression(column));
+	}
+	return std::move(conjunction);
+}
+
 } // namespace duckdb

package/src/duckdb/src/planner/filter/constant_filter.cpp

@@ -1,11 +1,16 @@
 #include "duckdb/planner/filter/constant_filter.hpp"
 #include "duckdb/storage/statistics/base_statistics.hpp"
+#include "duckdb/planner/expression/bound_comparison_expression.hpp"
+#include "duckdb/planner/expression/bound_constant_expression.hpp"
 
 namespace duckdb {
 
 ConstantFilter::ConstantFilter(ExpressionType comparison_type_p, Value constant_p)
     : TableFilter(TableFilterType::CONSTANT_COMPARISON), comparison_type(comparison_type_p),
       constant(std::move(constant_p)) {
+	if (constant.IsNull()) {
+		throw InternalException("ConstantFilter constant cannot be NULL - use IsNullFilter instead");
+	}
 }
 
 FilterPropagateResult ConstantFilter::CheckStatistics(BaseStatistics &stats) {
@@ -35,6 +40,12 @@ string ConstantFilter::ToString(const string &column_name) {
 	return column_name + ExpressionTypeToOperator(comparison_type) + constant.ToSQLString();
 }
 
+unique_ptr<Expression> ConstantFilter::ToExpression(const Expression &column) const {
+	auto bound_constant = make_uniq<BoundConstantExpression>(constant);
+	auto result = make_uniq<BoundComparisonExpression>(comparison_type, column.Copy(), std::move(bound_constant));
+	return std::move(result);
+}
+
 bool ConstantFilter::Equals(const TableFilter &other_p) const {
 	if (!TableFilter::Equals(other_p)) {
 		return false;
@@ -43,4 +54,8 @@ bool ConstantFilter::Equals(const TableFilter &other_p) const {
 	return other.comparison_type == comparison_type && other.constant == constant;
 }
 
+unique_ptr<TableFilter> ConstantFilter::Copy() const {
+	return make_uniq<ConstantFilter>(comparison_type, constant);
+}
+
 } // namespace duckdb

package/src/duckdb/src/planner/filter/null_filter.cpp

@@ -1,4 +1,6 @@
 #include "duckdb/planner/filter/null_filter.hpp"
+
+#include "duckdb/planner/expression/bound_operator_expression.hpp"
 #include "duckdb/storage/statistics/base_statistics.hpp"
 
 namespace duckdb {
@@ -22,6 +24,16 @@ string IsNullFilter::ToString(const string &column_name) {
 	return column_name + "IS NULL";
 }
 
+unique_ptr<TableFilter> IsNullFilter::Copy() const {
+	return make_uniq<IsNullFilter>();
+}
+
+unique_ptr<Expression> IsNullFilter::ToExpression(const Expression &column) const {
+	auto result = make_uniq<BoundOperatorExpression>(ExpressionType::OPERATOR_IS_NULL, LogicalType::BOOLEAN);
+	result->children.push_back(column.Copy());
+	return std::move(result);
+}
+
 IsNotNullFilter::IsNotNullFilter() : TableFilter(TableFilterType::IS_NOT_NULL) {
 }
 
@@ -41,4 +53,14 @@ string IsNotNullFilter::ToString(const string &column_name) {
 	return column_name + " IS NOT NULL";
 }
 
+unique_ptr<TableFilter> IsNotNullFilter::Copy() const {
+	return make_uniq<IsNotNullFilter>();
+}
+
+unique_ptr<Expression> IsNotNullFilter::ToExpression(const Expression &column) const {
+	auto result = make_uniq<BoundOperatorExpression>(ExpressionType::OPERATOR_IS_NOT_NULL, LogicalType::BOOLEAN);
+	result->children.push_back(column.Copy());
+	return std::move(result);
+}
+
 } // namespace duckdb

package/src/duckdb/src/planner/filter/struct_filter.cpp

@@ -2,6 +2,9 @@
 #include "duckdb/storage/statistics/base_statistics.hpp"
 #include "duckdb/storage/statistics/struct_stats.hpp"
 #include "duckdb/common/string_util.hpp"
+#include "duckdb/planner/expression/bound_constant_expression.hpp"
+#include "duckdb/planner/expression/bound_function_expression.hpp"
+#include "duckdb/function/scalar/nested_functions.hpp"
 
 namespace duckdb {
 
@@ -30,4 +33,17 @@ bool StructFilter::Equals(const TableFilter &other_p) const {
 	       other.child_filter->Equals(*child_filter);
 }
 
+unique_ptr<TableFilter> StructFilter::Copy() const {
+	return make_uniq<StructFilter>(child_idx, child_name, child_filter->Copy());
+}
+
+unique_ptr<Expression> StructFilter::ToExpression(const Expression &column) const {
+	auto &child_type = StructType::GetChildType(column.return_type, child_idx);
+	vector<unique_ptr<Expression>> arguments;
+	arguments.push_back(column.Copy());
+	arguments.push_back(make_uniq<BoundConstantExpression>(Value::BIGINT(NumericCast<int64_t>(child_idx))));
+	auto child = make_uniq<BoundFunctionExpression>(child_type, StructExtractFun::IndexExtractFunction(),
+	                                                std::move(arguments), StructExtractFun::GetBindData(child_idx));
+	return child_filter->ToExpression(*child);
+}
 } // namespace duckdb

package/src/duckdb/src/planner/logical_operator.cpp

@@ -26,6 +26,17 @@ vector<ColumnBinding> LogicalOperator::GetColumnBindings() {
 	return {ColumnBinding(0, 0)};
 }
 
+void LogicalOperator::SetParamsEstimatedCardinality(InsertionOrderPreservingMap<string> &result) const {
+	if (has_estimated_cardinality) {
+		result[RenderTreeNode::ESTIMATED_CARDINALITY] = StringUtil::Format("%llu", estimated_cardinality);
+	}
+}
+
+void LogicalOperator::SetEstimatedCardinality(idx_t _estimated_cardinality) {
+	estimated_cardinality = _estimated_cardinality;
+	has_estimated_cardinality = true;
+}
+
 // LCOV_EXCL_START
 string LogicalOperator::ColumnBindingsToString(const vector<ColumnBinding> &bindings) {
 	string result = "{";
@@ -47,14 +58,17 @@ string LogicalOperator::GetName() const {
 	return LogicalOperatorToString(type);
 }
 
-string LogicalOperator::ParamsToString() const {
-	string result;
+InsertionOrderPreservingMap<string> LogicalOperator::ParamsToString() const {
+	InsertionOrderPreservingMap<string> result;
+	string expressions_info;
 	for (idx_t i = 0; i < expressions.size(); i++) {
 		if (i > 0) {
-			result += "\n";
+			expressions_info += "\n";
 		}
-		result += expressions[i]->GetName();
+		expressions_info += expressions[i]->GetName();
 	}
+	result["Expressions"] = expressions_info;
+	SetParamsEstimatedCardinality(result);
 	return result;
 }
 
@@ -107,9 +121,12 @@ vector<ColumnBinding> LogicalOperator::MapBindings(const vector<ColumnBinding> &
 	}
 }
 
-string LogicalOperator::ToString() const {
-	TreeRenderer renderer;
-	return renderer.ToString(*this);
+string LogicalOperator::ToString(ExplainFormat format) const {
+	auto renderer = TreeRenderer::CreateRenderer(format);
+	stringstream ss;
+	auto tree = RenderTree::CreateRenderTree(*this);
+	renderer->ToStream(*tree, ss);
+	return ss.str();
 }
 
 void LogicalOperator::Verify(ClientContext &context) {

package/src/duckdb/src/planner/operator/logical_aggregate.cpp

@@ -40,20 +40,26 @@ vector<ColumnBinding> LogicalAggregate::GetColumnBindings() {
 	return result;
 }
 
-string LogicalAggregate::ParamsToString() const {
-	string result;
+InsertionOrderPreservingMap<string> LogicalAggregate::ParamsToString() const {
+	InsertionOrderPreservingMap<string> result;
+	string groups_info;
 	for (idx_t i = 0; i < groups.size(); i++) {
 		if (i > 0) {
-			result += "\n";
+			groups_info += "\n";
 		}
-		result += groups[i]->GetName();
+		groups_info += groups[i]->GetName();
 	}
+	result["Groups"] = groups_info;
+
+	string expressions_info;
 	for (idx_t i = 0; i < expressions.size(); i++) {
-		if (i > 0 || !groups.empty()) {
-			result += "\n";
+		if (i > 0) {
+			expressions_info += "\n";
 		}
-		result += expressions[i]->GetName();
+		expressions_info += expressions[i]->GetName();
 	}
+	result["Expressions"] = expressions_info;
+	SetParamsEstimatedCardinality(result);
 	return result;
 }
 

package/src/duckdb/src/planner/operator/logical_any_join.cpp

@@ -5,8 +5,11 @@ namespace duckdb {
 LogicalAnyJoin::LogicalAnyJoin(JoinType type) : LogicalJoin(type, LogicalOperatorType::LOGICAL_ANY_JOIN) {
 }
 
-string LogicalAnyJoin::ParamsToString() const {
-	return condition->ToString();
+InsertionOrderPreservingMap<string> LogicalAnyJoin::ParamsToString() const {
+	InsertionOrderPreservingMap<string> result;
+	result["Condition"] = condition->ToString();
+	SetParamsEstimatedCardinality(result);
+	return result;
 }
 
 } // namespace duckdb

package/src/duckdb/src/planner/operator/logical_comparison_join.cpp

@@ -8,14 +8,22 @@ LogicalComparisonJoin::LogicalComparisonJoin(JoinType join_type, LogicalOperator
     : LogicalJoin(join_type, logical_type) {
 }
 
-string LogicalComparisonJoin::ParamsToString() const {
-	string result = EnumUtil::ToChars(join_type);
-	for (auto &condition : conditions) {
-		result += "\n";
+InsertionOrderPreservingMap<string> LogicalComparisonJoin::ParamsToString() const {
+	InsertionOrderPreservingMap<string> result;
+	result["Join Type"] = EnumUtil::ToChars(join_type);
+
+	string conditions_info;
+	for (idx_t i = 0; i < conditions.size(); i++) {
+		if (i > 0) {
+			conditions_info += "\n";
+		}
+		auto &condition = conditions[i];
 		auto expr =
 		    make_uniq<BoundComparisonExpression>(condition.comparison, condition.left->Copy(), condition.right->Copy());
-		result += expr->ToString();
+		conditions_info += expr->ToString();
 	}
+	result["Conditions"] = conditions_info;
+	SetParamsEstimatedCardinality(result);
 
 	return result;
 }

package/src/duckdb/src/planner/operator/logical_copy_to_file.cpp

@@ -8,6 +8,36 @@
 
 namespace duckdb {
 
+vector<LogicalType> LogicalCopyToFile::GetTypesWithoutPartitions(const vector<LogicalType> &col_types,
+                                                                 const vector<idx_t> &part_cols, bool write_part_cols) {
+	if (write_part_cols || part_cols.empty()) {
+		return col_types;
+	}
+	vector<LogicalType> types;
+	set<idx_t> part_col_set(part_cols.begin(), part_cols.end());
+	for (idx_t col_idx = 0; col_idx < col_types.size(); col_idx++) {
+		if (part_col_set.find(col_idx) == part_col_set.end()) {
+			types.push_back(col_types[col_idx]);
+		}
+	}
+	return types;
+}
+
+vector<string> LogicalCopyToFile::GetNamesWithoutPartitions(const vector<string> &col_names,
+                                                            const vector<column_t> &part_cols, bool write_part_cols) {
+	if (write_part_cols || part_cols.empty()) {
+		return col_names;
+	}
+	vector<string> names;
+	set<idx_t> part_col_set(part_cols.begin(), part_cols.end());
+	for (idx_t col_idx = 0; col_idx < col_names.size(); col_idx++) {
+		if (part_col_set.find(col_idx) == part_col_set.end()) {
+			names.push_back(col_names[col_idx]);
+		}
+	}
+	return names;
+}
+
 void LogicalCopyToFile::Serialize(Serializer &serializer) const {
 	LogicalOperator::Serialize(serializer);
 	serializer.WriteProperty(200, "file_path", file_path);
@@ -33,6 +63,9 @@ void LogicalCopyToFile::Serialize(Serializer &serializer) const {
 	}
 
 	serializer.WriteProperty(213, "file_extension", file_extension);
+	serializer.WriteProperty(214, "rotate", rotate);
+	serializer.WriteProperty(215, "return_type", return_type);
+	serializer.WriteProperty(216, "write_partition_columns", write_partition_columns);
 }
 
 unique_ptr<LogicalOperator> LogicalCopyToFile::Deserialize(Deserializer &deserializer) {
@@ -66,21 +99,30 @@ unique_ptr<LogicalOperator> LogicalCopyToFile::Deserialize(Deserializer &deseria
 		// Just deserialize the bind data
 		deserializer.ReadObject(212, "function_data",
 		                        [&](Deserializer &obj) { bind_data = function.deserialize(obj, function); });
-	} else {
-		// Otherwise, re-bind with the copy info
+	}
+
+	auto default_extension = function.extension;
+
+	auto file_extension =
+	    deserializer.ReadPropertyWithExplicitDefault<string>(213, "file_extension", std::move(default_extension));
+
+	auto rotate = deserializer.ReadPropertyWithExplicitDefault(214, "rotate", false);
+	auto return_type =
+	    deserializer.ReadPropertyWithExplicitDefault(215, "return_type", CopyFunctionReturnType::CHANGED_ROWS);
+	auto write_partition_columns = deserializer.ReadProperty<bool>(216, "write_partition_columns");
+
+	if (!has_serialize) {
+		// If not serialized, re-bind with the copy info
 		if (!function.copy_to_bind) {
 			throw InternalException("Copy function \"%s\" has neither bind nor (de)serialize", function.name);
 		}
 
 		CopyFunctionBindInput function_bind_input(*copy_info);
-		bind_data = function.copy_to_bind(context, function_bind_input, names, expected_types);
+		auto names_to_write = GetNamesWithoutPartitions(names, partition_columns, write_partition_columns);
+		auto types_to_write = GetTypesWithoutPartitions(expected_types, partition_columns, write_partition_columns);
+		bind_data = function.copy_to_bind(context, function_bind_input, names_to_write, types_to_write);
 	}
 
-	auto default_extension = function.extension;
-
-	auto file_extension =
-	    deserializer.ReadPropertyWithDefault<string>(213, "file_extension", std::move(default_extension));
-
 	auto result = make_uniq<LogicalCopyToFile>(function, std::move(bind_data), std::move(copy_info));
 	result->file_path = file_path;
 	result->use_tmp_file = use_tmp_file;
@@ -92,10 +134,24 @@ unique_ptr<LogicalOperator> LogicalCopyToFile::Deserialize(Deserializer &deseria
 	result->partition_columns = partition_columns;
 	result->names = names;
 	result->expected_types = expected_types;
+	result->rotate = rotate;
+	result->return_type = return_type;
+	result->write_partition_columns = write_partition_columns;
 
 	return std::move(result);
 }
 
+vector<ColumnBinding> LogicalCopyToFile::GetColumnBindings() {
+	switch (return_type) {
+	case CopyFunctionReturnType::CHANGED_ROWS:
+		return {ColumnBinding(0, 0)};
+	case CopyFunctionReturnType::CHANGED_ROWS_AND_FILE_LIST:
+		return {ColumnBinding(0, 0), ColumnBinding(0, 1)};
+	default:
+		throw NotImplementedException("Unknown CopyFunctionReturnType");
+	}
+}
+
 idx_t LogicalCopyToFile::EstimateCardinality(ClientContext &context) {
 	return 1;
 }

package/src/duckdb/src/planner/operator/logical_cteref.cpp

@@ -4,6 +4,13 @@
 
 namespace duckdb {
 
+InsertionOrderPreservingMap<string> LogicalCTERef::ParamsToString() const {
+	InsertionOrderPreservingMap<string> result;
+	result["CTE Index"] = StringUtil::Format("%llu", cte_index);
+	SetParamsEstimatedCardinality(result);
+	return result;
+}
+
 vector<idx_t> LogicalCTERef::GetTableIndex() const {
 	return vector<idx_t> {table_index};
 }