duckdb 0.10.3-dev0.0 → 0.10.3-dev6.0

package/src/duckdb/src/parser/transform/statement/transform_update.cpp

@@ -1,4 +1,5 @@
 #include "duckdb/parser/statement/update_statement.hpp"
+#include "duckdb/parser/statement/update_extensions_statement.hpp"
 #include "duckdb/parser/transformer.hpp"
 
 namespace duckdb {
@@ -38,4 +39,22 @@ unique_ptr<UpdateStatement> Transformer::TransformUpdate(duckdb_libpgquery::PGUp
 	return result;
 }
 
+unique_ptr<UpdateExtensionsStatement>
+Transformer::TransformUpdateExtensions(duckdb_libpgquery::PGUpdateExtensionsStmt &stmt) {
+	auto result = make_uniq<UpdateExtensionsStatement>();
+	auto info = make_uniq<UpdateExtensionsInfo>();
+
+	if (stmt.extensions) {
+		auto column_list = PGPointerCast<duckdb_libpgquery::PGList>(stmt.extensions);
+		for (auto c = column_list->head; c != nullptr; c = lnext(c)) {
+			auto extension = reinterpret_cast<duckdb_libpgquery::PGValue *>(c->data.ptr_value)->val.str;
+			info->extensions_to_update.emplace_back(extension);
+		}
+	}
+
+	result->info = std::move(info);
+
+	return result;
+}
+
 } // namespace duckdb

package/src/duckdb/src/parser/transformer.cpp

@@ -134,8 +134,8 @@ unique_ptr<SQLStatement> Transformer::TransformStatementInternal(duckdb_libpgque
 		auto &raw_stmt = PGCast<duckdb_libpgquery::PGRawStmt>(stmt);
 		auto result = TransformStatement(*raw_stmt.stmt);
 		if (result) {
-			result->stmt_location = raw_stmt.stmt_location;
-			result->stmt_length = raw_stmt.stmt_len;
+			result->stmt_location = NumericCast<idx_t>(raw_stmt.stmt_location);
+			result->stmt_length = NumericCast<idx_t>(raw_stmt.stmt_len);
 		}
 		return result;
 	}
@@ -163,6 +163,8 @@ unique_ptr<SQLStatement> Transformer::TransformStatementInternal(duckdb_libpgque
 		return TransformDelete(PGCast<duckdb_libpgquery::PGDeleteStmt>(stmt));
 	case duckdb_libpgquery::T_PGUpdateStmt:
 		return TransformUpdate(PGCast<duckdb_libpgquery::PGUpdateStmt>(stmt));
+	case duckdb_libpgquery::T_PGUpdateExtensionsStmt:
+		return TransformUpdateExtensions(PGCast<duckdb_libpgquery::PGUpdateExtensionsStmt>(stmt));
 	case duckdb_libpgquery::T_PGIndexStmt:
 		return TransformCreateIndex(PGCast<duckdb_libpgquery::PGIndexStmt>(stmt));
 	case duckdb_libpgquery::T_PGAlterTableStmt:
@@ -225,6 +227,7 @@ unique_ptr<SQLStatement> Transformer::TransformStatementInternal(duckdb_libpgque
 unique_ptr<QueryNode> Transformer::TransformMaterializedCTE(unique_ptr<QueryNode> root) {
 	// Extract materialized CTEs from cte_map
 	vector<unique_ptr<CTENode>> materialized_ctes;
+
 	for (auto &cte : root->cte_map.map) {
 		auto &cte_entry = cte.second;
 		if (cte_entry->materialized == CTEMaterialize::CTE_MATERIALIZE_ALWAYS) {

package/src/duckdb/src/planner/bind_context.cpp

@@ -514,13 +514,13 @@ void BindContext::AddGenericBinding(idx_t index, const string &alias, const vect
 
 void BindContext::AddCTEBinding(idx_t index, const string &alias, const vector<string> &names,
                                 const vector<LogicalType> &types) {
-	auto binding = make_shared<Binding>(BindingType::BASE, alias, types, names, index);
+	auto binding = make_shared_ptr<Binding>(BindingType::BASE, alias, types, names, index);
 
 	if (cte_bindings.find(alias) != cte_bindings.end()) {
 		throw BinderException("Duplicate alias \"%s\" in query!", alias);
 	}
 	cte_bindings[alias] = std::move(binding);
-	cte_references[alias] = std::make_shared<idx_t>(0);
+	cte_references[alias] = make_shared_ptr<idx_t>(0);
 }
 
 void BindContext::AddContext(BindContext other) {

package/src/duckdb/src/planner/binder/expression/bind_aggregate_expression.cpp

@@ -6,6 +6,8 @@
 #include "duckdb/planner/expression/bound_cast_expression.hpp"
 #include "duckdb/planner/expression/bound_columnref_expression.hpp"
 #include "duckdb/planner/expression/bound_constant_expression.hpp"
+#include "duckdb/planner/expression/bound_function_expression.hpp"
+#include "duckdb/planner/expression_iterator.hpp"
 #include "duckdb/planner/expression_binder/aggregate_binder.hpp"
 #include "duckdb/planner/expression_binder/base_select_binder.hpp"
 #include "duckdb/planner/query_node/bound_select_node.hpp"
@@ -17,6 +19,18 @@
 
 namespace duckdb {
 
+static bool ExtractFunctionalDependencies(column_binding_set_t &deps, const unique_ptr<Expression> &expr) {
+	if (expr->type == ExpressionType::BOUND_COLUMN_REF) {
+		auto &colref = expr->Cast<BoundColumnRefExpression>();
+		deps.insert(colref.binding);
+	}
+
+	bool is_volatile = expr->IsVolatile();
+	ExpressionIterator::EnumerateChildren(
+	    *expr, [&](unique_ptr<Expression> &child) { is_volatile |= ExtractFunctionalDependencies(deps, child); });
+	return is_volatile;
+}
+
 static Value NegatePercentileValue(const Value &v, const bool desc) {
 	if (v.IsNull()) {
 		return v;
@@ -233,19 +247,32 @@ BindResult BaseSelectBinder::BindAggregate(FunctionExpression &aggr, AggregateFu
 		}
 	}
 
-	// If the aggregate is DISTINCT then the ORDER BYs need to be arguments.
+	// If the aggregate is DISTINCT then the ORDER BYs need to be functional dependencies of the arguments.
 	if (aggr.distinct && order_bys) {
+		column_binding_set_t child_dependencies;
+		bool children_volatile = false;
+		for (const auto &child : children) {
+			children_volatile |= ExtractFunctionalDependencies(child_dependencies, child);
+		}
+
+		column_binding_set_t order_dependencies;
+		bool order_volatile = false;
 		for (const auto &order_by : order_bys->orders) {
-			bool is_arg = false;
-			for (const auto &child : children) {
-				if (order_by.expression->Equals(*child)) {
-					is_arg = true;
+			order_volatile |= ExtractFunctionalDependencies(order_dependencies, order_by.expression);
+		}
+
+		bool in_args = !children_volatile && !order_volatile;
+		if (in_args) {
+			for (const auto &binding : order_dependencies) {
+				if (!child_dependencies.count(binding)) {
+					in_args = false;
 					break;
 				}
 			}
-			if (!is_arg) {
-				throw BinderException("In a DISTINCT aggregate, ORDER BY expressions must appear in the argument list");
-			}
+		}
+
+		if (!in_args) {
+			throw BinderException("In a DISTINCT aggregate, ORDER BY expressions must appear in the argument list");
 		}
 	}
 

package/src/duckdb/src/planner/binder/expression/bind_cast_expression.cpp

@@ -14,7 +14,7 @@ BindResult ExpressionBinder::BindExpression(CastExpression &expr, idx_t depth) {
 	}
 	// FIXME: We can also implement 'hello'::schema.custom_type; and pass by the schema down here.
 	// Right now just considering its DEFAULT_SCHEMA always
-	Binder::BindLogicalType(context, expr.cast_type);
+	binder.BindLogicalType(expr.cast_type);
 	// the children have been successfully resolved
 	auto &child = BoundExpression::GetExpression(*expr.child);
 	if (expr.try_cast) {

package/src/duckdb/src/planner/binder/expression/bind_function_expression.cpp

@@ -39,13 +39,12 @@ BindResult ExpressionBinder::BindExpression(FunctionExpression &function, idx_t
 	// lookup the function in the catalog
 	QueryErrorContext error_context(function.query_location);
 	binder.BindSchemaOrCatalog(function.catalog, function.schema);
-	auto func = Catalog::GetEntry(context, CatalogType::SCALAR_FUNCTION_ENTRY, function.catalog, function.schema,
-	                              function.function_name, OnEntryNotFound::RETURN_NULL, error_context);
+	auto func = GetCatalogEntry(CatalogType::SCALAR_FUNCTION_ENTRY, function.catalog, function.schema,
+	                            function.function_name, OnEntryNotFound::RETURN_NULL, error_context);
 	if (!func) {
 		// function was not found - check if we this is a table function
-		auto table_func =
-		    Catalog::GetEntry(context, CatalogType::TABLE_FUNCTION_ENTRY, function.catalog, function.schema,
-		                      function.function_name, OnEntryNotFound::RETURN_NULL, error_context);
+		auto table_func = GetCatalogEntry(CatalogType::TABLE_FUNCTION_ENTRY, function.catalog, function.schema,
+		                                  function.function_name, OnEntryNotFound::RETURN_NULL, error_context);
 		if (table_func) {
 			throw BinderException(function,
 			                      "Function \"%s\" is a table function but it was used as a scalar function. This "
@@ -75,8 +74,8 @@ BindResult ExpressionBinder::BindExpression(FunctionExpression &function, idx_t
 			}
 		}
 		// rebind the function
-		func = Catalog::GetEntry(context, CatalogType::SCALAR_FUNCTION_ENTRY, function.catalog, function.schema,
-		                         function.function_name, OnEntryNotFound::THROW_EXCEPTION, error_context);
+		func = GetCatalogEntry(CatalogType::SCALAR_FUNCTION_ENTRY, function.catalog, function.schema,
+		                       function.function_name, OnEntryNotFound::THROW_EXCEPTION, error_context);
 	}
 
 	if (func->type != CatalogType::AGGREGATE_FUNCTION_ENTRY &&
@@ -281,4 +280,11 @@ string ExpressionBinder::UnsupportedUnnestMessage() {
 	return "UNNEST not supported here";
 }
 
+optional_ptr<CatalogEntry> ExpressionBinder::GetCatalogEntry(CatalogType type, const string &catalog,
+                                                             const string &schema, const string &name,
+                                                             OnEntryNotFound on_entry_not_found,
+                                                             QueryErrorContext &error_context) {
+	return binder.GetCatalogEntry(type, catalog, schema, name, on_entry_not_found, error_context);
+}
+
 } // namespace duckdb

package/src/duckdb/src/planner/binder/query_node/bind_select_node.cpp

@@ -45,7 +45,7 @@ unique_ptr<Expression> Binder::BindOrderExpression(OrderBinder &order_binder, un
 
 BoundLimitNode Binder::BindLimitValue(OrderBinder &order_binder, unique_ptr<ParsedExpression> limit_val,
                                       bool is_percentage, bool is_offset) {
-	auto new_binder = Binder::CreateBinder(context, this, true);
+	auto new_binder = Binder::CreateBinder(context, this);
 	ExpressionBinder expr_binder(*new_binder, context);
 	auto target_type = is_percentage ? LogicalType::DOUBLE : LogicalType::BIGINT;
 	expr_binder.target_type = target_type;

package/src/duckdb/src/planner/binder/query_node/plan_setop.cpp

@@ -4,6 +4,8 @@
 #include "duckdb/planner/operator/logical_projection.hpp"
 #include "duckdb/planner/operator/logical_set_operation.hpp"
 #include "duckdb/planner/query_node/bound_set_operation_node.hpp"
+#include "duckdb/function/table/read_csv.hpp"
+#include "duckdb/planner/operator/logical_get.hpp"
 
 namespace duckdb {
 
@@ -23,14 +25,45 @@ unique_ptr<LogicalOperator> Binder::CastLogicalOperatorToTypes(vector<LogicalTyp
 	if (node->type == LogicalOperatorType::LOGICAL_PROJECTION) {
 		// "node" is a projection; we can just do the casts in there
 		D_ASSERT(node->expressions.size() == source_types.size());
+		if (node->children.size() == 1 && node->children[0]->type == LogicalOperatorType::LOGICAL_GET) {
+			// If this projection only has one child and that child is a logical get we can try to pushdown types
+			auto &logical_get = node->children[0]->Cast<LogicalGet>();
+			if (logical_get.function.type_pushdown) {
+				unordered_map<idx_t, LogicalType> new_column_types;
+				bool do_pushdown = true;
+				for (idx_t i = 0; i < op->expressions.size(); i++) {
+					if (op->expressions[i]->type == ExpressionType::BOUND_COLUMN_REF) {
+						auto &col_ref = op->expressions[i]->Cast<BoundColumnRefExpression>();
+						if (new_column_types.find(logical_get.column_ids[col_ref.binding.column_index]) !=
+						    new_column_types.end()) {
+							// Only one reference per column is accepted
+							do_pushdown = false;
+							break;
+						}
+						new_column_types[logical_get.column_ids[col_ref.binding.column_index]] = target_types[i];
+					} else {
+						do_pushdown = false;
+						break;
+					}
+				}
+				if (do_pushdown) {
+					logical_get.function.type_pushdown(context, logical_get.bind_data, new_column_types);
+					// We also have to modify the types to the logical_get.returned_types
+					for (auto &type : new_column_types) {
+						logical_get.returned_types[type.first] = type.second;
+					}
+					return op;
+				}
+			}
+		}
 		// add the casts to the selection list
 		for (idx_t i = 0; i < target_types.size(); i++) {
 			if (source_types[i] != target_types[i]) {
 				// differing types, have to add a cast
-				string alias = node->expressions[i]->alias;
+				string cur_alias = node->expressions[i]->alias;
 				node->expressions[i] =
 				    BoundCastExpression::AddCastToType(context, std::move(node->expressions[i]), target_types[i]);
-				node->expressions[i]->alias = alias;
+				node->expressions[i]->alias = cur_alias;
 			}
 		}
 		return op;

package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp

@@ -6,7 +6,6 @@
 #include "duckdb/planner/expression/bound_columnref_expression.hpp"
 #include "duckdb/planner/expression/bound_comparison_expression.hpp"
 #include "duckdb/planner/expression/bound_constant_expression.hpp"
-#include "duckdb/planner/expression/bound_reference_expression.hpp"
 #include "duckdb/planner/expression/bound_subquery_expression.hpp"
 #include "duckdb/planner/expression/bound_window_expression.hpp"
 #include "duckdb/planner/expression_iterator.hpp"
@@ -16,7 +15,6 @@
 #include "duckdb/planner/subquery/flatten_dependent_join.hpp"
 #include "duckdb/common/enums/logical_operator_type.hpp"
 #include "duckdb/planner/operator/logical_dependent_join.hpp"
-#include "duckdb/planner/expression_binder/lateral_binder.hpp"
 #include "duckdb/planner/subquery/recursive_dependent_join_planner.hpp"
 
 namespace duckdb {

package/src/duckdb/src/planner/binder/statement/bind_attach.cpp

@@ -12,6 +12,8 @@ BoundStatement Binder::Bind(AttachStatement &stmt) {
 	result.names = {"Success"};
 
 	result.plan = make_uniq<LogicalSimple>(LogicalOperatorType::LOGICAL_ATTACH, std::move(stmt.info));
+
+	auto &properties = GetStatementProperties();
 	properties.allow_stream_result = false;
 	properties.return_type = StatementReturnType::NOTHING;
 	return result;

package/src/duckdb/src/planner/binder/statement/bind_call.cpp

@@ -24,6 +24,8 @@ BoundStatement Binder::Bind(CallStatement &stmt) {
 	result.types = get.returned_types;
 	result.names = get.names;
 	result.plan = CreatePlan(*bound_func);
+
+	auto &properties = GetStatementProperties();
 	properties.return_type = StatementReturnType::QUERY_RESULT;
 	return result;
 }

package/src/duckdb/src/planner/binder/statement/bind_copy.cpp

@@ -45,8 +45,10 @@ BoundStatement Binder::BindCopyTo(CopyStatement &stmt) {
 		return copy_function.function.plan(*this, stmt);
 	}
 
+	auto &copy_info = *stmt.info;
 	// bind the select statement
-	auto select_node = Bind(*stmt.select_statement);
+	auto node_copy = copy_info.select_statement->Copy();
+	auto select_node = Bind(*node_copy);
 
 	if (!copy_function.function.copy_to_bind) {
 		throw NotImplementedException("COPY TO is not supported for FORMAT \"%s\"", stmt.info->format);
@@ -229,7 +231,7 @@ BoundStatement Binder::BindCopyFrom(CopyStatement &stmt) {
 }
 
 BoundStatement Binder::Bind(CopyStatement &stmt) {
-	if (!stmt.info->is_from && !stmt.select_statement) {
+	if (!stmt.info->is_from && !stmt.info->select_statement) {
 		// copy table into file without a query
 		// generate SELECT * FROM table;
 		auto ref = make_uniq<BaseTableRef>();
@@ -246,8 +248,10 @@ BoundStatement Binder::Bind(CopyStatement &stmt) {
 		} else {
 			statement->select_list.push_back(make_uniq<StarExpression>());
 		}
-		stmt.select_statement = std::move(statement);
+		stmt.info->select_statement = std::move(statement);
 	}
+
+	auto &properties = GetStatementProperties();
 	properties.allow_stream_result = false;
 	properties.return_type = StatementReturnType::CHANGED_ROWS;
 	if (stmt.info->is_from) {

package/src/duckdb/src/planner/binder/statement/bind_copy_database.cpp

@@ -15,81 +15,31 @@
 #include "duckdb/planner/expression/bound_constant_expression.hpp"
 #include "duckdb/planner/operator/logical_dummy_scan.hpp"
 #include "duckdb/planner/operator/logical_expression_get.hpp"
+#include "duckdb/catalog/duck_catalog.hpp"
+#include "duckdb/catalog/dependency_manager.hpp"
 
 namespace duckdb {
 
-unique_ptr<LogicalOperator> Binder::BindCopyDatabaseSchema(Catalog &source_catalog,
-                                                           const string &target_database_name) {
-	auto source_schemas = source_catalog.GetSchemas(context);
+unique_ptr<LogicalOperator> Binder::BindCopyDatabaseSchema(Catalog &from_database, const string &target_database_name) {
 
-	ExportEntries entries;
-	PhysicalExport::ExtractEntries(context, source_schemas, entries);
+	catalog_entry_vector_t catalog_entries;
+	catalog_entries = PhysicalExport::GetNaiveExportOrder(context, from_database);
 
 	auto info = make_uniq<CopyDatabaseInfo>(target_database_name);
-
-	// get a list of all schemas to copy over
-	for (auto &schema_ref : source_schemas) {
-		auto &schema = schema_ref.get().Cast<SchemaCatalogEntry>();
-		if (schema.internal) {
-			continue;
-		}
-		auto create_info = schema.GetInfo();
-		create_info->catalog = target_database_name;
-		create_info->on_conflict = OnCreateConflict::IGNORE_ON_CONFLICT;
-		info->entries.push_back(std::move(create_info));
-	}
-	// get a list of all types to copy over
-	for (auto &seq_ref : entries.sequences) {
-		auto &seq_entry = seq_ref.get().Cast<SequenceCatalogEntry>();
-		if (seq_entry.internal) {
-			continue;
-		}
-		auto create_info = seq_entry.GetInfo();
-		create_info->catalog = target_database_name;
-		create_info->on_conflict = OnCreateConflict::ERROR_ON_CONFLICT;
-		info->entries.push_back(std::move(create_info));
-	}
-	// get a list of all types to copy over
-	for (auto &type_ref : entries.custom_types) {
-		auto &type_entry = type_ref.get().Cast<TypeCatalogEntry>();
-		if (type_entry.internal) {
-			continue;
-		}
-		auto create_info = type_entry.GetInfo();
+	for (auto &entry : catalog_entries) {
+		auto create_info = entry.get().GetInfo();
 		create_info->catalog = target_database_name;
-		create_info->on_conflict = OnCreateConflict::ERROR_ON_CONFLICT;
-		info->entries.push_back(std::move(create_info));
-	}
-	// get a list of all tables to copy over
-	for (auto &table_ref : entries.tables) {
-		auto &table = table_ref.get().Cast<TableCatalogEntry>();
-		if (table.internal) {
-			continue;
+		auto on_conflict = create_info->type == CatalogType::SCHEMA_ENTRY ? OnCreateConflict::IGNORE_ON_CONFLICT
+		                                                                  : OnCreateConflict::ERROR_ON_CONFLICT;
+		// Update all the dependencies of the entry to point to the newly created entries on the target database
+		LogicalDependencyList altered_dependencies;
+		for (auto &dep : create_info->dependencies.Set()) {
+			auto altered_dep = dep;
+			altered_dep.catalog = target_database_name;
+			altered_dependencies.AddDependency(altered_dep);
 		}
-		auto create_info = table.GetInfo();
-		create_info->catalog = target_database_name;
-		create_info->on_conflict = OnCreateConflict::ERROR_ON_CONFLICT;
-		info->entries.push_back(std::move(create_info));
-	}
-	for (auto &macro_ref : entries.macros) {
-		auto &macro = macro_ref.get().Cast<MacroCatalogEntry>();
-		if (macro.internal) {
-			continue;
-		}
-		auto create_info = macro.GetInfo();
-		create_info->catalog = target_database_name;
-		create_info->on_conflict = OnCreateConflict::ERROR_ON_CONFLICT;
-		info->entries.push_back(std::move(create_info));
-	}
-	// get a list of all views to copy over
-	for (auto &view_ref : entries.views) {
-		auto &view = view_ref.get().Cast<ViewCatalogEntry>();
-		if (view.internal) {
-			continue;
-		}
-		auto create_info = view.GetInfo();
-		create_info->catalog = target_database_name;
-		create_info->on_conflict = OnCreateConflict::ERROR_ON_CONFLICT;
+		create_info->dependencies = altered_dependencies;
+		create_info->on_conflict = on_conflict;
 		info->entries.push_back(std::move(create_info));
 	}
 
@@ -100,6 +50,7 @@ unique_ptr<LogicalOperator> Binder::BindCopyDatabaseSchema(Catalog &source_catal
 unique_ptr<LogicalOperator> Binder::BindCopyDatabaseData(Catalog &source_catalog, const string &target_database_name) {
 	auto source_schemas = source_catalog.GetSchemas(context);
 
+	// We can just use ExtractEntries here because the order doesn't matter
 	ExportEntries entries;
 	PhysicalExport::ExtractEntries(context, source_schemas, entries);
 
@@ -130,7 +81,7 @@ unique_ptr<LogicalOperator> Binder::BindCopyDatabaseData(Catalog &source_catalog
 		if (result) {
 			// use UNION ALL to combine the individual copy statements into a single node
 			auto copy_union =
-			    make_uniq<LogicalSetOperation>(GenerateTableIndex(), 1, std::move(insert_plan), std::move(result),
+			    make_uniq<LogicalSetOperation>(GenerateTableIndex(), 1U, std::move(insert_plan), std::move(result),
 			                                   LogicalOperatorType::LOGICAL_UNION, true, false);
 			result = std::move(copy_union);
 		} else {
@@ -173,6 +124,8 @@ BoundStatement Binder::Bind(CopyDatabaseStatement &stmt) {
 	}
 
 	result.plan = std::move(plan);
+
+	auto &properties = GetStatementProperties();
 	properties.allow_stream_result = false;
 	properties.return_type = StatementReturnType::NOTHING;
 	properties.modified_databases.insert(target_catalog.GetName());