duckdb 1.0.1-dev22.0 → 1.0.1-dev27.0

package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp

@@ -2,6 +2,7 @@
 #include "duckdb/main/client_config.hpp"
 #include "duckdb/planner/binder.hpp"
 #include "duckdb/planner/expression/bound_aggregate_expression.hpp"
+#include "duckdb/planner/expression/bound_case_expression.hpp"
 #include "duckdb/planner/expression/bound_cast_expression.hpp"
 #include "duckdb/planner/expression/bound_columnref_expression.hpp"
 #include "duckdb/planner/expression/bound_comparison_expression.hpp"
@@ -16,6 +17,7 @@
 #include "duckdb/common/enums/logical_operator_type.hpp"
 #include "duckdb/planner/operator/logical_dependent_join.hpp"
 #include "duckdb/planner/subquery/recursive_dependent_join_planner.hpp"
+#include "duckdb/core_functions/scalar/generic_functions.hpp"
 
 namespace duckdb {
 
@@ -75,11 +77,8 @@ static unique_ptr<Expression> PlanUncorrelatedSubquery(Binder &binder, BoundSubq
 		D_ASSERT(bindings.size() == 1);
 		idx_t table_idx = bindings[0].table_index;
 
-		// in the uncorrelated case we are only interested in the first result of the query
-		// hence we simply push a LIMIT 1 to get the first row of the subquery
-		auto limit = make_uniq<LogicalLimit>(BoundLimitNode::ConstantValue(1), BoundLimitNode());
-		limit->AddChild(std::move(plan));
-		plan = std::move(limit);
+		auto &config = ClientConfig::GetConfig(binder.context);
+		bool error_on_multiple_rows = config.scalar_subquery_error_on_multiple_rows;
 
 		// we push an aggregate that returns the FIRST element
 		vector<unique_ptr<Expression>> expressions;
@@ -92,11 +91,51 @@ static unique_ptr<Expression> PlanUncorrelatedSubquery(Binder &binder, BoundSubq
 		    FirstFun::GetFunction(expr.return_type), std::move(first_children), nullptr, AggregateType::NON_DISTINCT);
 
 		expressions.push_back(std::move(first_agg));
+		if (error_on_multiple_rows) {
+			vector<unique_ptr<Expression>> count_children;
+			auto count_agg = function_binder.BindAggregateFunction(
+			    CountStarFun::GetFunction(), std::move(count_children), nullptr, AggregateType::NON_DISTINCT);
+			expressions.push_back(std::move(count_agg));
+		}
 		auto aggr_index = binder.GenerateTableIndex();
+
 		auto aggr = make_uniq<LogicalAggregate>(binder.GenerateTableIndex(), aggr_index, std::move(expressions));
 		aggr->AddChild(std::move(plan));
 		plan = std::move(aggr);
 
+		if (error_on_multiple_rows) {
+			// CASE WHEN count > 1 THEN error('Scalar subquery can only return a single row') ELSE first_agg END
+			idx_t proj_index = binder.GenerateTableIndex();
+
+			auto first_ref =
+			    make_uniq<BoundColumnRefExpression>(plan->expressions[0]->return_type, ColumnBinding(aggr_index, 0));
+			auto count_ref =
+			    make_uniq<BoundColumnRefExpression>(plan->expressions[1]->return_type, ColumnBinding(aggr_index, 1));
+
+			auto constant_one = make_uniq<BoundConstantExpression>(Value::BIGINT(1));
+			auto count_check = make_uniq<BoundComparisonExpression>(ExpressionType::COMPARE_GREATERTHAN,
+			                                                        std::move(count_ref), std::move(constant_one));
+
+			vector<unique_ptr<Expression>> error_children;
+			error_children.push_back(make_uniq<BoundConstantExpression>(
+			    Value("More than one row returned by a subquery used as an expression - scalar subqueries can only "
+			          "return a single row.\n\nUse \"SET scalar_subquery_error_on_multiple_rows=false\" to revert to "
+			          "previous behavior of returning a random row.")));
+			auto error_expr = function_binder.BindScalarFunction(ErrorFun::GetFunction(), std::move(error_children));
+			error_expr->return_type = first_ref->return_type;
+			auto case_expr =
+			    make_uniq<BoundCaseExpression>(std::move(count_check), std::move(error_expr), std::move(first_ref));
+
+			vector<unique_ptr<Expression>> proj_expressions;
+			proj_expressions.push_back(std::move(case_expr));
+
+			auto proj = make_uniq<LogicalProjection>(proj_index, std::move(proj_expressions));
+			proj->AddChild(std::move(plan));
+			plan = std::move(proj);
+
+			aggr_index = proj_index;
+		}
+
 		// in the uncorrelated case, we add the value to the main query through a cross product
 		// FIXME: should use something else besides cross product as we always add only one scalar constant and cross
 		// product is not optimized for this.
@@ -397,7 +436,7 @@ void Binder::PlanSubqueries(unique_ptr<Expression> &expr_ptr, unique_ptr<Logical
 	if (expr.expression_class == ExpressionClass::BOUND_SUBQUERY) {
 		auto &subquery = expr.Cast<BoundSubqueryExpression>();
 		// subquery node! plan it
-		if (subquery.IsCorrelated() && !is_outside_flattened) {
+		if (!is_outside_flattened) {
 			// detected a nested correlated subquery
 			// we don't plan it yet here, we are currently planning a subquery
 			// nested subqueries will only be planned AFTER the current subquery has been flattened entirely
@@ -429,7 +468,7 @@ unique_ptr<LogicalOperator> Binder::PlanLateralJoin(unique_ptr<LogicalOperator>
 	// first we check which logical operators have correlated expressions in the first place
 	flatten.DetectCorrelatedExpressions(*right, true);
 	// now we push the dependent join down
-	auto dependent_join = flatten.PushDownDependentJoin(std::move(right));
+	auto dependent_join = flatten.PushDownDependentJoin(std::move(right), join_type != JoinType::INNER);
 
 	// now the dependent join is fully eliminated
 	// we only need to create the join conditions between the LHS and the RHS

package/src/duckdb/src/planner/binder/statement/bind_call.cpp

@@ -1,32 +1,25 @@
-#include "duckdb/planner/binder.hpp"
 #include "duckdb/parser/statement/call_statement.hpp"
 #include "duckdb/parser/tableref/table_function_ref.hpp"
-#include "duckdb/planner/tableref/bound_table_function.hpp"
+#include "duckdb/planner/binder.hpp"
 #include "duckdb/planner/operator/logical_get.hpp"
+#include "duckdb/planner/tableref/bound_table_function.hpp"
+#include "duckdb/parser/query_node/select_node.hpp"
+#include "duckdb/parser/expression/star_expression.hpp"
 
 namespace duckdb {
 
 BoundStatement Binder::Bind(CallStatement &stmt) {
-	BoundStatement result;
-
-	TableFunctionRef ref;
-	ref.function = std::move(stmt.function);
-
-	auto bound_func = Bind(ref);
-	auto &bound_table_func = bound_func->Cast<BoundTableFunction>();
-	;
-	auto &get = bound_table_func.get->Cast<LogicalGet>();
-	D_ASSERT(get.returned_types.size() > 0);
-	for (idx_t i = 0; i < get.returned_types.size(); i++) {
-		get.column_ids.push_back(i);
-	}
-
-	result.types = get.returned_types;
-	result.names = get.names;
-	result.plan = CreatePlan(*bound_func);
+	SelectStatement select_statement;
+	auto select_node = make_uniq<SelectNode>();
+	auto table_function = make_uniq<TableFunctionRef>();
+	table_function->function = std::move(stmt.function);
+	select_node->select_list.push_back(make_uniq<StarExpression>());
+	select_node->from_table = std::move(table_function);
+	select_statement.node = std::move(select_node);
 
+	auto result = Bind(select_statement);
 	auto &properties = GetStatementProperties();
-	properties.return_type = StatementReturnType::QUERY_RESULT;
+	properties.allow_stream_result = false;
 	return result;
 }
 

package/src/duckdb/src/planner/binder/statement/bind_copy.cpp

@@ -18,6 +18,7 @@
 #include "duckdb/planner/operator/logical_copy_to_file.hpp"
 #include "duckdb/planner/operator/logical_get.hpp"
 #include "duckdb/planner/operator/logical_insert.hpp"
+#include "duckdb/planner/operator/logical_projection.hpp"
 
 #include <algorithm>
 
@@ -27,15 +28,12 @@ static bool GetBooleanArg(ClientContext &context, const vector<Value> &arg) {
 	return arg.empty() || arg[0].CastAs(context, LogicalType::BOOLEAN).GetValue<bool>();
 }
 
-BoundStatement Binder::BindCopyTo(CopyStatement &stmt) {
+BoundStatement Binder::BindCopyTo(CopyStatement &stmt, CopyToType copy_to_type) {
 	// COPY TO a file
 	auto &config = DBConfig::GetConfig(context);
 	if (!config.options.enable_external_access) {
 		throw PermissionException("COPY TO is disabled by configuration");
 	}
-	BoundStatement result;
-	result.types = {LogicalType::BIGINT};
-	result.names = {"Count"};
 
 	// lookup the format in the catalog
 	auto &copy_function =
@@ -62,6 +60,9 @@ BoundStatement Binder::BindCopyTo(CopyStatement &stmt) {
 	optional_idx file_size_bytes;
 	vector<idx_t> partition_cols;
 	bool seen_overwrite_mode = false;
+	bool seen_filepattern = false;
+	bool write_partition_columns = false;
+	CopyFunctionReturnType return_type = CopyFunctionReturnType::CHANGED_ROWS;
 
 	CopyFunctionBindInput bind_input(*stmt.info);
 
@@ -69,15 +70,14 @@ BoundStatement Binder::BindCopyTo(CopyStatement &stmt) {
 
 	auto original_options = stmt.info->options;
 	stmt.info->options.clear();
-
 	for (auto &option : original_options) {
 		auto loption = StringUtil::Lower(option.first);
 		if (loption == "use_tmp_file") {
 			use_tmp_file = GetBooleanArg(context, option.second);
 			user_set_use_tmp_file = true;
-		} else if (loption == "overwrite_or_ignore" || loption == "overwrite") {
+		} else if (loption == "overwrite_or_ignore" || loption == "overwrite" || loption == "append") {
 			if (seen_overwrite_mode) {
-				throw BinderException("Can only set one of OVERWRITE_OR_IGNORE or OVERWRITE");
+				throw BinderException("Can only set one of OVERWRITE_OR_IGNORE, OVERWRITE or APPEND");
 			}
 			seen_overwrite_mode = true;
 
@@ -87,6 +87,11 @@ BoundStatement Binder::BindCopyTo(CopyStatement &stmt) {
 					overwrite_mode = CopyOverwriteMode::COPY_OVERWRITE_OR_IGNORE;
 				} else if (loption == "overwrite") {
 					overwrite_mode = CopyOverwriteMode::COPY_OVERWRITE;
+				} else if (loption == "append") {
+					if (!seen_filepattern) {
+						filename_pattern.SetFilenamePattern("{uuid}");
+					}
+					overwrite_mode = CopyOverwriteMode::COPY_APPEND;
 				}
 			}
 		} else if (loption == "filename_pattern") {
@@ -95,6 +100,7 @@ BoundStatement Binder::BindCopyTo(CopyStatement &stmt) {
 			}
 			filename_pattern.SetFilenamePattern(
 			    option.second[0].CastAs(context, LogicalType::VARCHAR).GetValue<string>());
+			seen_filepattern = true;
 		} else if (loption == "file_extension") {
 			if (option.second.empty()) {
 				throw IOException("FILE_EXTENSION cannot be empty");
@@ -106,7 +112,7 @@ BoundStatement Binder::BindCopyTo(CopyStatement &stmt) {
 			if (option.second.empty()) {
 				throw BinderException("FILE_SIZE_BYTES cannot be empty");
 			}
-			if (!copy_function.function.file_size_bytes) {
+			if (!copy_function.function.rotate_files) {
 				throw NotImplementedException("FILE_SIZE_BYTES not implemented for FORMAT \"%s\"", stmt.info->format);
 			}
 			if (option.second[0].GetTypeMutable().id() == LogicalTypeId::VARCHAR) {
@@ -117,10 +123,19 @@ BoundStatement Binder::BindCopyTo(CopyStatement &stmt) {
 		} else if (loption == "partition_by") {
 			auto converted = ConvertVectorToValue(std::move(option.second));
 			partition_cols = ParseColumnsOrdered(converted, select_node.names, loption);
+		} else if (loption == "return_files") {
+			if (GetBooleanArg(context, option.second)) {
+				return_type = CopyFunctionReturnType::CHANGED_ROWS_AND_FILE_LIST;
+			}
+		} else if (loption == "write_partition_columns") {
+			write_partition_columns = true;
 		} else {
 			stmt.info->options[option.first] = option.second;
 		}
 	}
+	if (overwrite_mode == CopyOverwriteMode::COPY_APPEND && !filename_pattern.HasUUID()) {
+		throw BinderException("APPEND mode requires a {uuid} label in filename_pattern");
+	}
 	if (user_set_use_tmp_file && per_thread_output) {
 		throw NotImplementedException("Can't combine USE_TMP_FILE and PER_THREAD_OUTPUT for COPY");
 	}
@@ -136,6 +151,12 @@ BoundStatement Binder::BindCopyTo(CopyStatement &stmt) {
 	if (file_size_bytes.IsValid() && !partition_cols.empty()) {
 		throw NotImplementedException("Can't combine FILE_SIZE_BYTES and PARTITION_BY for COPY");
 	}
+	if (!write_partition_columns) {
+		if (partition_cols.size() == select_node.names.size()) {
+			throw NotImplementedException("No column to write as all columns are specified as partition columns. "
+			                              "WRITE_PARTITION_COLUMNS option can be used to write partition columns.");
+		}
+	}
 	bool is_remote_file = FileSystem::IsRemoteFile(stmt.info->file_path);
 	if (is_remote_file) {
 		use_tmp_file = false;
@@ -148,12 +169,65 @@ BoundStatement Binder::BindCopyTo(CopyStatement &stmt) {
 		}
 	}
 
+	// Allow the copy function to intercept the select list and types and push a new projection on top of the plan
+	if (copy_function.function.copy_to_select) {
+		auto bindings = select_node.plan->GetColumnBindings();
+
+		CopyToSelectInput input = {context, stmt.info->options, {}, copy_to_type};
+		input.select_list.reserve(bindings.size());
+
+		// Create column references for the select list
+		for (idx_t i = 0; i < bindings.size(); i++) {
+			auto &binding = bindings[i];
+			auto &name = select_node.names[i];
+			auto &type = select_node.types[i];
+			input.select_list.push_back(make_uniq<BoundColumnRefExpression>(name, type, binding));
+		}
+
+		auto new_select_list = copy_function.function.copy_to_select(input);
+		if (!new_select_list.empty()) {
+
+			// We have a new select list, create a projection on top of the current plan
+			auto projection = make_uniq<LogicalProjection>(GenerateTableIndex(), std::move(new_select_list));
+			projection->children.push_back(std::move(select_node.plan));
+			projection->ResolveOperatorTypes();
+
+			// Update the names and types of the select node
+			select_node.names.clear();
+			select_node.types.clear();
+			for (auto &expr : projection->expressions) {
+				select_node.names.push_back(expr->GetName());
+				select_node.types.push_back(expr->return_type);
+			}
+			select_node.plan = std::move(projection);
+		}
+	}
+
 	auto unique_column_names = select_node.names;
 	QueryResult::DeduplicateColumns(unique_column_names);
 	auto file_path = stmt.info->file_path;
 
-	auto function_data =
-	    copy_function.function.copy_to_bind(context, bind_input, unique_column_names, select_node.types);
+	auto names_to_write =
+	    LogicalCopyToFile::GetNamesWithoutPartitions(unique_column_names, partition_cols, write_partition_columns);
+	auto types_to_write =
+	    LogicalCopyToFile::GetTypesWithoutPartitions(select_node.types, partition_cols, write_partition_columns);
+	auto function_data = copy_function.function.copy_to_bind(context, bind_input, names_to_write, types_to_write);
+
+	const auto rotate =
+	    copy_function.function.rotate_files && copy_function.function.rotate_files(*function_data, file_size_bytes);
+	if (rotate) {
+		if (!copy_function.function.rotate_next_file) {
+			throw InternalException("rotate_next_file not implemented for \"%s\"", copy_function.function.extension);
+		}
+		if (user_set_use_tmp_file) {
+			throw NotImplementedException(
+			    "Can't combine USE_TMP_FILE and file rotation (e.g., ROW_GROUPS_PER_FILE) for COPY");
+		}
+		if (!partition_cols.empty()) {
+			throw NotImplementedException(
+			    "Can't combine file rotation (e.g., ROW_GROUPS_PER_FILE) and PARTITION_BY for COPY");
+		}
+	}
 
 	// now create the copy information
 	auto copy = make_uniq<LogicalCopyToFile>(copy_function.function, std::move(function_data), std::move(stmt.info));
@@ -166,14 +240,32 @@ BoundStatement Binder::BindCopyTo(CopyStatement &stmt) {
 	if (file_size_bytes.IsValid()) {
 		copy->file_size_bytes = file_size_bytes;
 	}
+	copy->rotate = rotate;
 	copy->partition_output = !partition_cols.empty();
+	copy->write_partition_columns = write_partition_columns;
 	copy->partition_columns = std::move(partition_cols);
+	copy->return_type = return_type;
 
 	copy->names = unique_column_names;
 	copy->expected_types = select_node.types;
 
 	copy->AddChild(std::move(select_node.plan));
 
+	auto &properties = GetStatementProperties();
+	switch (copy->return_type) {
+	case CopyFunctionReturnType::CHANGED_ROWS:
+		properties.return_type = StatementReturnType::CHANGED_ROWS;
+		break;
+	case CopyFunctionReturnType::CHANGED_ROWS_AND_FILE_LIST:
+		properties.return_type = StatementReturnType::QUERY_RESULT;
+		break;
+	default:
+		throw NotImplementedException("Unknown CopyFunctionReturnType");
+	}
+
+	BoundStatement result;
+	result.names = GetCopyFunctionReturnNames(copy->return_type);
+	result.types = GetCopyFunctionReturnLogicalTypes(copy->return_type);
 	result.plan = std::move(copy);
 
 	return result;
@@ -236,14 +328,14 @@ BoundStatement Binder::BindCopyFrom(CopyStatement &stmt) {
 	auto get = make_uniq<LogicalGet>(GenerateTableIndex(), copy_function.function.copy_from_function,
 	                                 std::move(function_data), bound_insert.expected_types, expected_names);
 	for (idx_t i = 0; i < bound_insert.expected_types.size(); i++) {
-		get->column_ids.push_back(i);
+		get->AddColumnId(i);
 	}
 	insert_statement.plan->children.push_back(std::move(get));
 	result.plan = std::move(insert_statement.plan);
 	return result;
 }
 
-BoundStatement Binder::Bind(CopyStatement &stmt) {
+BoundStatement Binder::Bind(CopyStatement &stmt, CopyToType copy_to_type) {
 	if (!stmt.info->is_from && !stmt.info->select_statement) {
 		// copy table into file without a query
 		// generate SELECT * FROM table;
@@ -270,7 +362,7 @@ BoundStatement Binder::Bind(CopyStatement &stmt) {
 	if (stmt.info->is_from) {
 		return BindCopyFrom(stmt);
 	} else {
-		return BindCopyTo(stmt);
+		return BindCopyTo(stmt, copy_to_type);
 	}
 }
 

package/src/duckdb/src/planner/binder/statement/bind_copy_database.cpp

@@ -56,7 +56,7 @@ unique_ptr<LogicalOperator> Binder::BindCopyDatabaseData(Catalog &source_catalog
 
 	unique_ptr<LogicalOperator> result;
 	for (auto &table_ref : entries.tables) {
-		auto &table = table_ref.get();
+		auto &table = table_ref.get().Cast<TableCatalogEntry>();
 		// generate the insert statement
 		InsertStatement insert_stmt;
 		insert_stmt.catalog = target_database_name;
@@ -69,7 +69,11 @@ unique_ptr<LogicalOperator> Binder::BindCopyDatabaseData(Catalog &source_catalog
 		from_tbl->table_name = table.name;
 
 		auto select_node = make_uniq<SelectNode>();
-		select_node->select_list.push_back(make_uniq<StarExpression>());
+		auto &select_list = select_node->select_list;
+		for (auto &col : table.GetColumns().Physical()) {
+			select_list.push_back(make_uniq<ColumnRefExpression>(col.Name(), table.name));
+		}
+
 		select_node->from_table = std::move(from_tbl);
 
 		auto select_stmt = make_uniq<SelectStatement>();
@@ -128,7 +132,7 @@ BoundStatement Binder::Bind(CopyDatabaseStatement &stmt) {
 	auto &properties = GetStatementProperties();
 	properties.allow_stream_result = false;
 	properties.return_type = StatementReturnType::NOTHING;
-	properties.modified_databases.insert(target_catalog.GetName());
+	properties.RegisterDBModify(target_catalog, context);
 	return result;
 }
 

package/src/duckdb/src/planner/binder/statement/bind_create.cpp

@@ -122,7 +122,7 @@ SchemaCatalogEntry &Binder::BindSchema(CreateInfo &info) {
 	info.schema = schema_obj.name;
 	if (!info.temporary) {
 		auto &properties = GetStatementProperties();
-		properties.modified_databases.insert(schema_obj.catalog.GetName());
+		properties.RegisterDBModify(schema_obj.catalog, context);
 	}
 	return schema_obj;
 }
@@ -172,60 +172,78 @@ void Binder::BindCreateViewInfo(CreateViewInfo &base) {
 
 SchemaCatalogEntry &Binder::BindCreateFunctionInfo(CreateInfo &info) {
 	auto &base = info.Cast<CreateMacroInfo>();
-	auto &scalar_function = base.function->Cast<ScalarMacroFunction>();
-
-	if (scalar_function.expression->HasParameter()) {
-		throw BinderException("Parameter expressions within macro's are not supported!");
-	}
-
-	// create macro binding in order to bind the function
-	vector<LogicalType> dummy_types;
-	vector<string> dummy_names;
-	// positional parameters
-	for (auto &param_expr : base.function->parameters) {
-		auto param = param_expr->Cast<ColumnRefExpression>();
-		if (param.IsQualified()) {
-			throw BinderException("Invalid parameter name '%s': must be unqualified", param.ToString());
-		}
-		dummy_types.emplace_back(LogicalType::SQLNULL);
-		dummy_names.push_back(param.GetColumnName());
-	}
-	// default parameters
-	for (auto &entry : base.function->default_parameters) {
-		auto &val = entry.second->Cast<ConstantExpression>();
-		dummy_types.push_back(val.value.type());
-		dummy_names.push_back(entry.first);
-	}
-	auto this_macro_binding = make_uniq<DummyBinding>(dummy_types, dummy_names, base.name);
-	macro_binding = this_macro_binding.get();
-
-	// create a copy of the expression because we do not want to alter the original
-	auto expression = scalar_function.expression->Copy();
-	ExpressionBinder::QualifyColumnNames(*this, expression);
-
-	// bind it to verify the function was defined correctly
-	ErrorData error;
-	BoundSelectNode sel_node;
-	BoundGroupInformation group_info;
-	SelectBinder binder(*this, context, sel_node, group_info);
+
 	auto &dependencies = base.dependencies;
 	auto &catalog = Catalog::GetCatalog(context, info.catalog);
 	auto &db_config = DBConfig::GetConfig(context);
-	auto should_create_dependencies = db_config.options.enable_macro_dependencies;
+	// try to bind each of the included functions
+	unordered_set<idx_t> positional_parameters;
+	for (auto &function : base.macros) {
+		auto &scalar_function = function->Cast<ScalarMacroFunction>();
+		if (scalar_function.expression->HasParameter()) {
+			throw BinderException("Parameter expressions within macro's are not supported!");
+		}
+		vector<LogicalType> dummy_types;
+		vector<string> dummy_names;
+		auto parameter_count = function->parameters.size();
+		if (positional_parameters.find(parameter_count) != positional_parameters.end()) {
+			throw BinderException(
+			    "Ambiguity in macro overloads - macro \"%s\" has multiple definitions with %llu parameters", base.name,
+			    parameter_count);
+		}
+		positional_parameters.insert(parameter_count);
 
-	if (should_create_dependencies) {
-		binder.SetCatalogLookupCallback([&dependencies, &catalog](CatalogEntry &entry) {
-			if (&catalog != &entry.ParentCatalog()) {
-				// Don't register any cross-catalog dependencies
-				return;
+		// positional parameters
+		for (auto &param_expr : function->parameters) {
+			auto param = param_expr->Cast<ColumnRefExpression>();
+			if (param.IsQualified()) {
+				throw BinderException("Invalid parameter name '%s': must be unqualified", param.ToString());
 			}
-			// Register any catalog entry required to bind the macro function
-			dependencies.AddDependency(entry);
-		});
-	}
-	error = binder.Bind(expression, 0, false);
-	if (error.HasError()) {
-		error.Throw();
+			dummy_types.emplace_back(LogicalType::SQLNULL);
+			dummy_names.push_back(param.GetColumnName());
+		}
+		// default parameters
+		for (auto &entry : function->default_parameters) {
+			auto &val = entry.second->Cast<ConstantExpression>();
+			dummy_types.push_back(val.value.type());
+			dummy_names.push_back(entry.first);
+		}
+		auto this_macro_binding = make_uniq<DummyBinding>(dummy_types, dummy_names, base.name);
+		macro_binding = this_macro_binding.get();
+
+		// create a copy of the expression because we do not want to alter the original
+		auto expression = scalar_function.expression->Copy();
+		ExpressionBinder::QualifyColumnNames(*this, expression);
+
+		// bind it to verify the function was defined correctly
+		BoundSelectNode sel_node;
+		BoundGroupInformation group_info;
+		SelectBinder binder(*this, context, sel_node, group_info);
+		auto should_create_dependencies = db_config.options.enable_macro_dependencies;
+
+		if (should_create_dependencies) {
+			binder.SetCatalogLookupCallback([&dependencies, &catalog](CatalogEntry &entry) {
+				if (&catalog != &entry.ParentCatalog()) {
+					// Don't register any cross-catalog dependencies
+					return;
+				}
+				// Register any catalog entry required to bind the macro function
+				dependencies.AddDependency(entry);
+			});
+		}
+		ErrorData error;
+		try {
+			error = binder.Bind(expression, 0, false);
+			if (error.HasError()) {
+				error.Throw();
+			}
+		} catch (const std::exception &ex) {
+			error = ErrorData(ex);
+		}
+		// if we cannot resolve parameters we postpone binding until the macro function is used
+		if (error.HasError() && error.Type() != ExceptionType::PARAMETER_NOT_RESOLVED) {
+			error.Throw();
+		}
 	}
 
 	return BindCreateSchema(info);
@@ -560,7 +578,8 @@ unique_ptr<LogicalOperator> DuckCatalog::BindCreateIndex(Binder &binder, CreateS
 	}
 
 	auto create_index_info = unique_ptr_cast<CreateInfo, CreateIndexInfo>(std::move(stmt.info));
-	for (auto &column_id : get.column_ids) {
+	auto &column_ids = get.GetColumnIds();
+	for (auto &column_id : column_ids) {
 		if (column_id == COLUMN_IDENTIFIER_ROW_ID) {
 			throw BinderException("Cannot create an index on the rowid!");
 		}
@@ -568,10 +587,11 @@ unique_ptr<LogicalOperator> DuckCatalog::BindCreateIndex(Binder &binder, CreateS
 	}
 	create_index_info->scan_types.emplace_back(LogicalType::ROW_TYPE);
 	create_index_info->names = get.names;
-	create_index_info->column_ids = get.column_ids;
+	create_index_info->column_ids = column_ids;
+	create_index_info->schema = table.schema.name;
 	auto &bind_data = get.bind_data->Cast<TableScanBindData>();
 	bind_data.is_create_index = true;
-	get.column_ids.push_back(COLUMN_IDENTIFIER_ROW_ID);
+	get.AddColumnId(COLUMN_IDENTIFIER_ROW_ID);
 
 	// the logical CREATE INDEX also needs all fields to scan the referenced table
 	auto result = make_uniq<LogicalCreateIndex>(std::move(create_index_info), std::move(expressions), table);
@@ -590,7 +610,7 @@ BoundStatement Binder::Bind(CreateStatement &stmt) {
 	case CatalogType::SCHEMA_ENTRY: {
 		auto &base = stmt.info->Cast<CreateInfo>();
 		auto catalog = BindCatalog(base.catalog);
-		properties.modified_databases.insert(catalog);
+		properties.RegisterDBModify(Catalog::GetCatalog(context, catalog), context);
 		result.plan = make_uniq<LogicalCreate>(LogicalOperatorType::LOGICAL_CREATE_SCHEMA, std::move(stmt.info));
 		break;
 	}
@@ -639,7 +659,7 @@ BoundStatement Binder::Bind(CreateStatement &stmt) {
 		if (table.temporary) {
 			stmt.info->temporary = true;
 		}
-		properties.modified_databases.insert(table.catalog.GetName());
+		properties.RegisterDBModify(table.catalog, context);
 
 		// create a plan over the bound table
 		auto plan = CreatePlan(*bound_table);

package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp

@@ -349,7 +349,7 @@ unique_ptr<BoundCreateTableInfo> Binder::BindCreateTableInfo(unique_ptr<CreateIn
 		if (column.Type().id() == LogicalTypeId::VARCHAR) {
 			ExpressionBinder::TestCollation(context, StringType::GetCollation(column.Type()));
 		}
-		BindLogicalType(column.TypeMutable(), &result->schema.catalog);
+		BindLogicalType(column.TypeMutable(), &result->schema.catalog, result->schema.name);
 	}
 	result->dependencies.VerifyDependencies(schema.catalog, result->Base().table);
 

package/src/duckdb/src/planner/binder/statement/bind_delete.cpp

@@ -30,7 +30,7 @@ BoundStatement Binder::Bind(DeleteStatement &stmt) {
 	if (!table.temporary) {
 		// delete from persistent table: not read only!
 		auto &properties = GetStatementProperties();
-		properties.modified_databases.insert(table.catalog.GetName());
+		properties.RegisterDBModify(table.catalog, context);
 	}
 
 	// Add CTEs as bindable
@@ -74,9 +74,10 @@ BoundStatement Binder::Bind(DeleteStatement &stmt) {
 	del->AddChild(std::move(root));
 
 	// set up the delete expression
-	del->expressions.push_back(make_uniq<BoundColumnRefExpression>(
-	    LogicalType::ROW_TYPE, ColumnBinding(get.table_index, get.column_ids.size())));
-	get.column_ids.push_back(COLUMN_IDENTIFIER_ROW_ID);
+	auto &column_ids = get.GetColumnIds();
+	del->expressions.push_back(
+	    make_uniq<BoundColumnRefExpression>(LogicalType::ROW_TYPE, ColumnBinding(get.table_index, column_ids.size())));
+	get.AddColumnId(COLUMN_IDENTIFIER_ROW_ID);
 
 	if (!stmt.returning_list.empty()) {
 		del->return_chunk = true;

package/src/duckdb/src/planner/binder/statement/bind_drop.cpp

@@ -24,7 +24,7 @@ BoundStatement Binder::Bind(DropStatement &stmt) {
 	case CatalogType::SCHEMA_ENTRY: {
 		// dropping a schema is never read-only because there are no temporary schemas
 		auto &catalog = Catalog::GetCatalog(context, stmt.info->catalog);
-		properties.modified_databases.insert(catalog.GetName());
+		properties.RegisterDBModify(catalog, context);
 		break;
 	}
 	case CatalogType::VIEW_ENTRY:
@@ -46,7 +46,7 @@ BoundStatement Binder::Bind(DropStatement &stmt) {
 		stmt.info->catalog = entry->ParentCatalog().GetName();
 		if (!entry->temporary) {
 			// we can only drop temporary schema entries in read-only mode
-			properties.modified_databases.insert(stmt.info->catalog);
+			properties.RegisterDBModify(entry->ParentCatalog(), context);
 		}
 		stmt.info->schema = entry->ParentSchema().name;
 		break;