duckdb 1.3.4 → 1.4.0

package/src/duckdb/src/include/duckdb/function/function_serialization.hpp

@@ -57,7 +57,8 @@ public:
 	}
 
 	template <class FUNC, class CATALOG_ENTRY>
-	static pair<FUNC, bool> DeserializeBase(Deserializer &deserializer, CatalogType catalog_type) {
+	static pair<FUNC, bool> DeserializeBase(Deserializer &deserializer, CatalogType catalog_type,
+	                                        optional_ptr<vector<unique_ptr<Expression>>> children = nullptr) {
 		auto &context = deserializer.Get<ClientContext &>();
 		auto name = deserializer.ReadProperty<string>(500, "name");
 		auto arguments = deserializer.ReadProperty<vector<LogicalType>>(501, "arguments");
@@ -70,6 +71,17 @@ public:
 		if (schema_name.empty()) {
 			schema_name = DEFAULT_SCHEMA;
 		}
+
+		if (arguments.empty() && original_arguments.empty() && children && !children->empty()) {
+			// The function is specified as having no arguments, but somehow expressions were passed anyway
+			// Assume this is a "varargs" function and use the types of the expressions as the arguments
+			// This can happen when we change a function that used to take varargs, to no longer do so.
+			arguments.reserve(children->size());
+			for (auto &child : *children) {
+				arguments.push_back(child->return_type);
+			}
+		}
+
 		auto function = DeserializeFunction<FUNC, CATALOG_ENTRY>(context, catalog_type, catalog_name, schema_name, name,
 		                                                         arguments, original_arguments);
 		auto has_serialize = deserializer.ReadProperty<bool>(503, "has_serialize");
@@ -100,6 +112,7 @@ public:
 			return true;
 		case LogicalTypeId::DECIMAL:
 		case LogicalTypeId::UNION:
+		case LogicalTypeId::VARIANT:
 		case LogicalTypeId::MAP:
 			if (!type.AuxInfo()) {
 				return true;
@@ -133,7 +146,7 @@ public:
 	                                                        vector<unique_ptr<Expression>> &children,
 	                                                        LogicalType return_type) { // NOLINT: clang-tidy bug
 		auto &context = deserializer.Get<ClientContext &>();
-		auto entry = DeserializeBase<FUNC, CATALOG_ENTRY>(deserializer, catalog_type);
+		auto entry = DeserializeBase<FUNC, CATALOG_ENTRY>(deserializer, catalog_type, children);
 		auto &function = entry.first;
 		auto has_serialize = entry.second;
 
@@ -143,6 +156,12 @@ public:
 			bind_data = FunctionDeserialize<FUNC>(deserializer, function);
 			deserializer.Unset<LogicalType>();
 		} else {
+
+			FunctionBinder binder(context);
+
+			// Resolve templates
+			binder.ResolveTemplateTypes(function, children);
+
 			if (function.bind) {
 				try {
 					bind_data = function.bind(context, function, children);
@@ -152,7 +171,10 @@ public:
 					                             error.RawMessage());
 				}
 			}
-			FunctionBinder binder(context);
+
+			// Verify that all templates are bound to concrete types.
+			binder.CheckTemplateTypesResolved(function);
+
 			binder.CastToFunctionArguments(function, children);
 		}
 

package/src/duckdb/src/include/duckdb/function/macro_function.hpp

@@ -17,6 +17,8 @@
 
 namespace duckdb {
 
+class ScalarMacroFunction;
+
 enum class MacroType : uint8_t { VOID_MACRO = 0, TABLE_MACRO = 1, SCALAR_MACRO = 2 };
 
 struct MacroBindResult {
@@ -35,10 +37,12 @@ public:
 
 	//! The type
 	MacroType type;
-	//! The positional parameters
+	//! The parameters (ColumnRefExpression)
 	vector<unique_ptr<ParsedExpression>> parameters;
-	//! The default parameters and their associated values
-	case_insensitive_map_t<unique_ptr<ParsedExpression>> default_parameters;
+	//! The default values of the parameters
+	InsertionOrderPreservingMap<unique_ptr<ParsedExpression>> default_parameters;
+	//! The types of the parameters
+	vector<LogicalType> types;
 
 public:
 	virtual ~MacroFunction() {
@@ -48,10 +52,18 @@ public:
 
 	virtual unique_ptr<MacroFunction> Copy() const = 0;
 
-	static MacroBindResult BindMacroFunction(const vector<unique_ptr<MacroFunction>> &macro_functions,
+	vector<unique_ptr<ParsedExpression>> GetPositionalParametersForSerialization(Serializer &serializer) const;
+	void FinalizeDeserialization();
+
+	static MacroBindResult BindMacroFunction(Binder &binder, const vector<unique_ptr<MacroFunction>> &macro_functions,
 	                                         const string &name, FunctionExpression &function_expr,
-	                                         vector<unique_ptr<ParsedExpression>> &positionals,
-	                                         unordered_map<string, unique_ptr<ParsedExpression>> &defaults);
+	                                         vector<unique_ptr<ParsedExpression>> &positional_arguments,
+	                                         InsertionOrderPreservingMap<unique_ptr<ParsedExpression>> &named_arguments,
+	                                         idx_t depth);
+	static unique_ptr<DummyBinding>
+	CreateDummyBinding(const MacroFunction &macro_def, const string &name,
+	                   vector<unique_ptr<ParsedExpression>> &positional_arguments,
+	                   InsertionOrderPreservingMap<unique_ptr<ParsedExpression>> &named_arguments);
 
 	virtual string ToSQL() const;
 

package/src/duckdb/src/include/duckdb/function/pragma/pragma_functions.hpp

@@ -21,7 +21,7 @@ struct PragmaFunctions {
 	static void RegisterFunction(BuiltinFunctions &set);
 };
 
-string PragmaShowTables();
+string PragmaShowTables(const string &catalog = "", const string &schema = "");
 string PragmaShowTablesExpanded();
 string PragmaShowDatabases();
 string PragmaShowVariables();

package/src/duckdb/src/include/duckdb/function/register_function_list_helper.hpp

@@ -30,6 +30,36 @@ static void FillFunctionParameters(FunctionDescription &function_description, co
 	}
 }
 
+static vector<string> GetExamplesForFunctionAlias(const string &function_name, const string &alias_of,
+                                                  vector<string> &all_examples) {
+	vector<string> filtered_examples;
+	bool is_operator = (!function_name.empty() && !(function_name[0] >= 'a' && function_name[0] <= 'z') &&
+	                    !(function_name[0] >= 'A' && function_name[0] <= 'Z'));
+	bool alias_of_is_operator = (!alias_of.empty() && !(alias_of[0] >= 'a' && alias_of[0] <= 'z') &&
+	                             !(alias_of[0] >= 'A' && alias_of[0] <= 'Z'));
+	// select examples with matching function name
+	for (string &example : all_examples) {
+		if (example.compare(0, function_name.size(), function_name) == 0 ||
+		    (is_operator && example.find(function_name) != string::npos)) {
+			filtered_examples.emplace_back(std::move(example));
+		}
+	}
+	// fallback 1: create fitting examples by replacing canonical name by function_name
+	if (filtered_examples.empty() && !alias_of.empty() && !alias_of_is_operator && !is_operator) {
+		for (string &example : all_examples) {
+			if (example.compare(0, alias_of.size(), alias_of) == 0) {
+				filtered_examples.emplace_back(function_name +
+				                               example.substr(alias_of.size(), example.size() - alias_of.size()));
+			}
+		}
+	}
+	// fallback 2: use available examples anyway
+	if (filtered_examples.empty()) {
+		filtered_examples = all_examples;
+	}
+	return filtered_examples;
+}
+
 template <class FunctionDefinition, class T>
 static void FillFunctionDescriptions(const FunctionDefinition &function, T &info) {
 	vector<string> variants = StringUtil::Split(function.parameters, '\1');
@@ -56,13 +86,15 @@ static void FillFunctionDescriptions(const FunctionDefinition &function, T &info
 			throw InternalException("Incorrect number of function descriptions for function '%s'", function.name);
 		}
 		// examples
+		duckdb::vector<string> variant_examples;
 		if (examples.size() == variants.size()) {
-			function_description.examples = StringUtil::Split(examples[variant_index], '\2');
+			variant_examples = StringUtil::Split(examples[variant_index], '\2');
 		} else if (examples.size() == 1) {
-			function_description.examples = StringUtil::Split(examples[0], '\2');
+			variant_examples = StringUtil::Split(examples[0], '\2');
 		} else if (!examples.empty()) {
 			throw InternalException("Incorrect number of function examples for function '%s'", function.name);
 		}
+		function_description.examples = GetExamplesForFunctionAlias(function.name, info.alias_of, variant_examples);
 		// categories
 		if (variant_index < categories.size()) {
 			function_description.categories = StringUtil::Split(categories[variant_index], ',');

package/src/duckdb/src/include/duckdb/function/scalar/compressed_materialization_functions.hpp

@@ -105,6 +105,16 @@ struct InternalCompressStringUhugeintFun {
 	static ScalarFunction GetFunction();
 };
 
+struct InternalCompressStringHugeintFun {
+	static constexpr const char *Name = "__internal_compress_string_hugeint";
+	static constexpr const char *Parameters = "";
+	static constexpr const char *Description = "";
+	static constexpr const char *Example = "";
+	static constexpr const char *Categories = "";
+
+	static ScalarFunction GetFunction();
+};
+
 struct InternalDecompressIntegralSmallintFun {
 	static constexpr const char *Name = "__internal_decompress_integral_smallint";
 	static constexpr const char *Parameters = "";

package/src/duckdb/src/include/duckdb/function/scalar/date_functions.hpp

@@ -18,7 +18,7 @@ namespace duckdb {
 struct StrfTimeFun {
 	static constexpr const char *Name = "strftime";
 	static constexpr const char *Parameters = "data,format";
-	static constexpr const char *Description = "Converts a date to a string according to the format string.";
+	static constexpr const char *Description = "Converts a `date` to a string according to the format string.";
 	static constexpr const char *Example = "strftime(date '1992-01-01', '%a, %-d %B %Y')";
 	static constexpr const char *Categories = "";
 
@@ -27,10 +27,10 @@ struct StrfTimeFun {
 
 struct StrpTimeFun {
 	static constexpr const char *Name = "strptime";
-	static constexpr const char *Parameters = "text::VARCHAR,format::VARCHAR\1text::VARCHAR,format-list::VARCHAR[]";
-	static constexpr const char *Description = "Converts the string text to timestamp according to the format string. Throws an error on failure. To return NULL on failure, use try_strptime.\1Converts the string text to timestamp applying the format strings in the list until one succeeds. Throws an error on failure. To return NULL on failure, use try_strptime.";
-	static constexpr const char *Example = "strptime('Wed, 1 January 1992 - 08:38:40 PM', '%a, %-d %B %Y - %I:%M:%S %p')\1strptime('4/15/2023 10:56:00', ['%d/%m/%Y %H:%M:%S', '%m/%d/%Y %H:%M:%S'])";
-	static constexpr const char *Categories = "\1";
+	static constexpr const char *Parameters = "text::VARCHAR,format::VARCHAR\001text::VARCHAR,format-list::VARCHAR[]";
+	static constexpr const char *Description = "Converts the `string` text to timestamp according to the format string. Throws an error on failure. To return `NULL` on failure, use try_strptime.\001Converts the `string` text to timestamp applying the format strings in the list until one succeeds. Throws an error on failure. To return `NULL` on failure, use try_strptime.";
+	static constexpr const char *Example = "strptime('Wed, 1 January 1992 - 08:38:40 PM', '%a, %-d %B %Y - %I:%M:%S %p')\001strptime('4/15/2023 10:56:00', ['%d/%m/%Y %H:%M:%S', '%m/%d/%Y %H:%M:%S'])";
+	static constexpr const char *Categories = "\001";
 
 	static ScalarFunctionSet GetFunctions();
 };
@@ -38,7 +38,7 @@ struct StrpTimeFun {
 struct TryStrpTimeFun {
 	static constexpr const char *Name = "try_strptime";
 	static constexpr const char *Parameters = "text,format";
-	static constexpr const char *Description = "Converts the string text to timestamp according to the format string. Returns NULL on failure.";
+	static constexpr const char *Description = "Converts the `string` text to timestamp according to the format string. Returns `NULL` on failure.";
 	static constexpr const char *Example = "try_strptime('Wed, 1 January 1992 - 08:38:40 PM', '%a, %-d %B %Y - %I:%M:%S %p')";
 	static constexpr const char *Categories = "";
 

package/src/duckdb/src/include/duckdb/function/scalar/list/contains_or_position.hpp

@@ -4,7 +4,7 @@
 
 namespace duckdb {
 
-template <class T, class RETURN_TYPE, bool FIND_NULLS = false>
+template <class T, class RETURN_TYPE, bool FIND_NULLS>
 idx_t ListSearchSimpleOp(Vector &input_list, Vector &list_child, Vector &target, Vector &result, const idx_t count) {
 	// If the return type is not a bool, return the position
 	const auto return_pos = std::is_same<RETURN_TYPE, int32_t>::value;
@@ -89,7 +89,7 @@ idx_t ListSearchSimpleOp(Vector &input_list, Vector &list_child, Vector &target,
 	return total_matches;
 }
 
-template <class RETURN_TYPE>
+template <class RETURN_TYPE, bool FIND_NULLS>
 idx_t ListSearchNestedOp(Vector &list_vec, Vector &source_vec, Vector &target_vec, Vector &result_vec,
                          const idx_t target_count) {
 	// Set up sort keys for nested types.
@@ -101,8 +101,8 @@ idx_t ListSearchNestedOp(Vector &list_vec, Vector &source_vec, Vector &target_ve
 	CreateSortKeyHelpers::CreateSortKeyWithValidity(source_vec, source_sort_key_vec, order_modifiers, source_count);
 	CreateSortKeyHelpers::CreateSortKeyWithValidity(target_vec, target_sort_key_vec, order_modifiers, target_count);
 
-	return ListSearchSimpleOp<string_t, RETURN_TYPE>(list_vec, source_sort_key_vec, target_sort_key_vec, result_vec,
-	                                                 target_count);
+	return ListSearchSimpleOp<string_t, RETURN_TYPE, FIND_NULLS>(list_vec, source_sort_key_vec, target_sort_key_vec,
+	                                                             result_vec, target_count);
 }
 
 //! "Search" each list in the list vector for the corresponding value in the target vector, returning either
@@ -152,7 +152,7 @@ idx_t ListSearchOp(Vector &list_v, Vector &source_v, Vector &target_v, Vector &r
 	case PhysicalType::STRUCT:
 	case PhysicalType::LIST:
 	case PhysicalType::ARRAY:
-		return ListSearchNestedOp<RETURN_TYPE>(list_v, source_v, target_v, result_v, target_count);
+		return ListSearchNestedOp<RETURN_TYPE, FIND_NULLS>(list_v, source_v, target_v, result_v, target_count);
 	default:
 		throw NotImplementedException("This function has not been implemented for logical type %s",
 		                              TypeIdToString(type));

package/src/duckdb/src/include/duckdb/function/scalar/list_functions.hpp

@@ -18,9 +18,9 @@ namespace duckdb {
 struct ListSelectFun {
 	static constexpr const char *Name = "list_select";
 	static constexpr const char *Parameters = "value_list,index_list";
-	static constexpr const char *Description = "Returns a list based on the elements selected by the index_list.";
+	static constexpr const char *Description = "Returns a list based on the elements selected by the `index_list`.";
 	static constexpr const char *Example = "list_select([10, 20, 30, 40], [1, 4])";
-	static constexpr const char *Categories = "";
+	static constexpr const char *Categories = "list";
 
 	static ScalarFunction GetFunction();
 };
@@ -34,9 +34,9 @@ struct ArraySelectFun {
 struct ListWhereFun {
 	static constexpr const char *Name = "list_where";
 	static constexpr const char *Parameters = "value_list,mask_list";
-	static constexpr const char *Description = "Returns a list with the BOOLEANs in mask_list applied as a mask to the value_list.";
+	static constexpr const char *Description = "Returns a list with the `BOOLEAN`s in `mask_list` applied as a mask to the `value_list`.";
 	static constexpr const char *Example = "list_where([10, 20, 30, 40], [true, false, false, true])";
-	static constexpr const char *Categories = "";
+	static constexpr const char *Categories = "list";
 
 	static ScalarFunction GetFunction();
 };
@@ -52,7 +52,7 @@ struct ListContainsFun {
 	static constexpr const char *Parameters = "list,element";
 	static constexpr const char *Description = "Returns true if the list contains the element.";
 	static constexpr const char *Example = "list_contains([1, 2, NULL], 1)";
-	static constexpr const char *Categories = "";
+	static constexpr const char *Categories = "list";
 
 	static ScalarFunction GetFunction();
 };
@@ -78,9 +78,9 @@ struct ArrayHasFun {
 struct ListPositionFun {
 	static constexpr const char *Name = "list_position";
 	static constexpr const char *Parameters = "list,element";
-	static constexpr const char *Description = "Returns the index of the element if the list contains the element. If the element is not found, it returns NULL.";
+	static constexpr const char *Description = "Returns the index of the `element` if the `list` contains the `element`. If the `element` is not found, it returns `NULL`.";
 	static constexpr const char *Example = "list_position([1, 2, NULL], 2)";
-	static constexpr const char *Categories = "";
+	static constexpr const char *Categories = "list";
 
 	static ScalarFunction GetFunction();
 };
@@ -105,10 +105,10 @@ struct ArrayIndexofFun {
 
 struct ListZipFun {
 	static constexpr const char *Name = "list_zip";
-	static constexpr const char *Parameters = "list1,list2,...";
-	static constexpr const char *Description = "Zips k LISTs to a new LIST whose length will be that of the longest list. Its elements are structs of k elements from each list list_1, …, list_k, missing elements are replaced with NULL. If truncate is set, all lists are truncated to the smallest list length.";
-	static constexpr const char *Example = "list_zip([1, 2], [3, 4], [5, 6])";
-	static constexpr const char *Categories = "";
+	static constexpr const char *Parameters = "list_1,...,truncate";
+	static constexpr const char *Description = "Zips n `LIST`s to a new `LIST` whose length will be that of the longest list. Its elements are structs of n elements from each list `list_1`, …, `list_n`, missing elements are replaced with `NULL`. If `truncate` is set, all lists are truncated to the smallest list length.";
+	static constexpr const char *Example = "list_zip([1, 2], [3, 4], [5, 6])\002list_zip([1, 2], [3, 4], [5, 6, 7])\002list_zip([1, 2], [3, 4], [5, 6, 7], true)";
+	static constexpr const char *Categories = "list";
 
 	static ScalarFunction GetFunction();
 };
@@ -122,9 +122,9 @@ struct ArrayZipFun {
 struct ListExtractFun {
 	static constexpr const char *Name = "list_extract";
 	static constexpr const char *Parameters = "list,index";
-	static constexpr const char *Description = "Extract the indexth (1-based) value from the list.";
+	static constexpr const char *Description = "Extract the `index`th (1-based) value from the list.";
 	static constexpr const char *Example = "list_extract([4, 5, 6], 3)";
-	static constexpr const char *Categories = "";
+	static constexpr const char *Categories = "list";
 
 	static ScalarFunctionSet GetFunctions();
 };
@@ -138,9 +138,9 @@ struct ListElementFun {
 struct ListResizeFun {
 	static constexpr const char *Name = "list_resize";
 	static constexpr const char *Parameters = "list,size[,value]";
-	static constexpr const char *Description = "Resizes the list to contain size elements. Initializes new elements with value or NULL if value is not set.";
+	static constexpr const char *Description = "Resizes the `list` to contain `size` elements. Initializes new elements with `value` or `NULL` if `value` is not set.";
 	static constexpr const char *Example = "list_resize([1, 2, 3], 5, 0)";
-	static constexpr const char *Categories = "";
+	static constexpr const char *Categories = "list";
 
 	static ScalarFunctionSet GetFunctions();
 };
@@ -153,10 +153,10 @@ struct ArrayResizeFun {
 
 struct ArrayExtractFun {
 	static constexpr const char *Name = "array_extract";
-	static constexpr const char *Parameters = "list,index";
-	static constexpr const char *Description = "Extract the indexth (1-based) value from the array.";
-	static constexpr const char *Example = "array_extract('DuckDB', 2)";
-	static constexpr const char *Categories = "";
+	static constexpr const char *Parameters = "string::VARCHAR,index::ANY\001struct::STRUCT,entry::VARCHAR\001struct::STRUCT,index::ANY\001list::ANY[],index::ANY";
+	static constexpr const char *Description = "Extracts a single character from a `string` using a (1-based) `index`.\001Extracts the named `entry` from the `STRUCT`.\001Extracts the entry from an unnamed `STRUCT` (tuple) using an index (1-based).\001Extracts the `index`th (1-based) value from the `list`.";
+	static constexpr const char *Example = "array_extract('DuckDB', 2)\001array_extract({'i': 3, 'v2': 3, 'v3': 0}, 'i')\001array_extract(row(42, 84), 1)\001array_extract([4, 5, 6], 3)";
+	static constexpr const char *Categories = "string\001struct\001struct\001list";
 
 	static ScalarFunctionSet GetFunctions();
 };