duckdb 0.7.2-dev717.0 → 0.7.2-dev865.0

package/src/duckdb/src/function/scalar/string/regexp/regexp_extract_all.cpp

@@ -0,0 +1,243 @@
+#include "duckdb/function/scalar/regexp.hpp"
+#include "duckdb/execution/expression_executor.hpp"
+#include "duckdb/planner/expression/bound_function_expression.hpp"
+#include "duckdb/function/scalar/string_functions.hpp"
+#include "re2/re2.h"
+
+namespace duckdb {
+
+using regexp_util::CreateStringPiece;
+using regexp_util::Extract;
+using regexp_util::ParseRegexOptions;
+using regexp_util::TryParseConstantPattern;
+
+unique_ptr<FunctionLocalState>
+RegexpExtractAll::InitLocalState(ExpressionState &state, const BoundFunctionExpression &expr, FunctionData *bind_data) {
+	auto &info = (RegexpBaseBindData &)*bind_data;
+	if (info.constant_pattern) {
+		return make_unique<RegexLocalState>(info, true);
+	}
+	return nullptr;
+}
+
+// Forwards startpos automatically
+bool ExtractAll(duckdb_re2::StringPiece &input, duckdb_re2::RE2 &pattern, idx_t *startpos,
+                duckdb_re2::StringPiece *groups, int ngroups) {
+
+	D_ASSERT(pattern.ok());
+	D_ASSERT(pattern.NumberOfCapturingGroups() == ngroups);
+
+	if (!pattern.Match(input, *startpos, input.size(), pattern.Anchored(), groups, ngroups + 1)) {
+		return false;
+	}
+	idx_t consumed = static_cast<size_t>(groups[0].end() - (input.begin() + *startpos));
+	if (!consumed) {
+		// Empty match found, have to manually forward the input
+		// to avoid an infinite loop
+		// FIXME: support unicode characters
+		consumed++;
+		while (*startpos + consumed < input.length() && !LengthFun::IsCharacter(input[*startpos + consumed])) {
+			consumed++;
+		}
+	}
+	*startpos += consumed;
+	return true;
+}
+
+void ExtractSingleTuple(const string_t &string, duckdb_re2::RE2 &pattern, int32_t group, RegexStringPieceArgs &args,
+                        Vector &result, idx_t row) {
+	auto input = CreateStringPiece(string);
+
+	auto &child_vector = ListVector::GetEntry(result);
+	auto list_content = FlatVector::GetData<string_t>(child_vector);
+	auto &child_validity = FlatVector::Validity(child_vector);
+
+	auto current_list_size = ListVector::GetListSize(result);
+	auto current_list_capacity = ListVector::GetListCapacity(result);
+
+	auto result_data = FlatVector::GetData<list_entry_t>(result);
+	auto &list_entry = result_data[row];
+	list_entry.offset = current_list_size;
+
+	if (group < 0) {
+		list_entry.length = 0;
+		return;
+	}
+	// If the requested group index is out of bounds
+	// we want to throw only if there is a match
+	bool throw_on_group_found = (idx_t)group > args.size;
+
+	idx_t startpos = 0;
+	for (idx_t iteration = 0; ExtractAll(input, pattern, &startpos, args.group_buffer, args.size); iteration++) {
+		if (!iteration && throw_on_group_found) {
+			throw InvalidInputException("Pattern has %d groups. Cannot access group %d", args.size, group);
+		}
+
+		// Make sure we have enough room for the new entries
+		if (current_list_size + 1 >= current_list_capacity) {
+			ListVector::Reserve(result, current_list_capacity * 2);
+			current_list_capacity = ListVector::GetListCapacity(result);
+			list_content = FlatVector::GetData<string_t>(child_vector);
+		}
+
+		// Write the captured groups into the list-child vector
+		auto &match_group = args.group_buffer[group];
+
+		idx_t child_idx = current_list_size;
+		if (match_group.empty()) {
+			// This group was not matched
+			list_content[child_idx] = string_t(string.GetDataUnsafe(), 0);
+			if (match_group.begin() == nullptr) {
+				// This group is optional
+				child_validity.SetInvalid(child_idx);
+			}
+		} else {
+			// Every group is a substring of the original, we can find out the offset using the pointer
+			// the 'match_group' address is guaranteed to be bigger than that of the source
+			D_ASSERT((const char *)match_group.begin() >= string.GetDataUnsafe());
+			idx_t offset = match_group.begin() - string.GetDataUnsafe();
+			list_content[child_idx] = string_t(string.GetDataUnsafe() + offset, match_group.size());
+		}
+		current_list_size++;
+		if (startpos > input.size()) {
+			// Empty match found at the end of the string
+			break;
+		}
+	}
+	list_entry.length = current_list_size - list_entry.offset;
+	ListVector::SetListSize(result, current_list_size);
+}
+
+int32_t GetGroupIndex(DataChunk &args, idx_t row, int32_t &result) {
+	if (args.ColumnCount() < 3) {
+		result = 0;
+		return true;
+	}
+	UnifiedVectorFormat format;
+	args.data[2].ToUnifiedFormat(args.size(), format);
+	idx_t index = format.sel->get_index(row);
+	if (!format.validity.RowIsValid(index)) {
+		return false;
+	}
+	result = ((int32_t *)format.data)[index];
+	return true;
+}
+
+duckdb_re2::RE2 &GetPattern(const RegexpBaseBindData &info, ExpressionState &state,
+                            unique_ptr<duckdb_re2::RE2> &pattern_p) {
+	if (info.constant_pattern) {
+		auto &lstate = (RegexLocalState &)*ExecuteFunctionState::GetFunctionState(state);
+		return lstate.constant_pattern;
+	}
+	D_ASSERT(pattern_p);
+	return *pattern_p;
+}
+
+RegexStringPieceArgs &GetGroupsBuffer(const RegexpBaseBindData &info, ExpressionState &state,
+                                      unique_ptr<RegexStringPieceArgs> &groups_p) {
+	if (info.constant_pattern) {
+		auto &lstate = (RegexLocalState &)*ExecuteFunctionState::GetFunctionState(state);
+		return lstate.group_buffer;
+	}
+	D_ASSERT(groups_p);
+	return *groups_p;
+}
+
+void RegexpExtractAll::Execute(DataChunk &args, ExpressionState &state, Vector &result) {
+	auto &func_expr = (BoundFunctionExpression &)state.expr;
+	const auto &info = (RegexpBaseBindData &)*func_expr.bind_info;
+
+	auto &strings = args.data[0];
+	auto &patterns = args.data[1];
+	D_ASSERT(result.GetType().id() == LogicalTypeId::LIST);
+	auto &output_child = ListVector::GetEntry(result);
+
+	UnifiedVectorFormat strings_data;
+	strings.ToUnifiedFormat(args.size(), strings_data);
+
+	UnifiedVectorFormat pattern_data;
+	patterns.ToUnifiedFormat(args.size(), pattern_data);
+
+	ListVector::Reserve(result, STANDARD_VECTOR_SIZE);
+	// Reference the 'strings' StringBuffer, because we won't need to allocate new data
+	// for the result, all returned strings are substrings of the originals
+	output_child.SetAuxiliary(strings.GetAuxiliary());
+
+	// Avoid doing extra work if all the inputs are constant
+	idx_t tuple_count = args.AllConstant() ? 1 : args.size();
+
+	unique_ptr<RegexStringPieceArgs> non_const_args;
+	unique_ptr<duckdb_re2::RE2> stored_re;
+	if (!info.constant_pattern) {
+		non_const_args = make_unique<RegexStringPieceArgs>();
+	} else {
+		// Verify that the constant pattern is valid
+		auto &re = GetPattern(info, state, stored_re);
+		auto group_count_p = re.NumberOfCapturingGroups();
+		if (group_count_p == -1) {
+			throw InvalidInputException("Pattern failed to parse, error: '%s'", re.error());
+		}
+	}
+
+	for (idx_t row = 0; row < tuple_count; row++) {
+		bool pattern_valid = true;
+		if (!info.constant_pattern) {
+			// Check if the pattern is NULL or not,
+			// and compile the pattern if it's not constant
+			auto pattern_idx = pattern_data.sel->get_index(row);
+			if (!pattern_data.validity.RowIsValid(pattern_idx)) {
+				pattern_valid = false;
+			} else {
+				auto &pattern_p = ((string_t *)pattern_data.data)[pattern_idx];
+				auto pattern_strpiece = CreateStringPiece(pattern_p);
+				stored_re = make_unique<duckdb_re2::RE2>(pattern_strpiece, info.options);
+
+				// Increase the size of the args buffer if needed
+				auto group_count_p = stored_re->NumberOfCapturingGroups();
+				if (group_count_p == -1) {
+					throw InvalidInputException("Pattern failed to parse, error: '%s'", stored_re->error());
+				}
+				non_const_args->SetSize(group_count_p);
+			}
+		}
+
+		auto string_idx = strings_data.sel->get_index(row);
+		int32_t group_index;
+		if (!pattern_valid || !strings_data.validity.RowIsValid(string_idx) || !GetGroupIndex(args, row, group_index)) {
+			// If something is NULL, the result is NULL
+			// FIXME: do we even need 'SPECIAL_HANDLING'?
+			auto result_data = FlatVector::GetData<list_entry_t>(result);
+			auto &result_validity = FlatVector::Validity(result);
+			result_data[row].length = 0;
+			result_data[row].offset = ListVector::GetListSize(result);
+			result_validity.SetInvalid(row);
+			continue;
+		}
+
+		auto &re = GetPattern(info, state, stored_re);
+		auto &groups = GetGroupsBuffer(info, state, non_const_args);
+		auto &string = ((string_t *)strings_data.data)[string_idx];
+		ExtractSingleTuple(string, re, group_index, groups, result, row);
+	}
+
+	if (args.AllConstant()) {
+		result.SetVectorType(VectorType::CONSTANT_VECTOR);
+	}
+}
+
+unique_ptr<FunctionData> RegexpExtractAll::Bind(ClientContext &context, ScalarFunction &bound_function,
+                                                vector<unique_ptr<Expression>> &arguments) {
+	D_ASSERT(arguments.size() >= 2);
+
+	duckdb_re2::RE2::Options options;
+
+	string constant_string;
+	bool constant_pattern = TryParseConstantPattern(context, *arguments[1], constant_string);
+
+	if (arguments.size() >= 4) {
+		ParseRegexOptions(context, *arguments[3], options);
+	}
+	return make_unique<RegexpExtractBindData>(options, std::move(constant_string), constant_pattern, "");
+}
+
+} // namespace duckdb

package/src/duckdb/src/function/scalar/string/regexp/regexp_util.cpp

@@ -0,0 +1,79 @@
+#include "duckdb/function/scalar/regexp.hpp"
+#include "duckdb/execution/expression_executor.hpp"
+
+namespace duckdb {
+
+namespace regexp_util {
+
+bool TryParseConstantPattern(ClientContext &context, Expression &expr, string &constant_string) {
+	if (!expr.IsFoldable()) {
+		return false;
+	}
+	Value pattern_str = ExpressionExecutor::EvaluateScalar(context, expr);
+	if (!pattern_str.IsNull() && pattern_str.type().id() == LogicalTypeId::VARCHAR) {
+		constant_string = StringValue::Get(pattern_str);
+		return true;
+	}
+	return false;
+}
+
+void ParseRegexOptions(const string &options, duckdb_re2::RE2::Options &result, bool *global_replace) {
+	for (idx_t i = 0; i < options.size(); i++) {
+		switch (options[i]) {
+		case 'c':
+			// case-sensitive matching
+			result.set_case_sensitive(true);
+			break;
+		case 'i':
+			// case-insensitive matching
+			result.set_case_sensitive(false);
+			break;
+		case 'l':
+			// literal matching
+			result.set_literal(true);
+			break;
+		case 'm':
+		case 'n':
+		case 'p':
+			// newline-sensitive matching
+			result.set_dot_nl(false);
+			break;
+		case 's':
+			// non-newline-sensitive matching
+			result.set_dot_nl(true);
+			break;
+		case 'g':
+			// global replace, only available for regexp_replace
+			if (global_replace) {
+				*global_replace = true;
+			} else {
+				throw InvalidInputException("Option 'g' (global replace) is only valid for regexp_replace");
+			}
+			break;
+		case ' ':
+		case '\t':
+		case '\n':
+			// ignore whitespace
+			break;
+		default:
+			throw InvalidInputException("Unrecognized Regex option %c", options[i]);
+		}
+	}
+}
+
+void ParseRegexOptions(ClientContext &context, Expression &expr, RE2::Options &target, bool *global_replace) {
+	if (expr.HasParameter()) {
+		throw ParameterNotResolvedException();
+	}
+	if (!expr.IsFoldable()) {
+		throw InvalidInputException("Regex options field must be a constant");
+	}
+	Value options_str = ExpressionExecutor::EvaluateScalar(context, expr);
+	if (!options_str.IsNull() && options_str.type().id() == LogicalTypeId::VARCHAR) {
+		ParseRegexOptions(StringValue::Get(options_str), target, global_replace);
+	}
+}
+
+} // namespace regexp_util
+
+} // namespace duckdb

package/src/duckdb/src/function/scalar/string/regexp.cpp

@@ -12,6 +12,11 @@
 
 namespace duckdb {
 
+using regexp_util::CreateStringPiece;
+using regexp_util::Extract;
+using regexp_util::ParseRegexOptions;
+using regexp_util::TryParseConstantPattern;
+
 static bool RegexOptionsEquals(const duckdb_re2::RE2::Options &opt_a, const duckdb_re2::RE2::Options &opt_b) {
 	return opt_a.case_sensitive() == opt_b.case_sensitive();
 }
@@ -32,10 +37,6 @@ bool RegexpBaseBindData::Equals(const FunctionData &other_p) const {
 	       RegexOptionsEquals(options, other.options);
 }
 
-static inline duckdb_re2::StringPiece CreateStringPiece(string_t &input) {
-	return duckdb_re2::StringPiece(input.GetDataUnsafe(), input.GetSize());
-}
-
 unique_ptr<FunctionLocalState> RegexInitLocalState(ExpressionState &state, const BoundFunctionExpression &expr,
                                                    FunctionData *bind_data) {
 	auto &info = (RegexpBaseBindData &)*bind_data;
@@ -45,75 +46,6 @@ unique_ptr<FunctionLocalState> RegexInitLocalState(ExpressionState &state, const
 	return nullptr;
 }
 
-static void ParseRegexOptions(const string &options, duckdb_re2::RE2::Options &result, bool *global_replace = nullptr) {
-	for (idx_t i = 0; i < options.size(); i++) {
-		switch (options[i]) {
-		case 'c':
-			// case-sensitive matching
-			result.set_case_sensitive(true);
-			break;
-		case 'i':
-			// case-insensitive matching
-			result.set_case_sensitive(false);
-			break;
-		case 'l':
-			// literal matching
-			result.set_literal(true);
-			break;
-		case 'm':
-		case 'n':
-		case 'p':
-			// newline-sensitive matching
-			result.set_dot_nl(false);
-			break;
-		case 's':
-			// non-newline-sensitive matching
-			result.set_dot_nl(true);
-			break;
-		case 'g':
-			// global replace, only available for regexp_replace
-			if (global_replace) {
-				*global_replace = true;
-			} else {
-				throw InvalidInputException("Option 'g' (global replace) is only valid for regexp_replace");
-			}
-			break;
-		case ' ':
-		case '\t':
-		case '\n':
-			// ignore whitespace
-			break;
-		default:
-			throw InvalidInputException("Unrecognized Regex option %c", options[i]);
-		}
-	}
-}
-
-void ParseRegexOptions(ClientContext &context, Expression &expr, RE2::Options &target, bool *global_replace = nullptr) {
-	if (expr.HasParameter()) {
-		throw ParameterNotResolvedException();
-	}
-	if (!expr.IsFoldable()) {
-		throw InvalidInputException("Regex options field must be a constant");
-	}
-	Value options_str = ExpressionExecutor::EvaluateScalar(context, expr);
-	if (!options_str.IsNull() && options_str.type().id() == LogicalTypeId::VARCHAR) {
-		ParseRegexOptions(StringValue::Get(options_str), target, global_replace);
-	}
-}
-
-static bool TryParseConstantPattern(ClientContext &context, Expression &expr, string &constant_string) {
-	if (!expr.IsFoldable()) {
-		return false;
-	}
-	Value pattern_str = ExpressionExecutor::EvaluateScalar(context, expr);
-	if (!pattern_str.IsNull() && pattern_str.type().id() == LogicalTypeId::VARCHAR) {
-		constant_string = StringValue::Get(pattern_str);
-		return true;
-	}
-	return false;
-}
-
 //===--------------------------------------------------------------------===//
 // Regexp Matches
 //===--------------------------------------------------------------------===//
@@ -321,13 +253,6 @@ static unique_ptr<FunctionData> RegexExtractBind(ClientContext &context, ScalarF
 	                                          std::move(group_string));
 }
 
-inline static string_t Extract(const string_t &input, Vector &result, const RE2 &re,
-                               const duckdb_re2::StringPiece &rewrite) {
-	std::string extracted;
-	RE2::Extract(input.GetString(), re, rewrite, &extracted);
-	return StringVector::AddString(result, extracted.c_str(), extracted.size());
-}
-
 static void RegexExtractFunction(DataChunk &args, ExpressionState &state, Vector &result) {
 	auto &func_expr = (BoundFunctionExpression &)state.expr;
 	const auto &info = (RegexpExtractBindData &)*func_expr.bind_info;
@@ -391,10 +316,26 @@ void RegexpFun::RegisterFunction(BuiltinFunctions &set) {
 	    RegexExtractFunction, RegexExtractBind, nullptr, nullptr, RegexInitLocalState, LogicalType::INVALID,
 	    FunctionSideEffects::NO_SIDE_EFFECTS, FunctionNullHandling::SPECIAL_HANDLING));
 
+	ScalarFunctionSet regexp_extract_all("regexp_extract_all");
+	regexp_extract_all.AddFunction(ScalarFunction(
+	    {LogicalType::VARCHAR, LogicalType::VARCHAR}, LogicalType::LIST(LogicalType::VARCHAR),
+	    RegexpExtractAll::Execute, RegexpExtractAll::Bind, nullptr, nullptr, RegexpExtractAll::InitLocalState,
+	    LogicalType::INVALID, FunctionSideEffects::NO_SIDE_EFFECTS, FunctionNullHandling::SPECIAL_HANDLING));
+	regexp_extract_all.AddFunction(ScalarFunction(
+	    {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::INTEGER}, LogicalType::LIST(LogicalType::VARCHAR),
+	    RegexpExtractAll::Execute, RegexpExtractAll::Bind, nullptr, nullptr, RegexpExtractAll::InitLocalState,
+	    LogicalType::INVALID, FunctionSideEffects::NO_SIDE_EFFECTS, FunctionNullHandling::SPECIAL_HANDLING));
+	regexp_extract_all.AddFunction(
+	    ScalarFunction({LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::INTEGER, LogicalType::VARCHAR},
+	                   LogicalType::LIST(LogicalType::VARCHAR), RegexpExtractAll::Execute, RegexpExtractAll::Bind,
+	                   nullptr, nullptr, RegexpExtractAll::InitLocalState, LogicalType::INVALID,
+	                   FunctionSideEffects::NO_SIDE_EFFECTS, FunctionNullHandling::SPECIAL_HANDLING));
+
 	set.AddFunction(regexp_full_match);
 	set.AddFunction(regexp_partial_match);
 	set.AddFunction(regexp_replace);
 	set.AddFunction(regexp_extract);
+	set.AddFunction(regexp_extract_all);
 }
 
 } // namespace duckdb

package/src/duckdb/src/function/table/arrow_conversion.cpp

@@ -20,7 +20,13 @@ void ShiftRight(unsigned char *ar, int size, int shift) {
 
 void GetValidityMask(ValidityMask &mask, ArrowArray &array, ArrowScanLocalState &scan_state, idx_t size,
                      int64_t nested_offset = -1, bool add_null = false) {
-	if (array.null_count != 0 && array.buffers[0]) {
+	// In certains we don't need to or cannot copy arrow's validity mask to duckdb.
+	//
+	// The conditions where we do want to copy arrow's mask to duckdb are:
+	// 1. nulls exist
+	// 2. n_buffers > 0, meaning the array's arrow type is not `null`
+	// 3. the validity buffer (the first buffer) is not a nullptr
+	if (array.null_count != 0 && array.n_buffers > 0 && array.buffers[0]) {
 		auto bit_offset = scan_state.chunk_offset + array.offset;
 		if (nested_offset != -1) {
 			bit_offset = nested_offset;

package/src/duckdb/src/function/table/table_scan.cpp

@@ -332,7 +332,7 @@ void TableScanPushdownComplexFilter(ClientContext &context, LogicalGet &get, Fun
 				auto comparison_type = comparison->type;
 				if (comparison->left->type == ExpressionType::VALUE_CONSTANT) {
 					// the expression is on the right side, we flip them around
-					comparison_type = FlipComparisionExpression(comparison_type);
+					comparison_type = FlipComparisonExpression(comparison_type);
 				}
 				if (comparison_type == ExpressionType::COMPARE_EQUAL) {
 					// equality value

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.7.2-dev717"
+#define DUCKDB_VERSION "0.7.2-dev865"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "cd47ad8e2d"
+#define DUCKDB_SOURCE_ID "61325fdd83"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/catalog/catalog.hpp

@@ -14,6 +14,7 @@
 #include "duckdb/catalog/catalog_transaction.hpp"
 #include "duckdb/common/unordered_set.hpp"
 #include "duckdb/common/atomic.hpp"
+#include "duckdb/common/optional_ptr.hpp"
 
 #include <functional>
 
@@ -80,6 +81,8 @@ public:
 	DUCKDB_API static Catalog &GetCatalog(ClientContext &context, const string &catalog_name);
 	//! Get the specified Catalog from the DatabaseInstance
 	DUCKDB_API static Catalog &GetCatalog(DatabaseInstance &db, const string &catalog_name);
+	//! Gets the specified Catalog from the database if it exists
+	DUCKDB_API static optional_ptr<Catalog> GetCatalogEntry(ClientContext &context, const string &catalog_name);
 	//! Get the specific Catalog from the AttachedDatabase
 	DUCKDB_API static Catalog &GetCatalog(AttachedDatabase &db);
 

package/src/duckdb/src/include/duckdb/common/enums/aggregate_handling.hpp

@@ -19,4 +19,6 @@ enum class AggregateHandling : uint8_t {
 	FORCE_AGGREGATES       // force aggregates: any non-aggregate select list entry will become a GROUP
 };
 
+const char *ToString(AggregateHandling value);
+
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/enums/expression_type.hpp

@@ -205,11 +205,10 @@ string ExpressionTypeToOperator(ExpressionType type);
 
 // Operator String to ExpressionType (e.g. + => OPERATOR_ADD)
 ExpressionType OperatorToExpressionType(const string &op);
-
 //! Negate a comparison expression, turning e.g. = into !=, or < into >=
-ExpressionType NegateComparisionExpression(ExpressionType type);
+ExpressionType NegateComparisonExpression(ExpressionType type);
 //! Flip a comparison expression, turning e.g. < into >, or = into =
-ExpressionType FlipComparisionExpression(ExpressionType type);
+ExpressionType FlipComparisonExpression(ExpressionType type);
 
 DUCKDB_API string ExpressionClassToString(ExpressionClass type);
 

package/src/duckdb/src/include/duckdb/common/enums/joinref_type.hpp

@@ -22,4 +22,6 @@ enum class JoinRefType : uint8_t {
 	POSITIONAL // Positional condition
 };
 
+const char *ToString(JoinRefType value);
+
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/enums/order_type.hpp

@@ -9,10 +9,12 @@
 #pragma once
 
 #include "duckdb/common/constants.hpp"
+#include "duckdb/common/exception.hpp"
 
 namespace duckdb {
 
 enum class OrderType : uint8_t { INVALID = 0, ORDER_DEFAULT = 1, ASCENDING = 2, DESCENDING = 3 };
+
 enum class OrderByNullType : uint8_t { INVALID = 0, ORDER_DEFAULT = 1, NULLS_FIRST = 2, NULLS_LAST = 3 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/enums/set_operation_type.hpp

@@ -13,4 +13,5 @@
 namespace duckdb {
 
 enum class SetOperationType : uint8_t { NONE = 0, UNION = 1, EXCEPT = 2, INTERSECT = 3, UNION_BY_NAME = 4 };
-}
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/exception.hpp

@@ -13,6 +13,7 @@
 #include "duckdb/common/exception_format_value.hpp"
 #include "duckdb/common/vector.hpp"
 
+#include <map>
 #include <stdexcept>
 
 namespace duckdb {
@@ -283,29 +284,59 @@ public:
 
 class HTTPException : public IOException {
 public:
-	DUCKDB_API explicit HTTPException(int status_code, string response, const string &msg)
-	    : IOException(ExceptionType::HTTP, msg), status_code(status_code), response(std::move(response)) {
+	template <typename>
+	struct ResponseShape {
+		typedef int status;
+	};
+
+	template <class RESPONSE, typename ResponseShape<decltype(RESPONSE::status)>::status = 0, typename... ARGS>
+	explicit HTTPException(RESPONSE &response, const string &msg, ARGS... params)
+	    : HTTPException(response.status, response.body, response.headers, response.reason, msg, params...) {
+	}
+
+	template <typename>
+	struct ResponseWrapperShape {
+		typedef int code;
+	};
+	template <class RESPONSE, typename ResponseWrapperShape<decltype(RESPONSE::code)>::code = 0, typename... ARGS>
+	explicit HTTPException(RESPONSE &response, const string &msg, ARGS... params)
+	    : HTTPException(response.code, response.body, response.headers, response.error, msg, params...) {
 	}
 
-	template <typename... ARGS>
-	explicit HTTPException(int status_code, string response, const string &msg, ARGS... params)
-	    : HTTPException(status_code, std::move(response), ConstructMessage(msg, params...)) {
+	template <typename HEADERS, typename... ARGS>
+	explicit HTTPException(int status_code, string response_body, HEADERS headers, const string &reason,
+	                       const string &msg, ARGS... params)
+	    : IOException(ExceptionType::HTTP, ConstructMessage(msg, params...)), status_code(status_code), reason(reason),
+	      response_body(std::move(response_body)) {
+		this->headers.insert(headers.begin(), headers.end());
+		D_ASSERT(this->headers.size() > 0);
 	}
 
 	std::shared_ptr<Exception> Copy() const {
-		return make_shared<HTTPException>(status_code, response, RawMessage());
+		return make_shared<HTTPException>(status_code, response_body, headers, reason, RawMessage());
 	}
 
+	const std::multimap<std::string, std::string> GetHeaders() const {
+		return headers;
+	}
 	int GetStatusCode() const {
 		return status_code;
 	}
-	const string &GetResponse() const {
-		return response;
+	const string &GetResponseBody() const {
+		return response_body;
+	}
+	const string &GetReason() const {
+		return reason;
+	}
+	[[noreturn]] void Throw() const {
+		throw HTTPException(status_code, response_body, headers, reason, RawMessage());
 	}
 
 private:
 	int status_code;
-	string response; // we can keep a copy for the user
+	string reason;
+	string response_body;
+	std::multimap<string, string> headers;
 };
 
 class SerializationException : public Exception {

package/src/duckdb/src/include/duckdb/common/preserved_error.hpp

@@ -39,6 +39,9 @@ public:
 	//! Let's us do things like 'if (error)'
 	DUCKDB_API operator bool() const;
 	DUCKDB_API bool operator==(const PreservedError &other) const;
+	DUCKDB_API const shared_ptr<Exception> &GetError() {
+		return exception_instance;
+	}
 
 private:
 	//! Whether this PreservedError contains an exception or not