duckdb 0.9.3-dev2.0 → 0.10.0

package/src/duckdb/src/main/materialized_query_result.cpp

@@ -13,7 +13,7 @@ MaterializedQueryResult::MaterializedQueryResult(StatementType statement_type, S
       collection(std::move(collection_p)), scan_initialized(false) {
 }
 
-MaterializedQueryResult::MaterializedQueryResult(PreservedError error)
+MaterializedQueryResult::MaterializedQueryResult(ErrorData error)
     : QueryResult(QueryResultType::MATERIALIZED_RESULT, std::move(error)), scan_initialized(false) {
 }
 

package/src/duckdb/src/main/pending_query_result.cpp

@@ -11,7 +11,7 @@ PendingQueryResult::PendingQueryResult(shared_ptr<ClientContext> context_p, Prep
       context(std::move(context_p)), allow_stream_result(allow_stream_result) {
 }
 
-PendingQueryResult::PendingQueryResult(PreservedError error)
+PendingQueryResult::PendingQueryResult(ErrorData error)
     : BaseQueryResult(QueryResultType::PENDING_RESULT, std::move(error)) {
 }
 
@@ -32,7 +32,7 @@ unique_ptr<ClientContextLock> PendingQueryResult::LockContext() {
 void PendingQueryResult::CheckExecutableInternal(ClientContextLock &lock) {
 	bool invalidated = HasError() || !context;
 	if (!invalidated) {
-		invalidated = !context->IsActiveResult(lock, this);
+		invalidated = !context->IsActiveResult(lock, *this);
 	}
 	if (invalidated) {
 		if (HasError()) {
@@ -48,15 +48,30 @@ PendingExecutionResult PendingQueryResult::ExecuteTask() {
 	return ExecuteTaskInternal(*lock);
 }
 
+PendingExecutionResult PendingQueryResult::CheckPulse() {
+	auto lock = LockContext();
+	CheckExecutableInternal(*lock);
+	return context->ExecuteTaskInternal(*lock, *this, true);
+}
+
+bool PendingQueryResult::AllowStreamResult() const {
+	return allow_stream_result;
+}
+
 PendingExecutionResult PendingQueryResult::ExecuteTaskInternal(ClientContextLock &lock) {
 	CheckExecutableInternal(lock);
-	return context->ExecuteTaskInternal(lock, *this);
+	return context->ExecuteTaskInternal(lock, *this, false);
 }
 
 unique_ptr<QueryResult> PendingQueryResult::ExecuteInternal(ClientContextLock &lock) {
 	CheckExecutableInternal(lock);
 	// Busy wait while execution is not finished
-	while (!IsFinished(ExecuteTaskInternal(lock))) {
+	if (allow_stream_result) {
+		while (!IsFinishedOrBlocked(ExecuteTaskInternal(lock))) {
+		}
+	} else {
+		while (!IsFinished(ExecuteTaskInternal(lock))) {
+		}
 	}
 	if (HasError()) {
 		return make_uniq<MaterializedQueryResult>(error);
@@ -76,10 +91,12 @@ void PendingQueryResult::Close() {
 }
 
 bool PendingQueryResult::IsFinished(PendingExecutionResult result) {
-	if (result == PendingExecutionResult::RESULT_READY || result == PendingExecutionResult::EXECUTION_ERROR) {
-		return true;
-	}
-	return false;
+	return (result == PendingExecutionResult::RESULT_READY || result == PendingExecutionResult::EXECUTION_ERROR);
+}
+
+bool PendingQueryResult::IsFinishedOrBlocked(PendingExecutionResult result) {
+	return (result == PendingExecutionResult::RESULT_READY || result == PendingExecutionResult::EXECUTION_ERROR ||
+	        result == PendingExecutionResult::BLOCKED);
 }
 
 } // namespace duckdb

package/src/duckdb/src/main/prepared_statement.cpp

@@ -12,7 +12,7 @@ PreparedStatement::PreparedStatement(shared_ptr<ClientContext> context, shared_p
 	D_ASSERT(data || !success);
 }
 
-PreparedStatement::PreparedStatement(PreservedError error) : context(nullptr), success(false), error(std::move(error)) {
+PreparedStatement::PreparedStatement(ErrorData error) : context(nullptr), success(false), error(std::move(error)) {
 }
 
 PreparedStatement::~PreparedStatement() {
@@ -23,7 +23,7 @@ const string &PreparedStatement::GetError() {
 	return error.Message();
 }
 
-PreservedError &PreparedStatement::GetErrorObject() {
+ErrorData &PreparedStatement::GetErrorObject() {
 	return error;
 }
 
@@ -98,15 +98,15 @@ unique_ptr<PendingQueryResult> PreparedStatement::PendingQuery(case_insensitive_
                                                                bool allow_stream_result) {
 	if (!success) {
 		auto exception = InvalidInputException("Attempting to execute an unsuccessfully prepared statement!");
-		return make_uniq<PendingQueryResult>(PreservedError(exception));
+		return make_uniq<PendingQueryResult>(ErrorData(exception));
 	}
 	PendingQueryParameters parameters;
 	parameters.parameters = &named_values;
 
 	try {
 		VerifyParameters(named_values, named_param_map);
-	} catch (const Exception &ex) {
-		return make_uniq<PendingQueryResult>(PreservedError(ex));
+	} catch (const std::exception &ex) {
+		return make_uniq<PendingQueryResult>(ErrorData(ex));
 	}
 
 	D_ASSERT(data);

package/src/duckdb/src/main/prepared_statement_data.cpp

@@ -1,6 +1,7 @@
 #include "duckdb/main/prepared_statement_data.hpp"
 #include "duckdb/execution/physical_operator.hpp"
 #include "duckdb/parser/sql_statement.hpp"
+#include "duckdb/common/exception/binder_exception.hpp"
 
 namespace duckdb {
 
@@ -25,6 +26,10 @@ bool PreparedStatementData::RequireRebind(ClientContext &context, optional_ptr<c
 		// no unbound statement!? cannot rebind?
 		return false;
 	}
+	if (properties.always_require_rebind) {
+		// this statement must always be re-bound
+		return true;
+	}
 	if (!properties.bound_all_parameters) {
 		// parameters not yet bound: query always requires a rebind
 		return true;
@@ -36,7 +41,9 @@ bool PreparedStatementData::RequireRebind(ClientContext &context, optional_ptr<c
 	for (auto &it : value_map) {
 		auto &identifier = it.first;
 		auto lookup = values->find(identifier);
-		D_ASSERT(lookup != values->end());
+		if (lookup == values->end()) {
+			break;
+		}
 		if (lookup->second.type() != it.second->return_type) {
 			return true;
 		}

package/src/duckdb/src/main/query_profiler.cpp

@@ -9,7 +9,7 @@
 #include "duckdb/common/tree_renderer.hpp"
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/execution/operator/helper/physical_execute.hpp"
-#include "duckdb/execution/operator/join/physical_delim_join.hpp"
+#include "duckdb/execution/operator/join/physical_left_delim_join.hpp"
 #include "duckdb/execution/physical_operator.hpp"
 #include "duckdb/main/client_config.hpp"
 #include "duckdb/main/client_context.hpp"
@@ -101,7 +101,8 @@ bool QueryProfiler::OperatorRequiresProfiling(PhysicalOperatorType op_type) {
 	case PhysicalOperatorType::CROSS_PRODUCT:
 	case PhysicalOperatorType::PIECEWISE_MERGE_JOIN:
 	case PhysicalOperatorType::IE_JOIN:
-	case PhysicalOperatorType::DELIM_JOIN:
+	case PhysicalOperatorType::LEFT_DELIM_JOIN:
+	case PhysicalOperatorType::RIGHT_DELIM_JOIN:
 	case PhysicalOperatorType::UNION:
 	case PhysicalOperatorType::RECURSIVE_CTE:
 	case PhysicalOperatorType::EMPTY_RESULT:
@@ -381,15 +382,14 @@ void QueryProfiler::QueryTreeToStream(std::ostream &ss) const {
 		return;
 	}
 
-	if (context.client_data->http_state && !context.client_data->http_state->IsEmpty()) {
-		string read =
-		    "in: " + StringUtil::BytesToHumanReadableString(context.client_data->http_state->total_bytes_received);
-		string written =
-		    "out: " + StringUtil::BytesToHumanReadableString(context.client_data->http_state->total_bytes_sent);
-		string head = "#HEAD: " + to_string(context.client_data->http_state->head_count);
-		string get = "#GET: " + to_string(context.client_data->http_state->get_count);
-		string put = "#PUT: " + to_string(context.client_data->http_state->put_count);
-		string post = "#POST: " + to_string(context.client_data->http_state->post_count);
+	auto http_state = HTTPState::TryGetState(context, false);
+	if (http_state && !http_state->IsEmpty()) {
+		string read = "in: " + StringUtil::BytesToHumanReadableString(http_state->total_bytes_received);
+		string written = "out: " + StringUtil::BytesToHumanReadableString(http_state->total_bytes_sent);
+		string head = "#HEAD: " + to_string(http_state->head_count);
+		string get = "#GET: " + to_string(http_state->get_count);
+		string put = "#PUT: " + to_string(http_state->put_count);
+		string post = "#POST: " + to_string(http_state->post_count);
 
 		constexpr idx_t TOTAL_BOX_WIDTH = 39;
 		ss << "┌─────────────────────────────────────┐\n";

package/src/duckdb/src/main/query_result.cpp

@@ -13,8 +13,10 @@ BaseQueryResult::BaseQueryResult(QueryResultType type, StatementType statement_t
 	D_ASSERT(types.size() == names.size());
 }
 
-BaseQueryResult::BaseQueryResult(QueryResultType type, PreservedError error)
+BaseQueryResult::BaseQueryResult(QueryResultType type, ErrorData error)
     : type(type), success(false), error(std::move(error)) {
+	// Assert that the error object is initialized
+	D_ASSERT(this->error.HasError());
 }
 
 BaseQueryResult::~BaseQueryResult() {
@@ -25,13 +27,13 @@ void BaseQueryResult::ThrowError(const string &prepended_message) const {
 	error.Throw(prepended_message);
 }
 
-void BaseQueryResult::SetError(PreservedError error) {
-	success = !error;
+void BaseQueryResult::SetError(ErrorData error) {
+	success = !error.HasError();
 	this->error = std::move(error);
 }
 
 bool BaseQueryResult::HasError() const {
-	D_ASSERT((bool)error == !success);
+	D_ASSERT(error.HasError() == !success);
 	return !success;
 }
 
@@ -44,7 +46,7 @@ const std::string &BaseQueryResult::GetError() {
 	return error.Message();
 }
 
-PreservedError &BaseQueryResult::GetErrorObject() {
+ErrorData &BaseQueryResult::GetErrorObject() {
 	return error;
 }
 
@@ -58,13 +60,37 @@ QueryResult::QueryResult(QueryResultType type, StatementType statement_type, Sta
       client_properties(std::move(client_properties_p)) {
 }
 
-QueryResult::QueryResult(QueryResultType type, PreservedError error)
+QueryResult::QueryResult(QueryResultType type, ErrorData error)
     : BaseQueryResult(type, std::move(error)), client_properties("UTC", ArrowOffsetSize::REGULAR) {
 }
 
 QueryResult::~QueryResult() {
 }
 
+void QueryResult::DeduplicateColumns(vector<string> &names) {
+	unordered_map<string, idx_t> name_map;
+	for (auto &column_name : names) {
+		// put it all lower_case
+		auto low_column_name = StringUtil::Lower(column_name);
+		if (name_map.find(low_column_name) == name_map.end()) {
+			// Name does not exist yet
+			name_map[low_column_name]++;
+		} else {
+			// Name already exists, we add _x where x is the repetition number
+			string new_column_name = column_name + "_" + std::to_string(name_map[low_column_name]);
+			auto new_column_name_low = StringUtil::Lower(new_column_name);
+			while (name_map.find(new_column_name_low) != name_map.end()) {
+				// This name is already here due to a previous definition
+				name_map[low_column_name]++;
+				new_column_name = column_name + "_" + std::to_string(name_map[low_column_name]);
+				new_column_name_low = StringUtil::Lower(new_column_name);
+			}
+			column_name = new_column_name;
+			name_map[new_column_name_low]++;
+		}
+	}
+}
+
 const string &QueryResult::ColumnName(idx_t index) const {
 	D_ASSERT(index < names.size());
 	return names[index];

package/src/duckdb/src/main/relation/cross_product_relation.cpp

@@ -11,7 +11,7 @@ CrossProductRelation::CrossProductRelation(shared_ptr<Relation> left_p, shared_p
     : Relation(left_p->context, RelationType::CROSS_PRODUCT_RELATION), left(std::move(left_p)),
       right(std::move(right_p)), ref_type(ref_type) {
 	if (left->context.GetContext() != right->context.GetContext()) {
-		throw Exception("Cannot combine LEFT and RIGHT relations of different connections!");
+		throw InvalidInputException("Cannot combine LEFT and RIGHT relations of different connections!");
 	}
 	context.GetContext()->TryBindRelation(*this, this->columns);
 }

package/src/duckdb/src/main/relation/join_relation.cpp

@@ -12,7 +12,7 @@ JoinRelation::JoinRelation(shared_ptr<Relation> left_p, shared_ptr<Relation> rig
     : Relation(left_p->context, RelationType::JOIN_RELATION), left(std::move(left_p)), right(std::move(right_p)),
       condition(std::move(condition_p)), join_type(type), join_ref_type(join_ref_type) {
 	if (left->context.GetContext() != right->context.GetContext()) {
-		throw Exception("Cannot combine LEFT and RIGHT relations of different connections!");
+		throw InvalidInputException("Cannot combine LEFT and RIGHT relations of different connections!");
 	}
 	context.GetContext()->TryBindRelation(*this, this->columns);
 }
@@ -22,7 +22,7 @@ JoinRelation::JoinRelation(shared_ptr<Relation> left_p, shared_ptr<Relation> rig
     : Relation(left_p->context, RelationType::JOIN_RELATION), left(std::move(left_p)), right(std::move(right_p)),
       using_columns(std::move(using_columns_p)), join_type(type), join_ref_type(join_ref_type) {
 	if (left->context.GetContext() != right->context.GetContext()) {
-		throw Exception("Cannot combine LEFT and RIGHT relations of different connections!");
+		throw InvalidInputException("Cannot combine LEFT and RIGHT relations of different connections!");
 	}
 	context.GetContext()->TryBindRelation(*this, this->columns);
 }

package/src/duckdb/src/main/relation/read_csv_relation.cpp

@@ -1,14 +1,13 @@
 #include "duckdb/main/relation/read_csv_relation.hpp"
 
-#include "duckdb/execution/operator/scan/csv/buffered_csv_reader.hpp"
-#include "duckdb/execution/operator/scan/csv/csv_buffer_manager.hpp"
-#include "duckdb/execution/operator/scan/csv/csv_sniffer.hpp"
+#include "duckdb/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.hpp"
+#include "duckdb/execution/operator/csv_scanner/sniffer/csv_sniffer.hpp"
 #include "duckdb/parser/expression/columnref_expression.hpp"
 #include "duckdb/parser/expression/comparison_expression.hpp"
 #include "duckdb/parser/expression/constant_expression.hpp"
 #include "duckdb/parser/expression/function_expression.hpp"
 #include "duckdb/common/string_util.hpp"
-#include "duckdb/execution/operator/scan/csv/csv_reader_options.hpp"
+#include "duckdb/execution/operator/csv_scanner/options/csv_reader_options.hpp"
 #include "duckdb/common/multi_file_reader.hpp"
 #include "duckdb/parser/expression/star_expression.hpp"
 #include "duckdb/parser/query_node/select_node.hpp"
@@ -17,39 +16,32 @@
 
 namespace duckdb {
 
-ReadCSVRelation::ReadCSVRelation(const shared_ptr<ClientContext> &context, const string &csv_file,
-                                 vector<ColumnDefinition> columns_p, string alias_p)
-    : TableFunctionRelation(context, "read_csv", {Value(csv_file)}, nullptr, false), alias(std::move(alias_p)),
-      auto_detect(false) {
-
-	if (alias.empty()) {
-		alias = StringUtil::Split(csv_file, ".")[0];
-	}
-
-	columns = std::move(columns_p);
+void ReadCSVRelation::InitializeAlias(const vector<string> &input) {
+	D_ASSERT(!input.empty());
+	const auto &csv_file = input[0];
+	alias = StringUtil::Split(csv_file, ".")[0];
+}
 
-	child_list_t<Value> column_names;
-	for (idx_t i = 0; i < columns.size(); i++) {
-		column_names.push_back(make_pair(columns[i].Name(), Value(columns[i].Type().ToString())));
+static Value CreateValueFromFileList(const vector<string> &file_list) {
+	vector<Value> files;
+	for (auto &file : file_list) {
+		files.push_back(file);
 	}
-
-	AddNamedParameter("columns", Value::STRUCT(std::move(column_names)));
+	return Value::LIST(std::move(files));
 }
 
-ReadCSVRelation::ReadCSVRelation(const std::shared_ptr<ClientContext> &context, const string &csv_file,
+ReadCSVRelation::ReadCSVRelation(const std::shared_ptr<ClientContext> &context, const vector<string> &input,
                                  named_parameter_map_t &&options, string alias_p)
-    : TableFunctionRelation(context, "read_csv_auto", {Value(csv_file)}, nullptr, false), alias(std::move(alias_p)),
-      auto_detect(true) {
+    : TableFunctionRelation(context, "read_csv_auto", {CreateValueFromFileList(input)}, nullptr, false),
+      alias(std::move(alias_p)) {
 
-	if (alias.empty()) {
-		alias = StringUtil::Split(csv_file, ".")[0];
-	}
+	InitializeAlias(input);
 
-	auto files = MultiFileReader::GetFileList(*context, csv_file, "CSV");
+	auto file_list = CreateValueFromFileList(input);
+	auto files = MultiFileReader::GetFileList(*context, file_list, "CSV");
 	D_ASSERT(!files.empty());
 
 	auto &file_name = files[0];
-	options["auto_detect"] = Value::BOOLEAN(true);
 	CSVReaderOptions csv_options;
 	csv_options.file_path = file_name;
 	vector<string> empty;
@@ -57,12 +49,11 @@ ReadCSVRelation::ReadCSVRelation(const std::shared_ptr<ClientContext> &context,
 	vector<LogicalType> unused_types;
 	vector<string> unused_names;
 	csv_options.FromNamedParameters(options, *context, unused_types, unused_names);
+
 	// Run the auto-detect, populating the options with the detected settings
 
-	auto bm_file_handle = BaseCSVReader::OpenCSV(*context, csv_options);
-	auto buffer_manager = make_shared<CSVBufferManager>(*context, std::move(bm_file_handle), csv_options);
-	CSVStateMachineCache state_machine_cache;
-	CSVSniffer sniffer(csv_options, buffer_manager, state_machine_cache);
+	auto buffer_manager = make_shared<CSVBufferManager>(*context, csv_options, files[0], 0);
+	CSVSniffer sniffer(csv_options, buffer_manager, CSVStateMachineCache::Get(*context));
 	auto sniffer_result = sniffer.SniffCSV();
 	auto &types = sniffer_result.return_types;
 	auto &names = sniffer_result.names;
@@ -70,12 +61,27 @@ ReadCSVRelation::ReadCSVRelation(const std::shared_ptr<ClientContext> &context,
 		columns.emplace_back(names[i], types[i]);
 	}
 
-	//! Capture the options potentially set/altered by the auto detection phase
+	// After sniffing we can consider these set, so they are exported as named parameters
+	// FIXME: This is horribly hacky, should be refactored at some point
+	csv_options.dialect_options.state_machine_options.escape.ChangeSetByUserTrue();
+	csv_options.dialect_options.state_machine_options.delimiter.ChangeSetByUserTrue();
+	csv_options.dialect_options.state_machine_options.quote.ChangeSetByUserTrue();
+	csv_options.dialect_options.header.ChangeSetByUserTrue();
+	csv_options.dialect_options.skip_rows.ChangeSetByUserTrue();
+
+	// Capture the options potentially set/altered by the auto detection phase
 	csv_options.ToNamedParameters(options);
 
 	// No need to auto-detect again
 	options["auto_detect"] = Value::BOOLEAN(false);
 	SetNamedParameters(std::move(options));
+
+	child_list_t<Value> column_names;
+	for (idx_t i = 0; i < columns.size(); i++) {
+		column_names.push_back(make_pair(columns[i].Name(), Value(columns[i].Type().ToString())));
+	}
+
+	AddNamedParameter("columns", Value::STRUCT(std::move(column_names)));
 }
 
 string ReadCSVRelation::GetAlias() {

package/src/duckdb/src/main/relation/setop_relation.cpp

@@ -5,11 +5,12 @@
 
 namespace duckdb {
 
-SetOpRelation::SetOpRelation(shared_ptr<Relation> left_p, shared_ptr<Relation> right_p, SetOperationType setop_type_p)
+SetOpRelation::SetOpRelation(shared_ptr<Relation> left_p, shared_ptr<Relation> right_p, SetOperationType setop_type_p,
+                             bool setop_all)
     : Relation(left_p->context, RelationType::SET_OPERATION_RELATION), left(std::move(left_p)),
-      right(std::move(right_p)), setop_type(setop_type_p) {
+      right(std::move(right_p)), setop_type(setop_type_p), setop_all(setop_all) {
 	if (left->context.GetContext() != right->context.GetContext()) {
-		throw Exception("Cannot combine LEFT and RIGHT relations of different connections!");
+		throw InvalidInputException("Cannot combine LEFT and RIGHT relations of different connections!");
 	}
 	context.GetContext()->TryBindRelation(*this, this->columns);
 }
@@ -22,6 +23,7 @@ unique_ptr<QueryNode> SetOpRelation::GetQueryNode() {
 	result->left = left->GetQueryNode();
 	result->right = right->GetQueryNode();
 	result->setop_type = setop_type;
+	result->setop_all = setop_all;
 	return std::move(result);
 }
 

package/src/duckdb/src/main/relation.cpp

@@ -161,15 +161,15 @@ shared_ptr<Relation> Relation::CrossProduct(const shared_ptr<Relation> &other, J
 }
 
 shared_ptr<Relation> Relation::Union(const shared_ptr<Relation> &other) {
-	return make_shared<SetOpRelation>(shared_from_this(), other, SetOperationType::UNION);
+	return make_shared<SetOpRelation>(shared_from_this(), other, SetOperationType::UNION, true);
 }
 
 shared_ptr<Relation> Relation::Except(const shared_ptr<Relation> &other) {
-	return make_shared<SetOpRelation>(shared_from_this(), other, SetOperationType::EXCEPT);
+	return make_shared<SetOpRelation>(shared_from_this(), other, SetOperationType::EXCEPT, true);
 }
 
 shared_ptr<Relation> Relation::Intersect(const shared_ptr<Relation> &other) {
-	return make_shared<SetOpRelation>(shared_from_this(), other, SetOperationType::INTERSECT);
+	return make_shared<SetOpRelation>(shared_from_this(), other, SetOperationType::INTERSECT, true);
 }
 
 shared_ptr<Relation> Relation::Distinct() {
@@ -334,11 +334,11 @@ unique_ptr<QueryResult> Relation::Explain(ExplainType type) {
 }
 
 void Relation::Update(const string &update, const string &condition) {
-	throw Exception("UPDATE can only be used on base tables!");
+	throw InvalidInputException("UPDATE can only be used on base tables!");
 }
 
 void Relation::Delete(const string &condition) {
-	throw Exception("DELETE can only be used on base tables!");
+	throw InvalidInputException("DELETE can only be used on base tables!");
 }
 
 shared_ptr<Relation> Relation::TableFunction(const std::string &fname, const vector<Value> &values,

package/src/duckdb/src/main/secret/secret.cpp

@@ -0,0 +1,135 @@
+#include "duckdb/main/secret/secret.hpp"
+#include "duckdb/parser/parsed_data/create_info.hpp"
+#include "duckdb/planner/logical_operator.hpp"
+#include "duckdb/common/case_insensitive_map.hpp"
+#include "duckdb/common/pair.hpp"
+
+namespace duckdb {
+
+int64_t BaseSecret::MatchScore(const string &path) const {
+	int64_t longest_match = NumericLimits<int64_t>::Minimum();
+	for (const auto &prefix : prefix_paths) {
+		// Handle empty scope which matches all at lowest possible score
+		if (prefix.empty()) {
+			longest_match = 0;
+			continue;
+		}
+		if (StringUtil::StartsWith(path, prefix)) {
+			longest_match = MaxValue<int>(prefix.length(), longest_match);
+		}
+	}
+	return longest_match;
+}
+
+void BaseSecret::SerializeBaseSecret(Serializer &serializer) const {
+	serializer.WriteProperty(100, "type", type);
+	serializer.WriteProperty(101, "provider", provider);
+	serializer.WriteProperty(102, "name", name);
+	serializer.WriteList(103, "scope", prefix_paths.size(),
+	                     [&](Serializer::List &list, idx_t i) { list.WriteElement(prefix_paths[i]); });
+}
+
+string BaseSecret::ToString(SecretDisplayType display_type) const {
+	return "";
+}
+
+void BaseSecret::Serialize(Serializer &serializer) const {
+	throw InternalException("Attempted to serialize secret without serialize");
+}
+
+string KeyValueSecret::ToString(SecretDisplayType mode) const {
+	string result;
+
+	result += "name=" + name + ";";
+	result += "type=" + type + ";";
+	result += "provider=" + provider + ";";
+	result += string("serializable=") + (serializable ? "true" : "false") + ";";
+	result += "scope=";
+	for (const auto &scope_it : prefix_paths) {
+		result += scope_it + ",";
+	}
+	result = result.substr(0, result.size() - 1);
+	result += ";";
+	for (auto it = secret_map.begin(); it != secret_map.end(); it++) {
+		result.append(it->first);
+		result.append("=");
+		if (mode == SecretDisplayType::REDACTED && redact_keys.find(it->first) != redact_keys.end()) {
+			result.append("redacted");
+		} else {
+			result.append(it->second.ToString());
+		}
+		if (it != --secret_map.end()) {
+			result.append(";");
+		}
+	}
+
+	return result;
+}
+
+// FIXME: use serialization scripts
+void KeyValueSecret::Serialize(Serializer &serializer) const {
+	BaseSecret::SerializeBaseSecret(serializer);
+
+	vector<Value> map_values;
+	for (auto it = secret_map.begin(); it != secret_map.end(); it++) {
+		child_list_t<Value> map_struct;
+		map_struct.push_back(make_pair("key", Value(it->first)));
+		map_struct.push_back(make_pair("value", Value(it->second)));
+		map_values.push_back(Value::STRUCT(map_struct));
+	}
+
+	auto map_type = LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR);
+	auto map = Value::MAP(ListType::GetChildType(map_type), map_values);
+	serializer.WriteProperty(201, "secret_map", map);
+
+	vector<Value> redact_key_values;
+	for (auto it = redact_keys.begin(); it != redact_keys.end(); it++) {
+		redact_key_values.push_back(*it);
+	}
+	auto list = Value::LIST(LogicalType::VARCHAR, redact_key_values);
+	serializer.WriteProperty(202, "redact_keys", list);
+}
+
+Value KeyValueSecret::TryGetValue(const string &key, bool error_on_missing) const {
+	auto lookup = secret_map.find(key);
+	if (lookup == secret_map.end()) {
+		if (error_on_missing) {
+			throw InternalException("Failed to fetch key '%s' from secret '%s' of type '%s'", key, name, type);
+		}
+		return Value();
+	}
+
+	return lookup->second;
+}
+
+bool CreateSecretFunctionSet::ProviderExists(const string &provider_name) {
+	return functions.find(provider_name) != functions.end();
+}
+
+void CreateSecretFunctionSet::AddFunction(CreateSecretFunction &function, OnCreateConflict on_conflict) {
+	if (ProviderExists(function.provider)) {
+		if (on_conflict == OnCreateConflict::ERROR_ON_CONFLICT) {
+			throw InternalException(
+			    "Attempted to override a Create Secret Function with OnCreateConflict::ERROR_ON_CONFLICT for: '%s'",
+			    function.provider);
+		} else if (on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT) {
+			functions[function.provider] = function;
+		} else if (on_conflict == OnCreateConflict::ALTER_ON_CONFLICT) {
+			throw NotImplementedException("ALTER_ON_CONFLICT not implemented for CreateSecretFunctionSet");
+		}
+	} else {
+		functions[function.provider] = function;
+	}
+}
+
+CreateSecretFunction &CreateSecretFunctionSet::GetFunction(const string &provider) {
+	const auto &lookup = functions.find(provider);
+
+	if (lookup == functions.end()) {
+		throw InternalException("Could not find Create Secret Function with provider %s");
+	}
+
+	return lookup->second;
+}
+
+} // namespace duckdb