duckdb 1.3.2 → 1.3.3-dev4.0

package/src/duckdb/src/main/extension_manager.cpp

@@ -0,0 +1,115 @@
+#include "duckdb/main/extension_manager.hpp"
+#include "duckdb/main/database.hpp"
+#include "duckdb/planner/extension_callback.hpp"
+#include "duckdb/main/extension_helper.hpp"
+#include "duckdb/logging/log_manager.hpp"
+
+namespace duckdb {
+
+ExtensionInfo::ExtensionInfo() : is_loaded(false) {
+}
+
+ExtensionActiveLoad::ExtensionActiveLoad(DatabaseInstance &db, ExtensionInfo &info, string extension_name_p)
+    : db(db), load_lock(info.lock), info(info), extension_name(std::move(extension_name_p)) {
+}
+
+void ExtensionActiveLoad::FinishLoad(ExtensionInstallInfo &install_info) {
+	info.is_loaded = true;
+	info.install_info = make_uniq<ExtensionInstallInfo>(install_info);
+
+	auto &callbacks = DBConfig::GetConfig(db).extension_callbacks;
+	for (auto &callback : callbacks) {
+		callback->OnExtensionLoaded(db, extension_name);
+	}
+	DUCKDB_LOG_INFO(db, extension_name);
+}
+
+void ExtensionActiveLoad::LoadFail(const ErrorData &error) {
+	auto &callbacks = DBConfig::GetConfig(db).extension_callbacks;
+	for (auto &callback : callbacks) {
+		callback->OnExtensionLoadFail(db, extension_name, error);
+	}
+	DUCKDB_LOG_INFO(db, "Failed to load extension '%s': %s", extension_name, error.Message());
+}
+
+ExtensionManager::ExtensionManager(DatabaseInstance &db) : db(db) {
+}
+
+ExtensionManager &ExtensionManager::Get(DatabaseInstance &db) {
+	return db.GetExtensionManager();
+}
+
+ExtensionManager &ExtensionManager::Get(ClientContext &context) {
+	return ExtensionManager::Get(DatabaseInstance::GetDatabase(context));
+}
+
+optional_ptr<ExtensionInfo> ExtensionManager::GetExtensionInfo(const string &name) {
+	auto extension_name = ExtensionHelper::GetExtensionName(name);
+
+	lock_guard<mutex> guard(lock);
+	auto entry = loaded_extensions_info.find(extension_name);
+	if (entry == loaded_extensions_info.end()) {
+		return nullptr;
+	}
+	return entry->second.get();
+}
+
+vector<string> ExtensionManager::GetExtensions() {
+	lock_guard<mutex> guard(lock);
+
+	vector<string> result;
+	for (auto &entry : loaded_extensions_info) {
+		result.push_back(entry.first);
+	}
+	return result;
+}
+
+bool ExtensionManager::ExtensionIsLoaded(const string &name) {
+	auto info = GetExtensionInfo(name);
+	if (!info) {
+		return false;
+	}
+	return info->is_loaded;
+}
+
+unique_ptr<ExtensionActiveLoad> ExtensionManager::BeginLoad(const string &name) {
+	auto extension_name = ExtensionHelper::GetExtensionName(name);
+
+	unique_lock<mutex> extension_list_lock(lock);
+
+	optional_ptr<ExtensionInfo> info;
+	auto entry = loaded_extensions_info.find(extension_name);
+	if (entry == loaded_extensions_info.end()) {
+		// we don't have an entry yet - create one
+		auto extension_info = make_uniq<ExtensionInfo>();
+		info = extension_info.get();
+		loaded_extensions_info.emplace(extension_name, std::move(extension_info));
+	} else {
+		// we already have an entry
+		if (entry->second->is_loaded) {
+			// and it is loaded! we are done
+			return nullptr;
+		}
+		// it is not loaded yet - try to load it
+		info = entry->second.get();
+	}
+	extension_list_lock.unlock();
+
+	// we have an extension and we want to try to load it - instantiate the load
+	// we instantiate the ExtensionActiveLoad which also grabs the lock for loading the specific extension
+	auto result = make_uniq<ExtensionActiveLoad>(db, *info, extension_name);
+
+	// we now have a lock for loading the extension
+	// HOWEVER - another thread might have finished loading in the meantime - double check to avoid a double load
+	if (info->is_loaded) {
+		return nullptr;
+	}
+	auto &callbacks = DBConfig::GetConfig(db).extension_callbacks;
+	for (auto &callback : callbacks) {
+		callback->OnBeginExtensionLoad(db, extension_name);
+	}
+	// extension is not loaded yet and we are in charge of loading it - return
+	return result;
+}
+
+} // namespace duckdb

package/src/duckdb/src/main/http/http_util.cpp

@@ -1,11 +1,14 @@
 #include "duckdb/common/http_util.hpp"
-#include "duckdb/main/database.hpp"
+
+#include "duckdb/common/exception/http_exception.hpp"
 #include "duckdb/common/operator/cast_operators.hpp"
 #include "duckdb/common/string_util.hpp"
-#include "duckdb/common/exception/http_exception.hpp"
+#include "duckdb/main/client_context.hpp"
 #include "duckdb/main/client_context_file_opener.hpp"
-#include "duckdb/main/database_file_opener.hpp"
 #include "duckdb/main/client_data.hpp"
+#include "duckdb/main/database.hpp"
+#include "duckdb/main/database_file_opener.hpp"
+
 #ifndef DISABLE_DUCKDB_REMOTE_INSTALL
 #ifndef DUCKDB_DISABLE_EXTENSION_LOAD
 #include "httplib.hpp"
@@ -224,7 +227,13 @@ unique_ptr<HTTPResponse> HTTPUtil::SendRequest(BaseRequest &request, unique_ptr<
 	}
 
 	std::function<unique_ptr<HTTPResponse>(void)> on_request([&]() {
-		auto response = client->Request(request);
+		unique_ptr<HTTPResponse> response;
+		try {
+			response = client->Request(request);
+		} catch (...) {
+			LogRequest(request, nullptr);
+			throw;
+		}
 		LogRequest(request, response ? response.get() : nullptr);
 		return response;
 	});
@@ -370,8 +379,12 @@ HTTPUtil::RunRequestWithRetry(const std::function<unique_ptr<HTTPResponse>(void)
 		// Note: request errors will always be retried
 		bool should_retry = !response || response->ShouldRetry();
 		if (!should_retry) {
+			auto response_code = static_cast<uint16_t>(response->status);
+			if (response_code >= 200 && response_code < 300) {
+				response->success = true;
+				return response;
+			}
 			switch (response->status) {
-			case HTTPStatusCode::OK_200:
 			case HTTPStatusCode::NotModified_304:
 				response->success = true;
 				break;

package/src/duckdb/src/main/profiling_info.cpp

@@ -52,7 +52,9 @@ profiler_settings_t ProfilingInfo::DefaultSettings() {
 	        MetricsType::OPERATOR_TIMING,
 	        MetricsType::RESULT_SET_SIZE,
 	        MetricsType::LATENCY,
-	        MetricsType::ROWS_RETURNED};
+	        MetricsType::ROWS_RETURNED,
+	        MetricsType::TOTAL_BYTES_READ,
+	        MetricsType::TOTAL_BYTES_WRITTEN};
 }
 
 profiler_settings_t ProfilingInfo::DefaultRootSettings() {
@@ -97,6 +99,8 @@ void ProfilingInfo::ResetMetrics() {
 		case MetricsType::OPERATOR_ROWS_SCANNED:
 		case MetricsType::SYSTEM_PEAK_BUFFER_MEMORY:
 		case MetricsType::SYSTEM_PEAK_TEMP_DIR_SIZE:
+		case MetricsType::TOTAL_BYTES_READ:
+		case MetricsType::TOTAL_BYTES_WRITTEN:
 			metrics[metric] = Value::CreateValue<uint64_t>(0);
 			break;
 		case MetricsType::EXTRA_INFO:
@@ -223,7 +227,9 @@ void ProfilingInfo::WriteMetricsToJSON(yyjson_mut_doc *doc, yyjson_mut_val *dest
 		case MetricsType::CUMULATIVE_ROWS_SCANNED:
 		case MetricsType::OPERATOR_ROWS_SCANNED:
 		case MetricsType::SYSTEM_PEAK_BUFFER_MEMORY:
-		case MetricsType::SYSTEM_PEAK_TEMP_DIR_SIZE: {
+		case MetricsType::SYSTEM_PEAK_TEMP_DIR_SIZE:
+		case MetricsType::TOTAL_BYTES_READ:
+		case MetricsType::TOTAL_BYTES_WRITTEN: {
 			yyjson_mut_obj_add_uint(doc, dest, key_ptr, metrics[metric].GetValue<uint64_t>());
 			break;
 		}

package/src/duckdb/src/main/query_profiler.cpp

@@ -38,10 +38,12 @@ bool QueryProfiler::IsDetailedEnabled() const {
 
 ProfilerPrintFormat QueryProfiler::GetPrintFormat(ExplainFormat format) const {
 	auto print_format = ClientConfig::GetConfig(context).profiler_print_format;
-	if (format == ExplainFormat::DEFAULT) {
-		return print_format;
-	}
 	switch (format) {
+	case ExplainFormat::DEFAULT:
+		if (print_format != ProfilerPrintFormat::NO_OUTPUT) {
+			return print_format;
+		}
+		DUCKDB_EXPLICIT_FALLTHROUGH;
 	case ExplainFormat::TEXT:
 		return ProfilerPrintFormat::QUERY_TREE;
 	case ExplainFormat::JSON:
@@ -87,7 +89,25 @@ QueryProfiler &QueryProfiler::Get(ClientContext &context) {
 	return *ClientData::Get(context).profiler;
 }
 
-void QueryProfiler::StartQuery(string query, bool is_explain_analyze_p, bool start_at_optimizer) {
+void QueryProfiler::Start(const string &query) {
+	Reset();
+	running = true;
+	query_metrics.query = query;
+	query_metrics.latency.Start();
+}
+
+void QueryProfiler::Reset() {
+	tree_map.clear();
+	root = nullptr;
+	phase_timings.clear();
+	phase_stack.clear();
+	running = false;
+	query_metrics.query = "";
+	query_metrics.total_bytes_read = 0;
+	query_metrics.total_bytes_written = 0;
+}
+
+void QueryProfiler::StartQuery(const string &query, bool is_explain_analyze_p, bool start_at_optimizer) {
 	lock_guard<std::mutex> guard(lock);
 	if (is_explain_analyze_p) {
 		StartExplainAnalyze();
@@ -104,17 +124,15 @@ void QueryProfiler::StartQuery(string query, bool is_explain_analyze_p, bool sta
 		D_ASSERT(PrintOptimizerOutput());
 		return;
 	}
-
-	running = true;
-	query_info.query_name = std::move(query);
-	tree_map.clear();
-	root = nullptr;
-	phase_timings.clear();
-	phase_stack.clear();
-	main_query.Start();
+	Start(query);
 }
 
-bool QueryProfiler::OperatorRequiresProfiling(PhysicalOperatorType op_type) {
+bool QueryProfiler::OperatorRequiresProfiling(const PhysicalOperatorType op_type) {
+	const auto &config = ClientConfig::GetConfig(context);
+	if (config.profiling_coverage == ProfilingCoverage::ALL) {
+		return true;
+	}
+
 	switch (op_type) {
 	case PhysicalOperatorType::ORDER_BY:
 	case PhysicalOperatorType::RESERVOIR_SAMPLE:
@@ -214,7 +232,7 @@ void QueryProfiler::EndQuery() {
 		return;
 	}
 
-	main_query.End();
+	query_metrics.latency.End();
 	if (root) {
 		auto &info = root->GetProfilingInfo();
 		if (info.Enabled(info.expanded_settings, MetricsType::OPERATOR_CARDINALITY)) {
@@ -222,7 +240,6 @@ void QueryProfiler::EndQuery() {
 		}
 	}
 	running = false;
-
 	bool emit_output = false;
 
 	// Print or output the query profiling after query termination.
@@ -232,14 +249,20 @@ void QueryProfiler::EndQuery() {
 			auto &info = root->GetProfilingInfo();
 			info = ProfilingInfo(ClientConfig::GetConfig(context).profiler_settings);
 			auto &child_info = root->children[0]->GetProfilingInfo();
-			info.metrics[MetricsType::QUERY_NAME] = query_info.query_name;
+			info.metrics[MetricsType::QUERY_NAME] = query_metrics.query;
 
 			auto &settings = info.expanded_settings;
-			for (const auto &global_info_entry : query_info.query_global_info.metrics) {
+			for (const auto &global_info_entry : query_metrics.query_global_info.metrics) {
 				info.metrics[global_info_entry.first] = global_info_entry.second;
 			}
 			if (info.Enabled(settings, MetricsType::LATENCY)) {
-				info.metrics[MetricsType::LATENCY] = main_query.Elapsed();
+				info.metrics[MetricsType::LATENCY] = query_metrics.latency.Elapsed();
+			}
+			if (info.Enabled(settings, MetricsType::TOTAL_BYTES_READ)) {
+				info.metrics[MetricsType::TOTAL_BYTES_READ] = Value::UBIGINT(query_metrics.total_bytes_read);
+			}
+			if (info.Enabled(settings, MetricsType::TOTAL_BYTES_WRITTEN)) {
+				info.metrics[MetricsType::TOTAL_BYTES_WRITTEN] = Value::UBIGINT(query_metrics.total_bytes_written);
 			}
 			if (info.Enabled(settings, MetricsType::ROWS_RETURNED)) {
 				info.metrics[MetricsType::ROWS_RETURNED] = child_info.metrics[MetricsType::OPERATOR_CARDINALITY];
@@ -287,6 +310,18 @@ void QueryProfiler::EndQuery() {
 	}
 }
 
+void QueryProfiler::AddBytesRead(const idx_t nr_bytes) {
+	if (IsEnabled()) {
+		query_metrics.total_bytes_read += nr_bytes;
+	}
+}
+
+void QueryProfiler::AddBytesWritten(const idx_t nr_bytes) {
+	if (IsEnabled()) {
+		query_metrics.total_bytes_written += nr_bytes;
+	}
+}
+
 string QueryProfiler::ToString(ExplainFormat explain_format) const {
 	return ToString(GetPrintFormat(explain_format));
 }
@@ -308,14 +343,14 @@ string QueryProfiler::ToString(ProfilerPrintFormat format) const {
 		lock_guard<std::mutex> guard(lock);
 		// checking the tree to ensure the query is really empty
 		// the query string is empty when a logical plan is deserialized
-		if (query_info.query_name.empty() && !root) {
+		if (query_metrics.query.empty() && !root) {
 			return "";
 		}
 		auto renderer = TreeRenderer::CreateRenderer(GetExplainFormat(format));
 		duckdb::stringstream str;
 		auto &info = root->GetProfilingInfo();
 		if (info.Enabled(info.expanded_settings, MetricsType::OPERATOR_TIMING)) {
-			info.metrics[MetricsType::OPERATOR_TIMING] = main_query.Elapsed();
+			info.metrics[MetricsType::OPERATOR_TIMING] = query_metrics.latency.Elapsed();
 		}
 		renderer->Render(*root, str);
 		return str.str();
@@ -525,12 +560,12 @@ void QueryProfiler::Flush(OperatorProfiler &profiler) {
 			info.extra_info = node.second.extra_info;
 		}
 		if (ProfilingInfo::Enabled(profiler.settings, MetricsType::SYSTEM_PEAK_BUFFER_MEMORY)) {
-			query_info.query_global_info.MetricMax(MetricsType::SYSTEM_PEAK_BUFFER_MEMORY,
-			                                       node.second.system_peak_buffer_manager_memory);
+			query_metrics.query_global_info.MetricMax(MetricsType::SYSTEM_PEAK_BUFFER_MEMORY,
+			                                          node.second.system_peak_buffer_manager_memory);
 		}
 		if (ProfilingInfo::Enabled(profiler.settings, MetricsType::SYSTEM_PEAK_TEMP_DIR_SIZE)) {
-			query_info.query_global_info.MetricMax(MetricsType::SYSTEM_PEAK_TEMP_DIR_SIZE,
-			                                       node.second.system_peak_temp_directory_size);
+			query_metrics.query_global_info.MetricMax(MetricsType::SYSTEM_PEAK_TEMP_DIR_SIZE,
+			                                          node.second.system_peak_temp_directory_size);
 		}
 	}
 	profiler.operator_infos.clear();
@@ -544,7 +579,7 @@ void QueryProfiler::SetInfo(const double &blocked_thread_time) {
 
 	auto &info = root->GetProfilingInfo();
 	if (info.Enabled(info.expanded_settings, MetricsType::BLOCKED_THREAD_TIME)) {
-		query_info.query_global_info.metrics[MetricsType::BLOCKED_THREAD_TIME] = blocked_thread_time;
+		query_metrics.query_global_info.metrics[MetricsType::BLOCKED_THREAD_TIME] = blocked_thread_time;
 	}
 }
 
@@ -657,11 +692,11 @@ void QueryProfiler::QueryTreeToStream(std::ostream &ss) const {
 	ss << "││    Query Profiling Information    ││\n";
 	ss << "│└───────────────────────────────────┘│\n";
 	ss << "└─────────────────────────────────────┘\n";
-	ss << StringUtil::Replace(query_info.query_name, "\n", " ") + "\n";
+	ss << StringUtil::Replace(query_metrics.query, "\n", " ") + "\n";
 
 	// checking the tree to ensure the query is really empty
 	// the query string is empty when a logical plan is deserialized
-	if (query_info.query_name.empty() && !root) {
+	if (query_metrics.query.empty() && !root) {
 		return;
 	}
 
@@ -672,7 +707,7 @@ void QueryProfiler::QueryTreeToStream(std::ostream &ss) const {
 	constexpr idx_t TOTAL_BOX_WIDTH = 50;
 	ss << "┌────────────────────────────────────────────────┐\n";
 	ss << "│┌──────────────────────────────────────────────┐│\n";
-	string total_time = "Total Time: " + RenderTiming(main_query.Elapsed());
+	string total_time = "Total Time: " + RenderTiming(query_metrics.latency.Elapsed());
 	ss << "││" + DrawPadded(total_time, TOTAL_BOX_WIDTH - 4) + "││\n";
 	ss << "│└──────────────────────────────────────────────┘│\n";
 	ss << "└────────────────────────────────────────────────┘\n";
@@ -768,7 +803,7 @@ string QueryProfiler::ToJSON() const {
 	auto result_obj = yyjson_mut_obj(doc);
 	yyjson_mut_doc_set_root(doc, result_obj);
 
-	if (query_info.query_name.empty() && !root) {
+	if (query_metrics.query.empty() && !root) {
 		yyjson_mut_obj_add_str(doc, result_obj, "result", "empty");
 		return StringifyAndFree(doc, result_obj);
 	}
@@ -892,7 +927,7 @@ void QueryProfiler::Initialize(const PhysicalOperator &root_op) {
 	root = CreateTree(root_op, config.profiler_settings, 0);
 	if (!query_requires_profiling) {
 		// query does not require profiling: disable profiling for this query
-		this->running = false;
+		running = false;
 		tree_map.clear();
 		root = nullptr;
 		phase_timings.clear();

package/src/duckdb/src/main/query_result.cpp

@@ -62,7 +62,7 @@ QueryResult::QueryResult(QueryResultType type, StatementType statement_type, Sta
 
 QueryResult::QueryResult(QueryResultType type, ErrorData error)
     : BaseQueryResult(type, std::move(error)),
-      client_properties("UTC", ArrowOffsetSize::REGULAR, false, false, false, V1_0, nullptr) {
+      client_properties("UTC", ArrowOffsetSize::REGULAR, false, false, false, ArrowFormatVersion::V1_0, nullptr) {
 }
 
 QueryResult::~QueryResult() {

package/src/duckdb/src/main/relation/query_relation.cpp

@@ -10,6 +10,7 @@
 #include "duckdb/parser/query_node/select_node.hpp"
 #include "duckdb/planner/query_node/bound_select_node.hpp"
 #include "duckdb/parser/common_table_expression_info.hpp"
+#include "duckdb/parser/query_node/cte_node.hpp"
 
 namespace duckdb {
 
@@ -62,6 +63,7 @@ BoundStatement QueryRelation::Bind(Binder &binder) {
 	if (first_bind) {
 		auto &query_node = *select_stmt->node;
 		auto &cte_map = query_node.cte_map;
+		vector<unique_ptr<CTENode>> materialized_ctes;
 		for (auto &kv : replacements) {
 			auto &name = kv.first;
 			auto &tableref = kv.second;
@@ -82,7 +84,28 @@ BoundStatement QueryRelation::Bind(Binder &binder) {
 			cte_info->query = std::move(select);
 
 			cte_map.map[name] = std::move(cte_info);
+
+			// We can not rely on CTE inlining anymore, so we need to add a materialized CTE node
+			// to the query node to ensure that the CTE exists
+			auto &cte_entry = cte_map.map[name];
+			auto mat_cte = make_uniq<CTENode>();
+			mat_cte->ctename = name;
+			mat_cte->query = cte_entry->query->node->Copy();
+			mat_cte->aliases = cte_entry->aliases;
+			mat_cte->materialized = cte_entry->materialized;
+			materialized_ctes.push_back(std::move(mat_cte));
+		}
+
+		auto root = std::move(select_stmt->node);
+		while (!materialized_ctes.empty()) {
+			unique_ptr<CTENode> node_result;
+			node_result = std::move(materialized_ctes.back());
+			node_result->cte_map = root->cte_map.Copy();
+			node_result->child = std::move(root);
+			root = std::move(node_result);
+			materialized_ctes.pop_back();
 		}
+		select_stmt->node = std::move(root);
 	}
 	replacements.clear();
 	binder.SetBindingMode(saved_binding_mode);

package/src/duckdb/src/main/secret/secret_manager.cpp

@@ -75,14 +75,15 @@ void SecretManager::LoadSecretStorage(unique_ptr<SecretStorage> storage) {
 
 void SecretManager::LoadSecretStorageInternal(unique_ptr<SecretStorage> storage) {
 	if (secret_storages.find(storage->GetName()) != secret_storages.end()) {
-		throw InternalException("Secret Storage with name '%s' already registered!", storage->GetName());
+		throw InvalidConfigurationException("Secret Storage with name '%s' already registered!", storage->GetName());
 	}
 
 	// Check for tie-break offset collisions to ensure we can always tie-break cleanly
 	for (const auto &storage_ptr : secret_storages) {
 		if (storage_ptr.second->tie_break_offset == storage->tie_break_offset) {
-			throw InternalException("Failed to load secret storage '%s', tie break score collides with '%s'",
-			                        storage->GetName(), storage_ptr.second->GetName());
+			throw InvalidConfigurationException(
+			    "Failed to load secret storage '%s', tie break score collides with '%s'", storage->GetName(),
+			    storage_ptr.second->GetName());
 		}
 	}
 
@@ -97,8 +98,8 @@ unique_ptr<BaseSecret> SecretManager::DeserializeSecret(Deserializer &deserializ
 	vector<string> scope;
 	deserializer.ReadList(103, "scope",
 	                      [&](Deserializer::List &list, idx_t i) { scope.push_back(list.ReadElement<string>()); });
-	auto serialization_type =
-	    deserializer.ReadPropertyWithExplicitDefault(104, "serialization_type", SecretSerializationType::CUSTOM);
+	auto serialization_type = deserializer.ReadPropertyWithExplicitDefault(104, "serialization_type",
+	                                                                       SecretSerializationType::KEY_VALUE_SECRET);
 
 	switch (serialization_type) {
 	// This allows us to skip looking up the secret type for deserialization altogether
@@ -118,7 +119,7 @@ unique_ptr<BaseSecret> SecretManager::DeserializeSecret(Deserializer &deserializ
 	}
 
 	if (!deserialized_type.deserializer) {
-		throw InternalException(
+		throw InvalidConfigurationException(
 		    "Attempted to deserialize secret type '%s' which does not have a deserialization method", type);
 	}
 
@@ -351,7 +352,7 @@ unique_ptr<SecretEntry> SecretManager::GetSecretByName(CatalogTransaction transa
 		auto lookup = storage_ref.get().GetSecretByName(name, &transaction);
 		if (lookup) {
 			if (found) {
-				throw InternalException(
+				throw InvalidConfigurationException(
 				    "Ambiguity detected for secret name '%s', secret occurs in multiple storage backends.", name);
 			}