duckdb 0.10.2-dev0.0 → 0.10.2-dev3.0

package/src/duckdb/src/main/extension/extension_load.cpp

@@ -38,7 +38,7 @@ static void ComputeSHA256String(const std::string &to_hash, std::string *res) {
 
 static void ComputeSHA256FileSegment(FileHandle *handle, const idx_t start, const idx_t end, std::string *res) {
 	idx_t iter = start;
-	const idx_t segment_size = 1024 * 8;
+	const idx_t segment_size = 1024ULL * 8ULL;
 
 	duckdb_mbedtls::MbedTlsWrapper::SHA256State state;
 
@@ -57,6 +57,39 @@ static void ComputeSHA256FileSegment(FileHandle *handle, const idx_t start, cons
 }
 #endif
 
+static string FilterZeroAtEnd(string s) {
+	while (!s.empty() && s.back() == '\0') {
+		s.pop_back();
+	}
+	return s;
+}
+
+static string PrettyPrintString(const string &s) {
+	string res = "";
+	for (auto c : s) {
+		if (StringUtil::CharacterIsAlpha(c) || StringUtil::CharacterIsDigit(c) || c == '_' || c == '-' || c == ' ' ||
+		    c == '.') {
+			res += c;
+		} else {
+			uint8_t value = c;
+			res += "\\x";
+			uint8_t first = value / 16;
+			if (first < 10) {
+				res.push_back((char)('0' + first));
+			} else {
+				res.push_back((char)('a' + first - 10));
+			}
+			uint8_t second = value % 16;
+			if (second < 10) {
+				res.push_back((char)('0' + second));
+			} else {
+				res.push_back((char)('a' + second - 10));
+			}
+		}
+	}
+	return res;
+}
+
 bool ExtensionHelper::TryInitialLoad(DBConfig &config, FileSystem &fs, const string &extension,
                                      ExtensionInitResult &result, string &error) {
 #ifdef DUCKDB_DISABLE_EXTENSION_LOAD
@@ -119,15 +152,66 @@ bool ExtensionHelper::TryInitialLoad(DBConfig &config, FileSystem &fs, const str
 		error = StringUtil::Format("Extension \"%s\" not found.\n%s", filename, message);
 		return false;
 	}
-	if (!config.options.allow_unsigned_extensions) {
-		auto handle = fs.OpenFile(filename, FileFlags::FILE_FLAGS_READ);
 
-		// signature is the last 256 bytes of the file
+	string metadata_segment;
+	metadata_segment.resize(512);
+
+	const std::string engine_version = std::string(GetVersionDirectoryName());
+	const std::string engine_platform = std::string(DuckDB::Platform());
+
+	auto handle = fs.OpenFile(filename, FileFlags::FILE_FLAGS_READ);
+
+	idx_t file_size = handle->GetFileSize();
+
+	if (file_size < 1024) {
+		throw InvalidInputException(
+		    "Extension \"%s\" do not have metadata compatible with DuckDB loading it "
+		    "(version %s, platform %s). File size in particular is lower than minimum threshold of 1024",
+		    filename, engine_version, engine_platform);
+	}
+
+	auto metadata_offset = file_size - metadata_segment.size();
+
+	handle->Read((void *)metadata_segment.data(), metadata_segment.size(), metadata_offset);
+
+	std::vector<std::string> metadata_field;
+	for (idx_t i = 0; i < 8; i++) {
+		metadata_field.emplace_back(metadata_segment, i * 32, 32);
+	}
 
-		string signature;
-		signature.resize(256);
+	std::reverse(metadata_field.begin(), metadata_field.end());
+
+	std::string extension_duckdb_platform = FilterZeroAtEnd(metadata_field[1]);
+	std::string extension_duckdb_version = FilterZeroAtEnd(metadata_field[2]);
+	std::string extension_version = FilterZeroAtEnd(metadata_field[3]);
+
+	string metadata_mismatch_error = "";
+	{
+		char a[32] = {0};
+		a[0] = '4';
+		if (strncmp(a, metadata_field[0].data(), 32) != 0) {
+			// metadata do not looks right, add this to the error message
+			metadata_mismatch_error =
+			    "\n" + StringUtil::Format("Extension \"%s\" do not have metadata compatible with DuckDB "
+			                              "loading it (version %s, platform %s)",
+			                              filename, engine_version, engine_platform);
+		} else if (engine_version != extension_duckdb_version || engine_platform != extension_duckdb_platform) {
+			metadata_mismatch_error = "\n" + StringUtil::Format("Extension \"%s\" (version %s, platfrom %s) does not "
+			                                                    "match DuckDB loading it (version %s, platform %s)",
+			                                                    filename, PrettyPrintString(extension_duckdb_version),
+			                                                    PrettyPrintString(extension_duckdb_platform),
+			                                                    engine_version, engine_platform);
+
+		} else {
+			// All looks good
+		}
+	}
+
+	if (!config.options.allow_unsigned_extensions) {
+		// signature is the last 256 bytes of the file
+		string signature(metadata_segment, metadata_segment.size() - 256);
 
-		auto signature_offset = handle->GetFileSize() - signature.size();
+		auto signature_offset = metadata_offset + metadata_segment.size() - signature.size();
 
 		const idx_t maxLenChunks = 1024ULL * 1024ULL;
 		const idx_t numChunks = (signature_offset + maxLenChunks - 1) / maxLenChunks;
@@ -176,9 +260,26 @@ bool ExtensionHelper::TryInitialLoad(DBConfig &config, FileSystem &fs, const str
 			}
 		}
 		if (!any_valid) {
-			throw IOException(config.error_manager->FormatException(ErrorType::UNSIGNED_EXTENSION, filename));
+			throw IOException(config.error_manager->FormatException(ErrorType::UNSIGNED_EXTENSION, filename) +
+			                  metadata_mismatch_error);
+		}
+
+		if (!metadata_mismatch_error.empty()) {
+			// Signed extensions perform the full check
+			throw InvalidInputException(metadata_mismatch_error.substr(1));
+		}
+	} else if (!config.options.allow_extensions_metadata_mismatch) {
+		if (!metadata_mismatch_error.empty()) {
+			// Unsigned extensions AND configuration allowing metadata_mismatch_error, loading allowed, mainly for
+			// debugging purposes
+			throw InvalidInputException(metadata_mismatch_error.substr(1));
 		}
 	}
+
+	auto number_metadata_fields = 3;
+	D_ASSERT(number_metadata_fields == 3); // Currently hardcoded value
+	metadata_field.resize(number_metadata_fields + 1);
+
 	auto filebase = fs.ExtractBaseName(filename);
 
 #ifdef WASM_LOADABLE_EXTENSIONS
@@ -210,35 +311,8 @@ bool ExtensionHelper::TryInitialLoad(DBConfig &config, FileSystem &fs, const str
 
 	auto lowercase_extension_name = StringUtil::Lower(filebase);
 
-	ext_version_fun_t version_fun;
-	auto version_fun_name = lowercase_extension_name + "_version";
-
-	version_fun = LoadFunctionFromDLL<ext_version_fun_t>(lib_hdl, version_fun_name, filename);
-
-	std::string engine_version = std::string(DuckDB::LibraryVersion());
-
-	auto version_fun_result = (*version_fun)();
-	if (version_fun_result == nullptr) {
-		throw InvalidInputException("Extension \"%s\" returned a nullptr", filename);
-	}
-	std::string extension_version = std::string(version_fun_result);
-
-	// Trim v's if necessary
-	std::string extension_version_trimmed = extension_version;
-	std::string engine_version_trimmed = engine_version;
-	if (extension_version.length() > 0 && extension_version[0] == 'v') {
-		extension_version_trimmed = extension_version.substr(1);
-	}
-	if (engine_version.length() > 0 && engine_version[0] == 'v') {
-		engine_version_trimmed = engine_version.substr(1);
-	}
-
-	if (extension_version_trimmed != engine_version_trimmed) {
-		throw InvalidInputException("Extension \"%s\" version (%s) does not match DuckDB version (%s)", filename,
-		                            extension_version, engine_version);
-	}
-
 	result.filebase = lowercase_extension_name;
+	result.extension_version = extension_version;
 	result.filename = filename;
 	result.lib_hdl = lib_hdl;
 	return true;
@@ -304,7 +378,7 @@ void ExtensionHelper::LoadExternalExtension(DatabaseInstance &db, FileSystem &fs
 		                            init_fun_name, res.filename, error.RawMessage());
 	}
 
-	db.SetExtensionLoaded(extension);
+	db.SetExtensionLoaded(extension, res.extension_version);
 #endif
 }
 

package/src/duckdb/src/main/query_profiler.cpp

@@ -284,10 +284,6 @@ void OperatorProfiler::Flush(const PhysicalOperator &phys_op, ExpressionExecutor
 		return;
 	}
 	auto &operator_timing = timings.find(phys_op)->second;
-	if (int(operator_timing.executors_info.size()) <= id) {
-		operator_timing.executors_info.resize(id + 1);
-	}
-	operator_timing.executors_info[id] = make_uniq<ExpressionExecutorInfo>(expression_executor, name, id);
 	operator_timing.name = phys_op.GetName();
 }
 
@@ -307,16 +303,6 @@ void QueryProfiler::Flush(OperatorProfiler &profiler) {
 		if (!IsDetailedEnabled()) {
 			continue;
 		}
-		for (auto &info : node.second.executors_info) {
-			if (!info) {
-				continue;
-			}
-			auto info_id = info->id;
-			if (int32_t(tree_node.info.executors_info.size()) <= info_id) {
-				tree_node.info.executors_info.resize(info_id + 1);
-			}
-			tree_node.info.executors_info[info_id] = std::move(info);
-		}
 	}
 	profiler.timings.clear();
 }
@@ -484,70 +470,12 @@ static string JSONSanitize(const string &text) {
 	return result;
 }
 
-// Print a row
-static void PrintRow(std::ostream &ss, const string &annotation, int id, const string &name, double time,
-                     int sample_counter, int tuple_counter, const string &extra_info, int depth) {
-	ss << string(depth * 3, ' ') << " {\n";
-	ss << string(depth * 3, ' ') << "   \"annotation\": \"" + JSONSanitize(annotation) + "\",\n";
-	ss << string(depth * 3, ' ') << "   \"id\": " + to_string(id) + ",\n";
-	ss << string(depth * 3, ' ') << "   \"name\": \"" + JSONSanitize(name) + "\",\n";
-#if defined(RDTSC)
-	ss << string(depth * 3, ' ') << "   \"timing\": \"NULL\" ,\n";
-	ss << string(depth * 3, ' ') << "   \"cycles_per_tuple\": " + StringUtil::Format("%.4f", time) + ",\n";
-#else
-	ss << string(depth * 3, ' ') << "   \"timing\":" + to_string(time) + ",\n";
-	ss << string(depth * 3, ' ') << "   \"cycles_per_tuple\": \"NULL\" ,\n";
-#endif
-	ss << string(depth * 3, ' ') << "   \"sample_size\": " << to_string(sample_counter) + ",\n";
-	ss << string(depth * 3, ' ') << "   \"input_size\": " << to_string(tuple_counter) + ",\n";
-	ss << string(depth * 3, ' ') << "   \"extra_info\": \"" << JSONSanitize(extra_info) + "\"\n";
-	ss << string(depth * 3, ' ') << " },\n";
-}
-
-static void ExtractFunctions(std::ostream &ss, ExpressionInfo &info, int &fun_id, int depth) {
-	if (info.hasfunction) {
-		double time = info.sample_tuples_count == 0 ? 0 : int(info.function_time) / double(info.sample_tuples_count);
-		PrintRow(ss, "Function", fun_id++, info.function_name, time, NumericCast<int>(info.sample_tuples_count),
-		         NumericCast<int>(info.tuples_count), "", NumericCast<int>(depth));
-	}
-	if (info.children.empty()) {
-		return;
-	}
-	// extract the children of this node
-	for (auto &child : info.children) {
-		ExtractFunctions(ss, *child, fun_id, depth);
-	}
-}
-
-static void ToJSONRecursive(QueryProfiler::TreeNode &node, std::ostream &ss, int depth = 1) {
+static void ToJSONRecursive(QueryProfiler::TreeNode &node, std::ostream &ss, idx_t depth = 1) {
 	ss << string(depth * 3, ' ') << " {\n";
 	ss << string(depth * 3, ' ') << "   \"name\": \"" + JSONSanitize(node.name) + "\",\n";
 	ss << string(depth * 3, ' ') << "   \"timing\":" + to_string(node.info.time) + ",\n";
 	ss << string(depth * 3, ' ') << "   \"cardinality\":" + to_string(node.info.elements) + ",\n";
 	ss << string(depth * 3, ' ') << "   \"extra_info\": \"" + JSONSanitize(node.extra_info) + "\",\n";
-	ss << string(depth * 3, ' ') << "   \"timings\": [";
-	int32_t function_counter = 1;
-	int32_t expression_counter = 1;
-	ss << "\n ";
-	for (auto &expr_executor : node.info.executors_info) {
-		// For each Expression tree
-		if (!expr_executor) {
-			continue;
-		}
-		for (auto &expr_timer : expr_executor->roots) {
-			double time = expr_timer->sample_tuples_count == 0
-			                  ? 0
-			                  : double(expr_timer->time) / double(expr_timer->sample_tuples_count);
-			PrintRow(ss, "ExpressionRoot", expression_counter++, expr_timer->name, time,
-			         NumericCast<int>(expr_timer->sample_tuples_count), NumericCast<int>(expr_timer->tuples_count),
-			         expr_timer->extra_info, depth + 1);
-			// Extract all functions inside the tree
-			ExtractFunctions(ss, *expr_timer->root, function_counter, depth + 1);
-		}
-	}
-	ss.seekp(-2, ss.cur);
-	ss << "\n";
-	ss << string(depth * 3, ' ') << "   ],\n";
 	ss << string(depth * 3, ' ') << "   \"children\": [\n";
 	if (node.children.empty()) {
 		ss << string(depth * 3, ' ') << "   ]\n";
@@ -664,49 +592,4 @@ vector<QueryProfiler::PhaseTimingItem> QueryProfiler::GetOrderedPhaseTimings() c
 void QueryProfiler::Propagate(QueryProfiler &qp) {
 }
 
-void ExpressionInfo::ExtractExpressionsRecursive(unique_ptr<ExpressionState> &state) {
-	if (state->child_states.empty()) {
-		return;
-	}
-	// extract the children of this node
-	for (auto &child : state->child_states) {
-		auto expr_info = make_uniq<ExpressionInfo>();
-		if (child->expr.expression_class == ExpressionClass::BOUND_FUNCTION) {
-			expr_info->hasfunction = true;
-			expr_info->function_name = child->expr.Cast<BoundFunctionExpression>().function.ToString();
-			expr_info->function_time = child->profiler.time;
-			expr_info->sample_tuples_count = child->profiler.sample_tuples_count;
-			expr_info->tuples_count = child->profiler.tuples_count;
-		}
-		expr_info->ExtractExpressionsRecursive(child);
-		children.push_back(std::move(expr_info));
-	}
-	return;
-}
-
-ExpressionExecutorInfo::ExpressionExecutorInfo(ExpressionExecutor &executor, const string &name, int id) : id(id) {
-	// Extract Expression Root Information from ExpressionExecutorStats
-	for (auto &state : executor.GetStates()) {
-		roots.push_back(make_uniq<ExpressionRootInfo>(*state, name));
-	}
-}
-
-ExpressionRootInfo::ExpressionRootInfo(ExpressionExecutorState &state, string name)
-    : current_count(state.profiler.current_count), sample_count(state.profiler.sample_count),
-      sample_tuples_count(state.profiler.sample_tuples_count), tuples_count(state.profiler.tuples_count),
-      name("expression"), time(state.profiler.time) {
-	// Use the name of expression-tree as extra-info
-	extra_info = std::move(name);
-	auto expression_info_p = make_uniq<ExpressionInfo>();
-	// Maybe root has a function
-	if (state.root_state->expr.expression_class == ExpressionClass::BOUND_FUNCTION) {
-		expression_info_p->hasfunction = true;
-		expression_info_p->function_name = (state.root_state->expr.Cast<BoundFunctionExpression>()).function.name;
-		expression_info_p->function_time = state.root_state->profiler.time;
-		expression_info_p->sample_tuples_count = state.root_state->profiler.sample_tuples_count;
-		expression_info_p->tuples_count = state.root_state->profiler.tuples_count;
-	}
-	expression_info_p->ExtractExpressionsRecursive(state.root_state);
-	root = std::move(expression_info_p);
-}
 } // namespace duckdb

package/src/duckdb/src/main/secret/secret_manager.cpp

@@ -285,7 +285,7 @@ BoundStatement SecretManager::BindCreateSecret(CatalogTransaction transaction, C
 	BoundStatement result;
 	result.names = {"Success"};
 	result.types = {LogicalType::BOOLEAN};
-	result.plan = make_uniq<LogicalCreateSecret>(*function, std::move(bound_info));
+	result.plan = make_uniq<LogicalCreateSecret>(std::move(bound_info));
 	return result;
 }
 
@@ -607,7 +607,6 @@ unique_ptr<CatalogEntry> DefaultSecretGenerator::CreateDefaultEntry(ClientContex
 			auto deserialized_secret = secret_manager.DeserializeSecret(deserializer);
 			deserializer.End();
 
-			auto name = deserialized_secret->GetName();
 			auto entry = make_uniq<SecretCatalogEntry>(std::move(deserialized_secret), catalog);
 			entry->secret->storage_mode = SecretManager::LOCAL_FILE_STORAGE_NAME;
 			entry->secret->persist_type = SecretPersistType::PERSISTENT;

package/src/duckdb/src/main/secret/secret_storage.cpp

@@ -66,7 +66,7 @@ unique_ptr<SecretEntry> CatalogSetSecretStorage::StoreSecret(unique_ptr<const Ba
 	secret_entry->temporary = !persistent;
 	secret_entry->secret->storage_mode = storage_name;
 	secret_entry->secret->persist_type = persistent ? SecretPersistType::PERSISTENT : SecretPersistType::TEMPORARY;
-	DependencyList l;
+	LogicalDependencyList l;
 	secrets->CreateEntry(GetTransactionOrDefault(transaction), secret_name, std::move(secret_entry), l);
 
 	auto secret_catalog_entry =