duckdb 0.8.2-dev87.0 → 0.9.1-dev0.0

package/src/duckdb/src/main/config.cpp

@@ -59,21 +59,25 @@ static ConfigurationOption internal_options[] = {DUCKDB_GLOBAL(AccessModeSetting
                                                  DUCKDB_LOCAL(DebugForceExternal),
                                                  DUCKDB_LOCAL(DebugForceNoCrossProduct),
                                                  DUCKDB_LOCAL(DebugAsOfIEJoin),
+                                                 DUCKDB_LOCAL(PreferRangeJoins),
                                                  DUCKDB_GLOBAL(DebugWindowMode),
                                                  DUCKDB_GLOBAL_LOCAL(DefaultCollationSetting),
                                                  DUCKDB_GLOBAL(DefaultOrderSetting),
                                                  DUCKDB_GLOBAL(DefaultNullOrderSetting),
+                                                 DUCKDB_GLOBAL(DisabledFileSystemsSetting),
                                                  DUCKDB_GLOBAL(DisabledOptimizersSetting),
                                                  DUCKDB_GLOBAL(EnableExternalAccessSetting),
                                                  DUCKDB_GLOBAL(EnableFSSTVectors),
                                                  DUCKDB_GLOBAL(AllowUnsignedExtensionsSetting),
                                                  DUCKDB_LOCAL(CustomExtensionRepository),
+                                                 DUCKDB_LOCAL(AutoloadExtensionRepository),
+                                                 DUCKDB_GLOBAL(AutoinstallKnownExtensions),
+                                                 DUCKDB_GLOBAL(AutoloadKnownExtensions),
                                                  DUCKDB_GLOBAL(EnableObjectCacheSetting),
                                                  DUCKDB_GLOBAL(EnableHTTPMetadataCacheSetting),
                                                  DUCKDB_LOCAL(EnableProfilingSetting),
                                                  DUCKDB_LOCAL(EnableProgressBarSetting),
                                                  DUCKDB_LOCAL(EnableProgressBarPrintSetting),
-                                                 DUCKDB_GLOBAL(ExperimentalParallelCSVSetting),
                                                  DUCKDB_LOCAL(ExplainOutputSetting),
                                                  DUCKDB_GLOBAL(ExtensionDirectorySetting),
                                                  DUCKDB_GLOBAL(ExternalThreadsSetting),
@@ -92,6 +96,7 @@ static ConfigurationOption internal_options[] = {DUCKDB_GLOBAL(AccessModeSetting
                                                  DUCKDB_LOCAL(OrderedAggregateThreshold),
                                                  DUCKDB_GLOBAL(PasswordSetting),
                                                  DUCKDB_LOCAL(PerfectHashThresholdSetting),
+                                                 DUCKDB_LOCAL(PivotFilterThreshold),
                                                  DUCKDB_LOCAL(PivotLimitSetting),
                                                  DUCKDB_LOCAL(PreserveIdentifierCase),
                                                  DUCKDB_GLOBAL(PreserveInsertionOrder),
@@ -109,6 +114,7 @@ static ConfigurationOption internal_options[] = {DUCKDB_GLOBAL(AccessModeSetting
                                                  DUCKDB_GLOBAL_ALIAS("user", UsernameSetting),
                                                  DUCKDB_GLOBAL_ALIAS("wal_autocheckpoint", CheckpointThresholdSetting),
                                                  DUCKDB_GLOBAL_ALIAS("worker_threads", ThreadsSetting),
+                                                 DUCKDB_GLOBAL(FlushAllocatorSetting),
                                                  FINAL_SETTING};
 
 vector<ConfigurationOption> DBConfig::GetOptions() {
@@ -171,7 +177,7 @@ void DBConfig::SetOptionByName(const string &name, const Value &value) {
 void DBConfig::SetOption(DatabaseInstance *db, const ConfigurationOption &option, const Value &value) {
 	lock_guard<mutex> l(config_lock);
 	if (!option.set_global) {
-		throw InternalException("Could not set option \"%s\" as a global option", option.name);
+		throw InvalidInputException("Could not set option \"%s\" as a global option", option.name);
 	}
 	D_ASSERT(option.reset_global);
 	Value input = value.DefaultCastAs(option.parameter_type);
@@ -279,7 +285,7 @@ idx_t CGroupBandwidthQuota(idx_t physical_cores, FileSystem &fs) {
 	}
 }
 
-idx_t GetSystemMaxThreadsInternal(FileSystem &fs) {
+idx_t DBConfig::GetSystemMaxThreads(FileSystem &fs) {
 #ifndef DUCKDB_NO_THREADS
 	idx_t physical_cores = std::thread::hardware_concurrency();
 #ifdef __linux__
@@ -295,7 +301,7 @@ idx_t GetSystemMaxThreadsInternal(FileSystem &fs) {
 
 void DBConfig::SetDefaultMaxThreads() {
 #ifndef DUCKDB_NO_THREADS
-	options.maximum_threads = GetSystemMaxThreadsInternal(*file_system);
+	options.maximum_threads = GetSystemMaxThreads(*file_system);
 #else
 	options.maximum_threads = 1;
 #endif

package/src/duckdb/src/main/connection.cpp

@@ -1,7 +1,7 @@
 #include "duckdb/main/connection.hpp"
 
 #include "duckdb/common/types/column/column_data_collection.hpp"
-#include "duckdb/execution/operator/persistent/parallel_csv_reader.hpp"
+#include "duckdb/execution/operator/scan/csv/parallel_csv_reader.hpp"
 #include "duckdb/function/table/read_csv.hpp"
 #include "duckdb/main/appender.hpp"
 #include "duckdb/main/client_context.hpp"
@@ -219,14 +219,12 @@ shared_ptr<Relation> Connection::Values(const string &values, const vector<strin
 }
 
 shared_ptr<Relation> Connection::ReadCSV(const string &csv_file) {
-	BufferedCSVReaderOptions options;
-	return ReadCSV(csv_file, options);
+	named_parameter_map_t options;
+	return ReadCSV(csv_file, std::move(options));
 }
 
-shared_ptr<Relation> Connection::ReadCSV(const string &csv_file, BufferedCSVReaderOptions &options) {
-	options.file_path = csv_file;
-	options.auto_detect = true;
-	return make_shared<ReadCSVRelation>(context, csv_file, options);
+shared_ptr<Relation> Connection::ReadCSV(const string &csv_file, named_parameter_map_t &&options) {
+	return make_shared<ReadCSVRelation>(context, csv_file, std::move(options));
 }
 
 shared_ptr<Relation> Connection::ReadCSV(const string &csv_file, const vector<string> &columns) {

package/src/duckdb/src/main/database.cpp

@@ -19,6 +19,7 @@
 #include "duckdb/storage/storage_extension.hpp"
 #include "duckdb/storage/storage_manager.hpp"
 #include "duckdb/transaction/transaction_manager.hpp"
+#include "duckdb/planner/extension_callback.hpp"
 
 #ifndef DUCKDB_NO_THREADS
 #include "duckdb/common/thread.hpp"
@@ -158,7 +159,7 @@ duckdb::unique_ptr<AttachedDatabase> DatabaseInstance::CreateAttachedDatabase(At
 
 void DatabaseInstance::CreateMainDatabase() {
 	AttachInfo info;
-	info.name = AttachedDatabase::ExtractDatabaseName(config.options.database_path);
+	info.name = AttachedDatabase::ExtractDatabaseName(config.options.database_path, GetFileSystem());
 	info.path = config.options.database_path;
 
 	auto attached_database = CreateAttachedDatabase(info, config.options.database_type, config.options.access_mode);
@@ -178,7 +179,7 @@ void DatabaseInstance::CreateMainDatabase() {
 void ThrowExtensionSetUnrecognizedOptions(const unordered_map<string, Value> &unrecognized_options) {
 	auto unrecognized_options_iter = unrecognized_options.begin();
 	string unrecognized_option_keys = unrecognized_options_iter->first;
-	for (; unrecognized_options_iter == unrecognized_options.end(); ++unrecognized_options_iter) {
+	while (++unrecognized_options_iter != unrecognized_options.end()) {
 		unrecognized_option_keys = "," + unrecognized_options_iter->first;
 	}
 	throw InvalidInputException("Unrecognized configuration property \"%s\"", unrecognized_option_keys);
@@ -234,7 +235,7 @@ void DatabaseInstance::Initialize(const char *database_path, DBConfig *user_conf
 		if (!config.file_system) {
 			throw InternalException("No file system!?");
 		}
-		ExtensionHelper::LoadExternalExtension(*this, *config.file_system, config.options.database_type);
+		ExtensionHelper::LoadExternalExtension(*this, *config.file_system, config.options.database_type, nullptr);
 	}
 
 	if (!config.options.unrecognized_options.empty()) {
@@ -381,6 +382,11 @@ bool DuckDB::ExtensionIsLoaded(const std::string &name) {
 void DatabaseInstance::SetExtensionLoaded(const std::string &name) {
 	auto extension_name = ExtensionHelper::GetExtensionName(name);
 	loaded_extensions.insert(extension_name);
+
+	auto &callbacks = DBConfig::GetConfig(*this).extension_callbacks;
+	for (auto &callback : callbacks) {
+		callback->OnExtensionLoaded(*this, name);
+	}
 }
 
 bool DatabaseInstance::TryGetCurrentSetting(const std::string &key, Value &result) {
@@ -397,15 +403,6 @@ bool DatabaseInstance::TryGetCurrentSetting(const std::string &key, Value &resul
 	return true;
 }
 
-string ClientConfig::ExtractTimezone() const {
-	auto entry = set_variables.find("TimeZone");
-	if (entry == set_variables.end()) {
-		return "UTC";
-	} else {
-		return entry->second.GetValue<std::string>();
-	}
-}
-
 ValidChecker &DatabaseInstance::GetValidChecker() {
 	return db_validity;
 }

package/src/duckdb/src/main/db_instance_cache.cpp

@@ -3,7 +3,7 @@
 
 namespace duckdb {
 
-string GetDBAbsolutePath(const string &database_p) {
+string GetDBAbsolutePath(const string &database_p, FileSystem &fs) {
 	auto database = FileSystem::ExpandPath(database_p, nullptr);
 	if (database.empty()) {
 		return ":memory:";
@@ -16,15 +16,17 @@ string GetDBAbsolutePath(const string &database_p) {
 		// this database path is handled by a replacement open and is not a file path
 		return database;
 	}
-	if (FileSystem::IsPathAbsolute(database)) {
-		return FileSystem::NormalizeAbsolutePath(database);
+	if (fs.IsPathAbsolute(database)) {
+		return fs.NormalizeAbsolutePath(database);
 	}
-	return FileSystem::NormalizeAbsolutePath(FileSystem::JoinPath(FileSystem::GetWorkingDirectory(), database));
+	return fs.NormalizeAbsolutePath(fs.JoinPath(FileSystem::GetWorkingDirectory(), database));
 }
 
 shared_ptr<DuckDB> DBInstanceCache::GetInstanceInternal(const string &database, const DBConfig &config) {
 	shared_ptr<DuckDB> db_instance;
-	auto abs_database_path = GetDBAbsolutePath(database);
+
+	auto local_fs = FileSystem::CreateLocal();
+	auto abs_database_path = GetDBAbsolutePath(database, *local_fs);
 	if (db_instances.find(abs_database_path) != db_instances.end()) {
 		db_instance = db_instances[abs_database_path].lock();
 		if (db_instance) {
@@ -48,7 +50,13 @@ shared_ptr<DuckDB> DBInstanceCache::GetInstance(const string &database, const DB
 
 shared_ptr<DuckDB> DBInstanceCache::CreateInstanceInternal(const string &database, DBConfig &config,
                                                            bool cache_instance) {
-	auto abs_database_path = GetDBAbsolutePath(database);
+	string abs_database_path;
+	if (config.file_system) {
+		abs_database_path = GetDBAbsolutePath(database, *config.file_system);
+	} else {
+		auto tmp_fs = FileSystem::CreateLocal();
+		abs_database_path = GetDBAbsolutePath(database, *tmp_fs);
+	}
 	if (db_instances.find(abs_database_path) != db_instances.end()) {
 		throw duckdb::Exception(ExceptionType::CONNECTION,
 		                        "Instance with path: " + abs_database_path + " already exists.");

package/src/duckdb/src/main/extension/extension_helper.cpp

@@ -6,88 +6,93 @@
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/main/database.hpp"
 
-#if defined(BUILD_ICU_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
-#define ICU_STATICALLY_LOADED true
-#include "icu_extension.hpp"
-#else
-#define ICU_STATICALLY_LOADED false
+// Note that c++ preprocessor doesn't have a nice way to clean this up so we need to set the defines we use to false
+// explicitly when they are undefined
+#ifndef DUCKDB_EXTENSION_ICU_LINKED
+#define DUCKDB_EXTENSION_ICU_LINKED false
 #endif
 
-#if defined(BUILD_PARQUET_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
-#define PARQUET_STATICALLY_LOADED true
-#include "parquet_extension.hpp"
-#else
-#define PARQUET_STATICALLY_LOADED false
+#ifndef DUCKDB_EXTENSION_EXCEL_LINKED
+#define DUCKDB_EXTENSION_EXCEL_LINKED false
 #endif
 
-#if defined(BUILD_TPCH_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
-#define TPCH_STATICALLY_LOADED true
-#include "tpch_extension.hpp"
-#else
-#define TPCH_STATICALLY_LOADED false
+#ifndef DUCKDB_EXTENSION_PARQUET_LINKED
+#define DUCKDB_EXTENSION_PARQUET_LINKED false
 #endif
 
-#if defined(BUILD_TPCDS_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
-#define TPCDS_STATICALLY_LOADED true
-#include "tpcds_extension.hpp"
-#else
-#define TPCDS_STATICALLY_LOADED false
+#ifndef DUCKDB_EXTENSION_TPCH_LINKED
+#define DUCKDB_EXTENSION_TPCH_LINKED false
 #endif
 
-#if defined(BUILD_FTS_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
-#define FTS_STATICALLY_LOADED true
-#include "fts_extension.hpp"
-#else
-#define FTS_STATICALLY_LOADED false
+#ifndef DUCKDB_EXTENSION_TPCDS_LINKED
+#define DUCKDB_EXTENSION_TPCDS_LINKED false
 #endif
 
-#if defined(BUILD_HTTPFS_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
-#define HTTPFS_STATICALLY_LOADED true
-#include "httpfs_extension.hpp"
-#else
-#define HTTPFS_STATICALLY_LOADED false
+#ifndef DUCKDB_EXTENSION_FTS_LINKED
+#define DUCKDB_EXTENSION_FTS_LINKED false
 #endif
 
-#if defined(BUILD_VISUALIZER_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
-#include "visualizer_extension.hpp"
+#ifndef DUCKDB_EXTENSION_HTTPFS_LINKED
+#define DUCKDB_EXTENSION_HTTPFS_LINKED false
 #endif
 
-#if defined(BUILD_JSON_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
-#define JSON_STATICALLY_LOADED true
-#include "json_extension.hpp"
-#else
-#define JSON_STATICALLY_LOADED false
+#ifndef DUCKDB_EXTENSION_JSON_LINKED
+#define DUCKDB_EXTENSION_JSON_LINKED false
 #endif
 
-#if defined(BUILD_JEMALLOC_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
-#define JEMALLOC_STATICALLY_LOADED true
-#include "jemalloc_extension.hpp"
+#ifndef DUCKDB_EXTENSION_JEMALLOC_LINKED
+#define DUCKDB_EXTENSION_JEMALLOC_LINKED false
+#endif
+
+#ifndef DUCKDB_EXTENSION_AUTOCOMPLETE_LINKED
+#define DUCKDB_EXTENSION_AUTOCOMPLETE_LINKED false
+#endif
+
+// Load the generated header file containing our list of extension headers
+#if defined(GENERATED_EXTENSION_HEADERS) && GENERATED_EXTENSION_HEADERS && !defined(DUCKDB_AMALGAMATION)
+#include "duckdb/main/extension/generated_extension_loader.hpp"
 #else
-#define JEMALLOC_STATICALLY_LOADED false
+// TODO: rewrite package_build.py to allow also loading out-of-tree extensions in non-cmake builds, after that
+//		 these can be removed
+#if DUCKDB_EXTENSION_ICU_LINKED
+#include "icu_extension.hpp"
 #endif
 
-#if defined(BUILD_EXCEL_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
+#if DUCKDB_EXTENSION_EXCEL_LINKED
 #include "excel_extension.hpp"
 #endif
 
-#if defined(BUILD_SQLSMITH_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
-#include "sqlsmith_extension.hpp"
+#if DUCKDB_EXTENSION_PARQUET_LINKED
+#include "parquet_extension.hpp"
 #endif
 
-#if defined(BUILD_INET_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
-#include "inet_extension.hpp"
+#if DUCKDB_EXTENSION_TPCH_LINKED
+#include "tpch_extension.hpp"
 #endif
 
-#if defined(BUILD_AUTOCOMPLETE_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
-#define AUTOCOMPLETE_STATICALLY_LOADED true
-#include "autocomplete_extension.hpp"
-#else
-#define AUTOCOMPLETE_STATICALLY_LOADED false
+#if DUCKDB_EXTENSION_TPCDS_LINKED
+#include "tpcds_extension.hpp"
 #endif
 
-// Load the generated header file containing our list of extension headers
-#if defined(OOTE_HEADERS_AVAILABLE) && OOTE_HEADERS_AVAILABLE
-#include "extension_oote_loader.hpp"
+#if DUCKDB_EXTENSION_FTS_LINKED
+#include "fts_extension.hpp"
+#endif
+
+#if DUCKDB_EXTENSION_HTTPFS_LINKED
+#include "httpfs_extension.hpp"
+#endif
+
+#if DUCKDB_EXTENSION_JSON_LINKED
+#include "json_extension.hpp"
+#endif
+
+#if DUCKDB_EXTENSION_JEMALLOC_LINKED
+#include "jemalloc_extension.hpp"
+#endif
+
+#if DUCKDB_EXTENSION_AUTOCOMPLETE_LINKED
+#include "autocomplete_extension.hpp"
+#endif
 #endif
 
 namespace duckdb {
@@ -96,20 +101,27 @@ namespace duckdb {
 // Default Extensions
 //===--------------------------------------------------------------------===//
 static DefaultExtension internal_extensions[] = {
-    {"icu", "Adds support for time zones and collations using the ICU library", ICU_STATICALLY_LOADED},
-    {"parquet", "Adds support for reading and writing parquet files", PARQUET_STATICALLY_LOADED},
-    {"tpch", "Adds TPC-H data generation and query support", TPCH_STATICALLY_LOADED},
-    {"tpcds", "Adds TPC-DS data generation and query support", TPCDS_STATICALLY_LOADED},
-    {"fts", "Adds support for Full-Text Search Indexes", FTS_STATICALLY_LOADED},
-    {"httpfs", "Adds support for reading and writing files over a HTTP(S) connection", HTTPFS_STATICALLY_LOADED},
-    {"json", "Adds support for JSON operations", JSON_STATICALLY_LOADED},
-    {"jemalloc", "Overwrites system allocator with JEMalloc", JEMALLOC_STATICALLY_LOADED},
-    {"autocomplete", "Add supports for autocomplete in the shell", AUTOCOMPLETE_STATICALLY_LOADED},
+    {"icu", "Adds support for time zones and collations using the ICU library", DUCKDB_EXTENSION_ICU_LINKED},
+    {"excel", "Adds support for Excel-like format strings", DUCKDB_EXTENSION_EXCEL_LINKED},
+    {"parquet", "Adds support for reading and writing parquet files", DUCKDB_EXTENSION_PARQUET_LINKED},
+    {"tpch", "Adds TPC-H data generation and query support", DUCKDB_EXTENSION_TPCH_LINKED},
+    {"tpcds", "Adds TPC-DS data generation and query support", DUCKDB_EXTENSION_TPCDS_LINKED},
+    {"fts", "Adds support for Full-Text Search Indexes", DUCKDB_EXTENSION_FTS_LINKED},
+    {"httpfs", "Adds support for reading and writing files over a HTTP(S) connection", DUCKDB_EXTENSION_HTTPFS_LINKED},
+    {"json", "Adds support for JSON operations", DUCKDB_EXTENSION_JSON_LINKED},
+    {"jemalloc", "Overwrites system allocator with JEMalloc", DUCKDB_EXTENSION_JEMALLOC_LINKED},
+    {"autocomplete", "Adds support for autocomplete in the shell", DUCKDB_EXTENSION_AUTOCOMPLETE_LINKED},
     {"motherduck", "Enables motherduck integration with the system", false},
     {"sqlite_scanner", "Adds support for reading SQLite database files", false},
     {"postgres_scanner", "Adds support for reading from a Postgres database", false},
     {"inet", "Adds support for IP-related data types and functions", false},
     {"spatial", "Geospatial extension that adds support for working with spatial data and functions", false},
+    {"substrait", "Adds support for the Substrait integration", false},
+    {"aws", "Provides features that depend on the AWS SDK", false},
+    {"arrow", "A zero-copy data integration between Apache Arrow and DuckDB", false},
+    {"azure", "Adds a filesystem abstraction for Azure blob storage to DuckDB", false},
+    {"iceberg", "Adds support for Apache Iceberg", false},
+    {"visualizer", "Creates an HTML-based visualization of the query plan", false},
     {nullptr, nullptr, false}};
 
 idx_t ExtensionHelper::DefaultExtensionCount() {
@@ -129,6 +141,7 @@ DefaultExtension ExtensionHelper::GetDefaultExtension(idx_t index) {
 //===--------------------------------------------------------------------===//
 static const char *auto_install[] = {"motherduck", "postgres_scanner", "sqlite_scanner", nullptr};
 
+// TODO: unify with new autoload mechanism
 bool ExtensionHelper::AllowAutoInstall(const string &extension) {
 	auto lcase = StringUtil::Lower(extension);
 	for (idx_t i = 0; auto_install[i]; i++) {
@@ -139,18 +152,94 @@ bool ExtensionHelper::AllowAutoInstall(const string &extension) {
 	return false;
 }
 
+bool ExtensionHelper::CanAutoloadExtension(const string &ext_name) {
+#ifdef DUCKDB_DISABLE_EXTENSION_LOAD
+	return false;
+#endif
+
+	if (ext_name.empty()) {
+		return false;
+	}
+	for (const auto &ext : AUTOLOADABLE_EXTENSIONS) {
+		if (ext_name == ext) {
+			return true;
+		}
+	}
+	return false;
+}
+
+string ExtensionHelper::AddExtensionInstallHintToErrorMsg(ClientContext &context, const string &base_error,
+                                                          const string &extension_name) {
+	auto &dbconfig = DBConfig::GetConfig(context);
+	string install_hint;
+
+	if (!ExtensionHelper::CanAutoloadExtension(extension_name)) {
+		install_hint = "Please try installing and loading the " + extension_name + " extension:\nINSTALL " +
+		               extension_name + ";\nLOAD " + extension_name + ";\n\n";
+	} else if (!dbconfig.options.autoload_known_extensions) {
+		install_hint =
+		    "Please try installing and loading the " + extension_name + " extension by running:\nINSTALL " +
+		    extension_name + ";\nLOAD " + extension_name +
+		    ";\n\nAlternatively, consider enabling auto-install "
+		    "and auto-load by running:\nSET autoinstall_known_extensions=1;\nSET autoload_known_extensions=1;";
+	} else if (!dbconfig.options.autoinstall_known_extensions) {
+		install_hint =
+		    "Please try installing the " + extension_name + " extension by running:\nINSTALL " + extension_name +
+		    ";\n\nAlternatively, consider enabling autoinstall by running:\nSET autoinstall_known_extensions=1;";
+	}
+
+	if (!install_hint.empty()) {
+		return base_error + "\n\n" + install_hint;
+	}
+
+	return base_error;
+}
+
+bool ExtensionHelper::TryAutoLoadExtension(ClientContext &context, const string &extension_name) noexcept {
+	auto &dbconfig = DBConfig::GetConfig(context);
+	try {
+		if (dbconfig.options.autoinstall_known_extensions) {
+			ExtensionHelper::InstallExtension(context, extension_name, false,
+			                                  context.config.autoinstall_extension_repo);
+		}
+		ExtensionHelper::LoadExternalExtension(context, extension_name);
+		return true;
+	} catch (...) {
+		return false;
+	}
+	return false;
+}
+
+void ExtensionHelper::AutoLoadExtension(ClientContext &context, const string &extension_name) {
+	auto &dbconfig = DBConfig::GetConfig(context);
+	try {
+#ifndef DUCKDB_WASM
+		if (dbconfig.options.autoinstall_known_extensions) {
+			ExtensionHelper::InstallExtension(context, extension_name, false,
+			                                  context.config.autoinstall_extension_repo);
+		}
+#endif
+		ExtensionHelper::LoadExternalExtension(context, extension_name);
+	} catch (Exception &e) {
+		throw AutoloadException(extension_name, e);
+	}
+}
+
 //===--------------------------------------------------------------------===//
 // Load Statically Compiled Extension
 //===--------------------------------------------------------------------===//
 void ExtensionHelper::LoadAllExtensions(DuckDB &db) {
+	// The in-tree extensions that we check. Non-cmake builds are currently limited to these for static linking
+	// TODO: rewrite package_build.py to allow also loading out-of-tree extensions in non-cmake builds, after that
+	//		 these can be removed
 	unordered_set<string> extensions {"parquet", "icu",   "tpch",     "tpcds", "fts",      "httpfs",      "visualizer",
 	                                  "json",    "excel", "sqlsmith", "inet",  "jemalloc", "autocomplete"};
 	for (auto &ext : extensions) {
 		LoadExtensionInternal(db, ext, true);
 	}
 
-#if defined(OOTE_HEADERS_AVAILABLE) && OOTE_HEADERS_AVAILABLE
-	for (auto &ext : OOT_EXTENSIONS) {
+#if defined(GENERATED_EXTENSION_HEADERS) && GENERATED_EXTENSION_HEADERS
+	for (auto &ext : linked_extensions) {
 		LoadExtensionInternal(db, ext, true);
 	}
 #endif
@@ -178,105 +267,124 @@ ExtensionLoadResult ExtensionHelper::LoadExtensionInternal(DuckDB &db, const std
 		return ExtensionLoadResult::LOADED_EXTENSION;
 	}
 #endif
+
+#ifdef DUCKDB_EXTENSIONS_TEST_WITH_LOADABLE
+	// Note: weird comma's are on purpose to do easy string contains on a list of extension names
+	if (!initial_load && StringUtil::Contains(DUCKDB_EXTENSIONS_TEST_WITH_LOADABLE, "," + extension + ",")) {
+		Connection con(db);
+		auto result = con.Query((string) "LOAD '" + DUCKDB_EXTENSIONS_BUILD_PATH + "/" + extension + "/" + extension +
+		                        ".duckdb_extension'");
+		if (result->HasError()) {
+			result->Print();
+			return ExtensionLoadResult::EXTENSION_UNKNOWN;
+		}
+		return ExtensionLoadResult::LOADED_EXTENSION;
+	}
+#endif
+
+	// This is the main extension loading mechanism that loads the extension that are statically linked.
+#if defined(GENERATED_EXTENSION_HEADERS) && GENERATED_EXTENSION_HEADERS
+	if (TryLoadLinkedExtension(db, extension)) {
+		return ExtensionLoadResult::LOADED_EXTENSION;
+	} else {
+		return ExtensionLoadResult::NOT_LOADED;
+	}
+#endif
+
+	// This is the fallback to the "old" extension loading mechanism for non-cmake builds
+	// TODO: rewrite package_build.py to allow also loading out-of-tree extensions in non-cmake builds
 	if (extension == "parquet") {
-#if PARQUET_STATICALLY_LOADED
+#if DUCKDB_EXTENSION_PARQUET_LINKED
 		db.LoadExtension<ParquetExtension>();
 #else
 		// parquet extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "icu") {
-#if ICU_STATICALLY_LOADED
+#if DUCKDB_EXTENSION_ICU_LINKED
 		db.LoadExtension<IcuExtension>();
 #else
 		// icu extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "tpch") {
-#if TPCH_STATICALLY_LOADED
+#if DUCKDB_EXTENSION_TPCH_LINKED
 		db.LoadExtension<TpchExtension>();
 #else
 		// icu extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "tpcds") {
-#if TPCDS_STATICALLY_LOADED
+#if DUCKDB_EXTENSION_TPCDS_LINKED
 		db.LoadExtension<TpcdsExtension>();
 #else
 		// icu extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "fts") {
-#if FTS_STATICALLY_LOADED
-		db.LoadExtension<FtsExtension>();
+#if DUCKDB_EXTENSION_FTS_LINKED
+//		db.LoadExtension<FtsExtension>();
 #else
 		// fts extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "httpfs") {
-#if HTTPFS_STATICALLY_LOADED
+#if DUCKDB_EXTENSION_HTTPFS_LINKED
 		db.LoadExtension<HttpfsExtension>();
 #else
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "visualizer") {
-#if defined(BUILD_VISUALIZER_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
+#if DUCKDB_EXTENSION_VISUALIZER_LINKED
 		db.LoadExtension<VisualizerExtension>();
 #else
 		// visualizer extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "json") {
-#if JSON_STATICALLY_LOADED
+#if DUCKDB_EXTENSION_JSON_LINKED
 		db.LoadExtension<JsonExtension>();
 #else
 		// json extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "excel") {
-#if defined(BUILD_EXCEL_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
+#if DUCKDB_EXTENSION_EXCEL_LINKED
 		db.LoadExtension<ExcelExtension>();
 #else
 		// excel extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "sqlsmith") {
-#if defined(BUILD_SQLSMITH_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
+#if DUCKDB_EXTENSION_SQLSMITH_LINKED
 		db.LoadExtension<SqlsmithExtension>();
 #else
 		// excel extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "jemalloc") {
-#if defined(BUILD_JEMALLOC_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
+#if DUCKDB_EXTENSION_JEMALLOC_LINKED
 		db.LoadExtension<JemallocExtension>();
 #else
 		// jemalloc extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "autocomplete") {
-#if defined(BUILD_AUTOCOMPLETE_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
+#if DUCKDB_EXTENSION_AUTOCOMPLETE_LINKED
 		db.LoadExtension<AutocompleteExtension>();
 #else
 		// autocomplete extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "inet") {
-#if defined(BUILD_INET_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
+#if DUCKDB_EXTENSION_INET_LINKED
 		db.LoadExtension<InetExtension>();
 #else
 		// inet extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
-	} else {
-
-#if defined(OOTE_HEADERS_AVAILABLE) && OOTE_HEADERS_AVAILABLE
-		if (TryLoadLinkedExtension(db, extension)) {
-			return ExtensionLoadResult::LOADED_EXTENSION;
-		}
-#endif
-		return ExtensionLoadResult::EXTENSION_UNKNOWN;
 	}
+
 	return ExtensionLoadResult::LOADED_EXTENSION;
 }