duckdb 1.0.1-dev21.0 → 1.0.1-dev27.0

package/src/duckdb/src/include/duckdb/main/chunk_scan_state/batched_data_collection.hpp

@@ -0,0 +1,35 @@
+#pragma once
+
+#include "duckdb/main/chunk_scan_state.hpp"
+#include "duckdb/common/types/batched_data_collection.hpp"
+#include "duckdb/common/error_data.hpp"
+
+namespace duckdb {
+
+class BatchCollectionChunkScanState : public ChunkScanState {
+public:
+	BatchCollectionChunkScanState(BatchedDataCollection &collection, BatchedChunkIteratorRange &range,
+	                              ClientContext &context);
+	~BatchCollectionChunkScanState() override;
+
+public:
+	BatchCollectionChunkScanState(const BatchCollectionChunkScanState &other) = delete;
+	BatchCollectionChunkScanState &operator=(const BatchCollectionChunkScanState &other) = delete;
+	BatchCollectionChunkScanState(BatchCollectionChunkScanState &&other) = default;
+
+public:
+	bool LoadNextChunk(ErrorData &error) override;
+	bool HasError() const override;
+	ErrorData &GetError() override;
+	const vector<LogicalType> &Types() const override;
+	const vector<string> &Names() const override;
+
+private:
+	void InternalLoad(ErrorData &error);
+
+private:
+	BatchedDataCollection &collection;
+	BatchedChunkScanState state;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/main/client_config.hpp

@@ -12,8 +12,9 @@
 #include "duckdb/common/common.hpp"
 #include "duckdb/common/enums/output_type.hpp"
 #include "duckdb/common/enums/profiler_format.hpp"
-#include "duckdb/common/types/value.hpp"
 #include "duckdb/common/progress_bar/progress_bar.hpp"
+#include "duckdb/common/types/value.hpp"
+#include "duckdb/main/profiling_info.hpp"
 
 namespace duckdb {
 
@@ -37,6 +38,9 @@ struct ClientConfig {
 	//! The file to save query profiling information to, instead of printing it to the console
 	//! (empty = print to console)
 	string profiler_save_location;
+	//! The custom settings for the profiler
+	//! (empty = use the default settings)
+	profiler_settings_t profiler_settings = ProfilingInfo::DefaultSettings();
 
 	//! Allows suppressing profiler output, even if enabled. We turn on the profiler on all test runs but don't want
 	//! to output anything
@@ -91,6 +95,15 @@ struct ClientConfig {
 	idx_t ordered_aggregate_threshold = (idx_t(1) << 18);
 	//! The number of rows to accumulate before flushing during a partitioned write
 	idx_t partitioned_write_flush_threshold = idx_t(1) << idx_t(19);
+	//! The amount of rows we can keep open before we close and flush them during a partitioned write
+	idx_t partitioned_write_max_open_files = idx_t(100);
+	//! The number of rows we need on either table to choose a nested loop join
+	idx_t nested_loop_join_threshold = 5;
+	//! The number of rows we need on either table to choose a merge join over an IE join
+	idx_t merge_join_threshold = 1000;
+
+	//! The maximum amount of memory to keep buffered in a streaming query result. Default: 1mb.
+	idx_t streaming_buffer_size = 1000000;
 
 	//! Callback to create a progress bar display
 	progress_bar_display_create_func_t display_create_func = nullptr;
@@ -102,10 +115,16 @@ struct ClientConfig {
 	idx_t pivot_limit = 100000;
 
 	//! The threshold at which we switch from using filtered aggregates to LIST with a dedicated pivot operator
-	idx_t pivot_filter_threshold = 10;
+	idx_t pivot_filter_threshold = 20;
 
 	//! Whether or not the "/" division operator defaults to integer division or floating point division
 	bool integer_division = false;
+	//! When a scalar subquery returns multiple rows - return a random row instead of returning an error
+	bool scalar_subquery_error_on_multiple_rows = true;
+	//! Use IEE754-compliant floating point operations (returning NAN instead of errors/NULL)
+	bool ieee_floating_point_ops = true;
+	//! Allow ordering by non-integer literals - ordering by such literals has no effect
+	bool order_by_non_integer_literal = false;
 
 	//! Output error messages as structured JSON instead of as a raw string
 	bool errors_as_json = false;
@@ -113,6 +132,9 @@ struct ClientConfig {
 	//! Generic options
 	case_insensitive_map_t<Value> set_variables;
 
+	//! Variables set by the user
+	case_insensitive_map_t<Value> user_variables;
+
 	//! Function that is used to create the result collector for a materialized result
 	//! Defaults to PhysicalMaterializedCollector
 	get_result_collector_t result_collector = nullptr;
@@ -130,6 +152,50 @@ public:
 	bool AnyVerification() {
 		return query_verification_enabled || verify_external || verify_serializer || verify_fetch_row;
 	}
+
+	void SetUserVariable(const string &name, Value value) {
+		user_variables[name] = std::move(value);
+	}
+
+	bool GetUserVariable(const string &name, Value &result) {
+		auto entry = user_variables.find(name);
+		if (entry == user_variables.end()) {
+			return false;
+		}
+		result = entry->second;
+		return true;
+	}
+
+	void ResetUserVariable(const string &name) {
+		user_variables.erase(name);
+	}
+
+public:
+	void SetDefaultStreamingBufferSize();
+};
+
+struct ScopedConfigSetting {
+public:
+	using config_modify_func_t = std::function<void(ClientConfig &config)>;
+
+public:
+	explicit ScopedConfigSetting(ClientConfig &config, config_modify_func_t set_f = nullptr,
+	                             config_modify_func_t unset_f = nullptr)
+	    : config(config), set(std::move(set_f)), unset(std::move(unset_f)) {
+		if (set) {
+			set(config);
+		}
+	}
+	~ScopedConfigSetting() {
+		if (unset) {
+			unset(config);
+		}
+	}
+
+public:
+	ClientConfig &config;
+	config_modify_func_t set;
+	config_modify_func_t unset;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/main/client_context.hpp

@@ -10,24 +10,25 @@
 
 #include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_set.hpp"
-#include "duckdb/common/enums/pending_execution_result.hpp"
+#include "duckdb/common/atomic.hpp"
 #include "duckdb/common/deque.hpp"
+#include "duckdb/common/enums/pending_execution_result.hpp"
+#include "duckdb/common/enums/prepared_statement_mode.hpp"
+#include "duckdb/common/error_data.hpp"
 #include "duckdb/common/pair.hpp"
 #include "duckdb/common/unordered_set.hpp"
 #include "duckdb/common/winapi.hpp"
+#include "duckdb/main/client_config.hpp"
+#include "duckdb/main/client_context_state.hpp"
+#include "duckdb/main/client_properties.hpp"
+#include "duckdb/main/external_dependencies.hpp"
+#include "duckdb/main/pending_query_result.hpp"
 #include "duckdb/main/prepared_statement.hpp"
+#include "duckdb/main/settings.hpp"
 #include "duckdb/main/stream_query_result.hpp"
 #include "duckdb/main/table_description.hpp"
+#include "duckdb/planner/expression/bound_parameter_data.hpp"
 #include "duckdb/transaction/transaction_context.hpp"
-#include "duckdb/main/pending_query_result.hpp"
-#include "duckdb/common/atomic.hpp"
-#include "duckdb/main/client_config.hpp"
-#include "duckdb/main/external_dependencies.hpp"
-#include "duckdb/common/error_data.hpp"
-#include "duckdb/common/enums/prepared_statement_mode.hpp"
-#include "duckdb/main/client_properties.hpp"
-#include "duckdb/main/client_context_state.hpp"
-#include "duckdb/main/settings.hpp"
 
 namespace duckdb {
 class Appender;
@@ -47,12 +48,14 @@ class ScalarFunctionCatalogEntry;
 struct ActiveQueryContext;
 struct ParserOptions;
 class SimpleBufferedData;
+class BufferedData;
 struct ClientData;
 class ClientContextState;
+class RegisteredStateManager;
 
 struct PendingQueryParameters {
 	//! Prepared statement parameters (if any)
-	optional_ptr<case_insensitive_map_t<Value>> parameters;
+	optional_ptr<case_insensitive_map_t<BoundParameterData>> parameters;
 	//! Whether or not a stream result should be allowed
 	bool allow_stream_result = false;
 };
@@ -60,9 +63,11 @@ struct PendingQueryParameters {
 //! The ClientContext holds information relevant to the current client session
 //! during execution
 class ClientContext : public enable_shared_from_this<ClientContext> {
-	friend class PendingQueryResult; // LockContext
-	friend class SimpleBufferedData; // ExecuteTaskInternal
-	friend class StreamQueryResult;  // LockContext
+	friend class PendingQueryResult;  // LockContext
+	friend class BufferedData;        // ExecuteTaskInternal
+	friend class SimpleBufferedData;  // ExecuteTaskInternal
+	friend class BatchedBufferedData; // ExecuteTaskInternal
+	friend class StreamQueryResult;   // LockContext
 	friend class ConnectionManager;
 
 public:
@@ -73,10 +78,8 @@ public:
 	shared_ptr<DatabaseInstance> db;
 	//! Whether or not the query is interrupted
 	atomic<bool> interrupted;
-	//! External Objects (e.g., Python objects) that views depend of
-	unordered_map<string, vector<shared_ptr<ExternalDependency>>> external_dependencies;
 	//! Set of optional states (e.g. Caches) that can be held by the ClientContext
-	unordered_map<string, shared_ptr<ClientContextState>> registered_state;
+	unique_ptr<RegisteredStateManager> registered_state;
 	//! The client configuration
 	ClientConfig config;
 	//! The set of client-specific data
@@ -91,6 +94,8 @@ public:
 
 	//! Interrupt execution of a query
 	DUCKDB_API void Interrupt();
+	DUCKDB_API void CancelTransaction();
+
 	//! Enable query profiling
 	DUCKDB_API void EnableProfiling();
 	//! Disable query profiling
@@ -141,7 +146,8 @@ public:
 	//! It is possible that the prepared statement will be re-bound. This will generally happen if the catalog is
 	//! modified in between the prepared statement being bound and the prepared statement being run.
 	DUCKDB_API unique_ptr<QueryResult> Execute(const string &query, shared_ptr<PreparedStatementData> &prepared,
-	                                           case_insensitive_map_t<Value> &values, bool allow_stream_result = true);
+	                                           case_insensitive_map_t<BoundParameterData> &values,
+	                                           bool allow_stream_result = true);
 	DUCKDB_API unique_ptr<QueryResult> Execute(const string &query, shared_ptr<PreparedStatementData> &prepared,
 	                                           const PendingQueryParameters &parameters);
 
@@ -174,7 +180,6 @@ public:
 
 	//! Whether or not the given result object (streaming query result or pending query result) is active
 	DUCKDB_API bool IsActiveResult(ClientContextLock &lock, BaseQueryResult &result);
-	DUCKDB_API void SetActiveResult(ClientContextLock &lock, BaseQueryResult &result);
 
 	//! Returns the current executor
 	Executor &GetExecutor();
@@ -227,7 +232,7 @@ private:
 	//! Internally prepare a SQL statement. Caller must hold the context_lock.
 	shared_ptr<PreparedStatementData>
 	CreatePreparedStatement(ClientContextLock &lock, const string &query, unique_ptr<SQLStatement> statement,
-	                        optional_ptr<case_insensitive_map_t<Value>> values = nullptr,
+	                        optional_ptr<case_insensitive_map_t<BoundParameterData>> values = nullptr,
 	                        PreparedStatementMode mode = PreparedStatementMode::PREPARE_ONLY);
 	unique_ptr<PendingQueryResult> PendingStatementInternal(ClientContextLock &lock, const string &query,
 	                                                        unique_ptr<SQLStatement> statement,
@@ -243,8 +248,11 @@ private:
 	unique_ptr<ClientContextLock> LockContext();
 
 	void BeginQueryInternal(ClientContextLock &lock, const string &query);
-	ErrorData EndQueryInternal(ClientContextLock &lock, bool success, bool invalidate_transaction);
+	ErrorData EndQueryInternal(ClientContextLock &lock, bool success, bool invalidate_transaction,
+	                           optional_ptr<ErrorData> previous_error);
 
+	//! Wait until a task is available to execute
+	void WaitForTask(ClientContextLock &lock, BaseQueryResult &result);
 	PendingExecutionResult ExecuteTaskInternal(ClientContextLock &lock, BaseQueryResult &result, bool dry_run = false);
 
 	unique_ptr<PendingQueryResult> PendingStatementOrPreparedStatementInternal(
@@ -266,7 +274,7 @@ private:
 
 	shared_ptr<PreparedStatementData>
 	CreatePreparedStatementInternal(ClientContextLock &lock, const string &query, unique_ptr<SQLStatement> statement,
-	                                optional_ptr<case_insensitive_map_t<Value>> values);
+	                                optional_ptr<case_insensitive_map_t<BoundParameterData>> values);
 
 private:
 	//! Lock on using the ClientContext in parallel

package/src/duckdb/src/include/duckdb/main/client_context_state.hpp

@@ -8,8 +8,16 @@
 
 #pragma once
 
-#include "duckdb/common/common.hpp"
 #include "duckdb/common/enums/prepared_statement_mode.hpp"
+#include "duckdb/common/exception/transaction_exception.hpp"
+#include "duckdb/common/optional_ptr.hpp"
+#include "duckdb/main/config.hpp"
+#include "duckdb/main/valid_checker.hpp"
+#include "duckdb/transaction/meta_transaction.hpp"
+#include "duckdb/transaction/transaction_manager.hpp"
+#include "duckdb/main/database_manager.hpp"
+#include "duckdb/main/client_context.hpp"
+#include <mutex>
 
 namespace duckdb {
 class ClientContext;
@@ -18,6 +26,7 @@ class MetaTransaction;
 class PreparedStatementData;
 class SQLStatement;
 struct PendingQueryParameters;
+class RegisteredStateManager;
 
 enum class RebindQueryInfo { DO_NOT_REBIND, ATTEMPT_TO_REBIND };
 
@@ -42,12 +51,19 @@ public:
 	virtual void QueryEnd(ClientContext &context) {
 		QueryEnd();
 	}
+	virtual void QueryEnd(ClientContext &context, optional_ptr<ErrorData> error) {
+		QueryEnd(context);
+	}
 	virtual void TransactionBegin(MetaTransaction &transaction, ClientContext &context) {
 	}
 	virtual void TransactionCommit(MetaTransaction &transaction, ClientContext &context) {
 	}
 	virtual void TransactionRollback(MetaTransaction &transaction, ClientContext &context) {
 	}
+	virtual void TransactionRollback(MetaTransaction &transaction, ClientContext &context,
+	                                 optional_ptr<ErrorData> error) {
+		TransactionRollback(transaction, context);
+	}
 	virtual bool CanRequestRebind() {
 		return false;
 	}
@@ -62,6 +78,68 @@ public:
 	                                          RebindQueryInfo current_rebind) {
 		return RebindQueryInfo::DO_NOT_REBIND;
 	}
+	virtual void WriteProfilingInformation(std::ostream &ss) {
+	}
+
+public:
+	template <class TARGET>
+	TARGET &Cast() {
+		DynamicCastCheck<TARGET>(this);
+		return reinterpret_cast<TARGET &>(*this);
+	}
+	template <class TARGET>
+	const TARGET &Cast() const {
+		DynamicCastCheck<TARGET>(this);
+		return reinterpret_cast<const TARGET &>(*this);
+	}
+};
+
+class RegisteredStateManager {
+public:
+	template <class T, typename... ARGS>
+	shared_ptr<T> GetOrCreate(const string &key, ARGS &&... args) {
+		lock_guard<mutex> l(lock);
+		auto lookup = registered_state.find(key);
+		if (lookup != registered_state.end()) {
+			return shared_ptr_cast<ClientContextState, T>(lookup->second);
+		}
+		auto cache = make_shared_ptr<T>(std::forward<ARGS>(args)...);
+		registered_state[key] = cache;
+		return cache;
+	}
+
+	template <class T>
+	shared_ptr<T> Get(const string &key) {
+		lock_guard<mutex> l(lock);
+		auto lookup = registered_state.find(key);
+		if (lookup == registered_state.end()) {
+			return nullptr;
+		}
+		return shared_ptr_cast<ClientContextState, T>(lookup->second);
+	}
+
+	void Insert(const string &key, shared_ptr<ClientContextState> state_p) {
+		lock_guard<mutex> l(lock);
+		registered_state.insert(make_pair(key, std::move(state_p)));
+	}
+
+	void Remove(const string &key) {
+		lock_guard<mutex> l(lock);
+		registered_state.erase(key);
+	}
+
+	vector<shared_ptr<ClientContextState>> States() {
+		lock_guard<mutex> l(lock);
+		vector<shared_ptr<ClientContextState>> states;
+		for (auto &entry : registered_state) {
+			states.push_back(entry.second);
+		}
+		return states;
+	}
+
+private:
+	mutex lock;
+	unordered_map<string, shared_ptr<ClientContextState>> registered_state;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/main/client_properties.hpp

@@ -8,7 +8,8 @@
 
 #pragma once
 
-#include <string>
+#include "duckdb/common/string.hpp"
+#include "duckdb/common/types.hpp"
 
 namespace duckdb {
 
@@ -16,11 +17,16 @@ enum class ArrowOffsetSize : uint8_t { REGULAR, LARGE };
 
 //! A set of properties from the client context that can be used to interpret the query result
 struct ClientProperties {
-	ClientProperties(string time_zone_p, ArrowOffsetSize arrow_offset_size_p)
-	    : time_zone(std::move(time_zone_p)), arrow_offset_size(arrow_offset_size_p) {
+	ClientProperties(string time_zone_p, ArrowOffsetSize arrow_offset_size_p, bool arrow_use_list_view_p,
+	                 bool produce_arrow_string_view_p, bool lossless_conversion)
+	    : time_zone(std::move(time_zone_p)), arrow_offset_size(arrow_offset_size_p),
+	      arrow_use_list_view(arrow_use_list_view_p), arrow_lossless_conversion(lossless_conversion) {
 	}
 	ClientProperties() {};
 	string time_zone = "UTC";
 	ArrowOffsetSize arrow_offset_size = ArrowOffsetSize::REGULAR;
+	bool arrow_use_list_view = false;
+	bool produce_arrow_string_view = false;
+	bool arrow_lossless_conversion = false;
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/main/config.hpp

@@ -10,7 +10,9 @@
 
 #include "duckdb/common/allocator.hpp"
 #include "duckdb/common/case_insensitive_map.hpp"
+#include "duckdb/common/cgroups.hpp"
 #include "duckdb/common/common.hpp"
+#include "duckdb/common/encryption_state.hpp"
 #include "duckdb/common/enums/access_mode.hpp"
 #include "duckdb/common/enums/compression_type.hpp"
 #include "duckdb/common/enums/optimizer_type.hpp"
@@ -37,6 +39,7 @@ namespace duckdb {
 class BufferManager;
 class BufferPool;
 class CastFunctionSet;
+class CollationBinding;
 class ClientContext;
 class ErrorManager;
 class CompressionFunction;
@@ -45,6 +48,8 @@ class OperatorExtension;
 class StorageExtension;
 class ExtensionCallback;
 class SecretManager;
+class CompressionInfo;
+class EncryptionUtil;
 
 struct CompressionFunctionSet;
 struct DBConfig;
@@ -171,6 +176,12 @@ struct DBConfigOptions {
 	bool object_cache_enable = false;
 	//! Whether or not the global http metadata cache is used
 	bool http_metadata_cache_enable = false;
+	//! HTTP Proxy config as 'hostname:port'
+	string http_proxy;
+	//! HTTP Proxy username for basic auth
+	string http_proxy_username;
+	//! HTTP Proxy password for basic auth
+	string http_proxy_password;
 	//! Force checkpoint when CHECKPOINT is called or on shutdown, even if no changes have been made
 	bool force_checkpoint = false;
 	//! Run a checkpoint on successful shutdown and delete the WAL, to leave only a single database file behind
@@ -194,6 +205,13 @@ struct DBConfigOptions {
 	bool preserve_insertion_order = true;
 	//! Whether Arrow Arrays use Large or Regular buffers
 	ArrowOffsetSize arrow_offset_size = ArrowOffsetSize::REGULAR;
+	//! Whether LISTs should produce Arrow ListViews
+	bool arrow_use_list_view = false;
+	//! Whenever a DuckDB type does not have a clear native or canonical extension match in Arrow, export the types
+	//! with a duckdb.type_name extension name
+	bool arrow_arrow_lossless_conversion = false;
+	//! Whether when producing arrow objects we produce string_views or regular strings
+	bool produce_arrow_string_views = false;
 	//! Database configuration variables as controlled by SET
 	case_insensitive_map_t<Value> set_variables;
 	//! Database configuration variable default values;
@@ -216,22 +234,44 @@ struct DBConfigOptions {
 	bool immediate_transaction_mode = false;
 	//! Debug setting - how to initialize  blocks in the storage layer when allocating
 	DebugInitialize debug_initialize = DebugInitialize::NO_INITIALIZE;
+	//! The set of user-provided options
+	case_insensitive_map_t<Value> user_options;
 	//! The set of unrecognized (other) options
-	unordered_map<string, Value> unrecognized_options;
+	case_insensitive_map_t<Value> unrecognized_options;
 	//! Whether or not the configuration settings can be altered
 	bool lock_configuration = false;
 	//! Whether to print bindings when printing the plan (debug mode only)
 	static bool debug_print_bindings; // NOLINT: debug setting
 	//! The peak allocation threshold at which to flush the allocator after completing a task (1 << 27, ~128MB)
-	idx_t allocator_flush_threshold = 134217728;
+	idx_t allocator_flush_threshold = 134217728ULL;
+	//! If bulk deallocation larger than this occurs, flush outstanding allocations (1 << 30, ~1GB)
+	idx_t allocator_bulk_deallocation_flush_threshold = 1073741824ULL;
+	//! Whether the allocator background thread is enabled
+	bool allocator_background_threads = false;
 	//! DuckDB API surface
 	string duckdb_api;
 	//! Metadata from DuckDB callers
 	string custom_user_agent;
 	//! Use old implicit casting style (i.e. allow everything to be implicitly casted to VARCHAR)
 	bool old_implicit_casting = false;
+	//! The default block allocation size for new duckdb database files (new as-in, they do not yet exist).
+	idx_t default_block_alloc_size = DUCKDB_BLOCK_ALLOC_SIZE;
 	//!  Whether or not to abort if a serialization exception is thrown during WAL playback (when reading truncated WAL)
 	bool abort_on_wal_failure = false;
+	//! The index_scan_percentage sets a threshold for index scans.
+	//! If fewer than MAX(index_scan_max_count, index_scan_percentage * total_row_count)
+	//! rows match, we perform an index scan instead of a table scan.
+	double index_scan_percentage = 0.001;
+	//! The index_scan_max_count sets a threshold for index scans.
+	//! If fewer than MAX(index_scan_max_count, index_scan_percentage * total_row_count)
+	//! rows match, we perform an index scan instead of a table scan.
+	idx_t index_scan_max_count = STANDARD_VECTOR_SIZE;
+	//! The maximum number of schemas we will look through for "did you mean..." style errors in the catalog
+	idx_t catalog_error_max_schemas = 100;
+	//!  Whether or not to always write to the WAL file, even if this is not required
+	bool debug_skip_checkpoint_on_commit = false;
+	//! The maximum amount of vacuum tasks to schedule during a checkpoint
+	idx_t max_vacuum_tasks = 100;
 
 	bool operator==(const DBConfigOptions &other) const;
 };
@@ -279,6 +319,8 @@ public:
 	shared_ptr<BufferManager> buffer_manager;
 	//! Set of callbacks that can be installed by extensions
 	vector<unique_ptr<ExtensionCallback>> extension_callbacks;
+	//! Encryption Util for OpenSSL
+	shared_ptr<EncryptionUtil> encryption_util;
 
 public:
 	DUCKDB_API static DBConfig &GetConfig(ClientContext &context);
@@ -309,28 +351,34 @@ public:
 
 	DUCKDB_API static idx_t ParseMemoryLimit(const string &arg);
 
-	//! Return the list of possible compression functions for the specific physical type
-	DUCKDB_API vector<reference<CompressionFunction>> GetCompressionFunctions(PhysicalType data_type);
-	//! Return the compression function for the specified compression type/physical type combo
-	DUCKDB_API optional_ptr<CompressionFunction> GetCompressionFunction(CompressionType type, PhysicalType data_type);
+	//! Returns the list of possible compression functions for the physical type.
+	DUCKDB_API vector<reference<CompressionFunction>> GetCompressionFunctions(const PhysicalType physical_type);
+	//! Returns the compression function matching the compression and physical type.
+	DUCKDB_API optional_ptr<CompressionFunction> GetCompressionFunction(CompressionType type,
+	                                                                    const PhysicalType physical_type);
 
 	bool operator==(const DBConfig &other);
 	bool operator!=(const DBConfig &other);
 
 	DUCKDB_API CastFunctionSet &GetCastFunctions();
+	DUCKDB_API CollationBinding &GetCollationBinding();
 	DUCKDB_API IndexTypeSet &GetIndexTypes();
 	static idx_t GetSystemMaxThreads(FileSystem &fs);
+	static idx_t GetSystemAvailableMemory(FileSystem &fs);
+	static idx_t ParseMemoryLimitSlurm(const string &arg);
 	void SetDefaultMaxMemory();
 	void SetDefaultTempDirectory();
 
 	OrderType ResolveOrder(OrderType order_type) const;
 	OrderByNullType ResolveNullOrder(OrderType order_type, OrderByNullType null_type) const;
-	const std::string UserAgent() const;
+	const string UserAgent() const;
 
 private:
 	unique_ptr<CompressionFunctionSet> compression_functions;
 	unique_ptr<CastFunctionSet> cast_functions;
+	unique_ptr<CollationBinding> collation_bindings;
 	unique_ptr<IndexTypeSet> index_types;
+	bool is_user_config = true;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/main/connection.hpp

@@ -20,6 +20,7 @@
 #include "duckdb/main/stream_query_result.hpp"
 #include "duckdb/main/table_description.hpp"
 #include "duckdb/parser/sql_statement.hpp"
+#include "duckdb/main/profiling_node.hpp"
 
 namespace duckdb {
 
@@ -55,6 +56,9 @@ public:
 	//! Returns query profiling information for the current query
 	DUCKDB_API string GetProfilingInformation(ProfilerPrintFormat format = ProfilerPrintFormat::QUERY_TREE);
 
+	//! Returns the first node of the query profiling tree
+	DUCKDB_API optional_ptr<ProfilingNode> GetProfilingTree();
+
 	//! Interrupt execution of the current query
 	DUCKDB_API void Interrupt();
 
@@ -145,7 +149,7 @@ public:
 	DUCKDB_API shared_ptr<Relation> RelationFromQuery(const string &query, const string &alias = "queryrelation",
 	                                                  const string &error = "Expected a single SELECT statement");
 	DUCKDB_API shared_ptr<Relation> RelationFromQuery(unique_ptr<SelectStatement> select_stmt,
-	                                                  const string &alias = "queryrelation");
+	                                                  const string &alias = "queryrelation", const string &query = "");
 
 	//! Returns a substrait BLOB from a valid query
 	DUCKDB_API string GetSubstrait(const string &query);

package/src/duckdb/src/include/duckdb/main/database.hpp

@@ -26,7 +26,15 @@ class FileSystem;
 class TaskScheduler;
 class ObjectCache;
 struct AttachInfo;
+struct AttachOptions;
 class DatabaseFileSystem;
+struct DatabaseCacheEntry;
+
+struct ExtensionInfo {
+	bool is_loaded;
+	unique_ptr<ExtensionInstallInfo> install_info;
+	unique_ptr<ExtensionLoadedInfo> load_info;
+};
 
 class DatabaseInstance : public enable_shared_from_this<DatabaseInstance> {
 	friend class DuckDB;
@@ -55,17 +63,20 @@ public:
 	DUCKDB_API static DatabaseInstance &GetDatabase(ClientContext &context);
 	DUCKDB_API static const DatabaseInstance &GetDatabase(const ClientContext &context);
 
-	DUCKDB_API const unordered_set<string> &LoadedExtensions();
-	DUCKDB_API const unordered_map<string, ExtensionInstallInfo> &LoadedExtensionsData();
+	DUCKDB_API const unordered_map<string, ExtensionInfo> &GetExtensions();
 	DUCKDB_API bool ExtensionIsLoaded(const string &name);
 
 	DUCKDB_API SettingLookupResult TryGetCurrentSetting(const string &key, Value &result) const;
 
 	unique_ptr<AttachedDatabase> CreateAttachedDatabase(ClientContext &context, const AttachInfo &info,
-	                                                    const string &type, AccessMode access_mode);
+	                                                    const AttachOptions &options);
+
+	void AddExtensionInfo(const string &name, const ExtensionLoadedInfo &info);
+	void SetDatabaseCacheEntry(shared_ptr<DatabaseCacheEntry> entry);
 
 private:
 	void Initialize(const char *path, DBConfig *config);
+	void LoadExtensionSettings();
 	void CreateMainDatabase();
 
 	void Configure(DBConfig &config, const char *path);
@@ -76,10 +87,10 @@ private:
 	unique_ptr<TaskScheduler> scheduler;
 	unique_ptr<ObjectCache> object_cache;
 	unique_ptr<ConnectionManager> connection_manager;
-	unordered_set<string> loaded_extensions;
-	unordered_map<string, ExtensionInstallInfo> loaded_extensions_data;
+	unordered_map<string, ExtensionInfo> loaded_extensions_info;
 	ValidChecker db_validity;
 	unique_ptr<DatabaseFileSystem> db_file_system;
+	shared_ptr<DatabaseCacheEntry> db_cache_entry;
 };
 
 //! The database object. This object holds the catalog and all the