duckdb 0.7.2-dev0.0 → 0.7.2-dev1138.0

package/src/duckdb/src/include/duckdb.h

@@ -35,7 +35,10 @@
 #endif
 #endif
 
-// duplicate of duckdb/common/constants.hpp
+// API versions
+// if no explicit API version is defined, the latest API version is used
+// Note that using older API versions (i.e. not using DUCKDB_API_LATEST) is deprecated.
+// These will not be supported long-term, and will be removed in future versions.
 #ifndef DUCKDB_API_0_3_1
 #define DUCKDB_API_0_3_1 1
 #endif
@@ -570,11 +573,19 @@ Use `duckdb_result_chunk_count` to figure out how many chunks there are in the r
 */
 DUCKDB_API duckdb_data_chunk duckdb_result_get_chunk(duckdb_result result, idx_t chunk_index);
 
+/*!
+Checks if the type of the internal result is StreamQueryResult.
+
+* result: The result object to check.
+* returns: Whether or not the result object is of the type StreamQueryResult
+*/
+DUCKDB_API bool duckdb_result_is_streaming(duckdb_result result);
+
 /*!
 Returns the number of data chunks present in the result.
 
 * result: The result object
-* returns: The resulting data chunk. Returns `NULL` if the chunk index is out of bounds.
+* returns: Number of data chunks present in the result.
 */
 DUCKDB_API idx_t duckdb_result_chunk_count(duckdb_result result);
 
@@ -1106,6 +1117,21 @@ Note that after calling `duckdb_pending_prepared`, the pending result should alw
 DUCKDB_API duckdb_state duckdb_pending_prepared(duckdb_prepared_statement prepared_statement,
                                                 duckdb_pending_result *out_result);
 
+/*!
+Executes the prepared statement with the given bound parameters, and returns a pending result.
+This pending result will create a streaming duckdb_result when executed.
+The pending result represents an intermediate structure for a query that is not yet fully executed.
+
+Note that after calling `duckdb_pending_prepared_streaming`, the pending result should always be destroyed using
+`duckdb_destroy_pending`, even if this function returns DuckDBError.
+
+* prepared_statement: The prepared statement to execute.
+* out_result: The pending query result.
+* returns: `DuckDBSuccess` on success or `DuckDBError` on failure.
+*/
+DUCKDB_API duckdb_state duckdb_pending_prepared_streaming(duckdb_prepared_statement prepared_statement,
+                                                          duckdb_pending_result *out_result);
+
 /*!
 Closes the pending result and de-allocates all memory allocated for the result.
 
@@ -1684,6 +1710,16 @@ Adds a parameter to the table function.
 */
 DUCKDB_API void duckdb_table_function_add_parameter(duckdb_table_function table_function, duckdb_logical_type type);
 
+/*!
+Adds a named parameter to the table function.
+
+* table_function: The table function
+* name: The name of the parameter
+* type: The type of the parameter to add.
+*/
+DUCKDB_API void duckdb_table_function_add_named_parameter(duckdb_table_function table_function, const char *name,
+                                                          duckdb_logical_type type);
+
 /*!
 Assigns extra information to the table function that can be fetched during binding, etc.
 
@@ -1792,6 +1828,17 @@ The result must be destroyed with `duckdb_destroy_value`.
 */
 DUCKDB_API duckdb_value duckdb_bind_get_parameter(duckdb_bind_info info, idx_t index);
 
+/*!
+Retrieves a named parameter with the given name.
+
+The result must be destroyed with `duckdb_destroy_value`.
+
+* info: The info object
+* name: The name of the parameter
+* returns: The value of the parameter. Must be destroyed with `duckdb_destroy_value`.
+*/
+DUCKDB_API duckdb_value duckdb_bind_get_named_parameter(duckdb_bind_info info, const char *name);
+
 /*!
 Sets the user-provided bind data in the bind object. This object can be retrieved again during execution.
 
@@ -2319,6 +2366,28 @@ Returns true if execution of the current query is finished.
 */
 DUCKDB_API bool duckdb_execution_is_finished(duckdb_connection con);
 
+//===--------------------------------------------------------------------===//
+// Streaming Result Interface
+//===--------------------------------------------------------------------===//
+
+/*!
+Fetches a data chunk from the (streaming) duckdb_result. This function should be called repeatedly until the result is
+exhausted.
+
+The result must be destroyed with `duckdb_destroy_data_chunk`.
+
+This function can only be used on duckdb_results created with 'duckdb_pending_prepared_streaming'
+
+If this function is used, none of the other result functions can be used and vice versa (i.e. this function cannot be
+mixed with the legacy result functions or the materialized result functions).
+
+It is not known beforehand how many chunks will be returned by this result.
+
+* result: The result object to fetch the data chunk from.
+* returns: The resulting data chunk. Returns `NULL` if the result has an error.
+*/
+DUCKDB_API duckdb_data_chunk duckdb_stream_fetch_chunk(duckdb_result result);
+
 #ifdef __cplusplus
 }
 #endif

package/src/duckdb/src/include/duckdb.hpp

@@ -11,5 +11,4 @@
 #include "duckdb/main/connection.hpp"
 #include "duckdb/main/database.hpp"
 #include "duckdb/main/query_result.hpp"
-#include "duckdb/main/loadable_extension.hpp"
 #include "duckdb/main/appender.hpp"

package/src/duckdb/src/main/capi/pending-c.cpp

@@ -9,14 +9,16 @@ using duckdb::PendingQueryResult;
 using duckdb::PendingStatementWrapper;
 using duckdb::PreparedStatementWrapper;
 
-duckdb_state duckdb_pending_prepared(duckdb_prepared_statement prepared_statement, duckdb_pending_result *out_result) {
+duckdb_state duckdb_pending_prepared_internal(duckdb_prepared_statement prepared_statement,
+                                              duckdb_pending_result *out_result, bool allow_streaming) {
 	if (!prepared_statement || !out_result) {
 		return DuckDBError;
 	}
 	auto wrapper = (PreparedStatementWrapper *)prepared_statement;
 	auto result = new PendingStatementWrapper();
+	result->allow_streaming = allow_streaming;
 	try {
-		result->statement = wrapper->statement->PendingQuery(wrapper->values, false);
+		result->statement = wrapper->statement->PendingQuery(wrapper->values, allow_streaming);
 	} catch (const duckdb::Exception &ex) {
 		result->statement = make_unique<PendingQueryResult>(duckdb::PreservedError(ex));
 	} catch (std::exception &ex) {
@@ -28,6 +30,15 @@ duckdb_state duckdb_pending_prepared(duckdb_prepared_statement prepared_statemen
 	return return_value;
 }
 
+duckdb_state duckdb_pending_prepared(duckdb_prepared_statement prepared_statement, duckdb_pending_result *out_result) {
+	return duckdb_pending_prepared_internal(prepared_statement, out_result, false);
+}
+
+duckdb_state duckdb_pending_prepared_streaming(duckdb_prepared_statement prepared_statement,
+                                               duckdb_pending_result *out_result) {
+	return duckdb_pending_prepared_internal(prepared_statement, out_result, true);
+}
+
 void duckdb_destroy_pending(duckdb_pending_result *pending_result) {
 	if (!pending_result || !*pending_result) {
 		return;
@@ -90,7 +101,9 @@ duckdb_state duckdb_execute_pending(duckdb_pending_result pending_result, duckdb
 	if (!wrapper->statement) {
 		return DuckDBError;
 	}
-	auto result = wrapper->statement->Execute();
+
+	std::unique_ptr<duckdb::QueryResult> result;
+	result = wrapper->statement->Execute();
 	wrapper->statement.reset();
 	return duckdb_translate_result(std::move(result), out_result);
 }

package/src/duckdb/src/main/capi/result-c.cpp

@@ -292,6 +292,10 @@ bool deprecated_materialize_result(duckdb_result *result) {
 		// already used as a new result set
 		return false;
 	}
+	if (result_data->result_set_type == CAPIResultSetType::CAPI_RESULT_TYPE_STREAMING) {
+		// already used as a streaming result
+		return false;
+	}
 	// materialize as deprecated result set
 	result_data->result_set_type = CAPIResultSetType::CAPI_RESULT_TYPE_DEPRECATED;
 	auto column_count = result_data->result->ColumnCount();
@@ -413,6 +417,10 @@ idx_t duckdb_row_count(duckdb_result *result) {
 		return 0;
 	}
 	auto &result_data = *((duckdb::DuckDBResultData *)result->internal_data);
+	if (result_data.result->type == duckdb::QueryResultType::STREAM_RESULT) {
+		// We can't know the row count beforehand
+		return 0;
+	}
 	auto &materialized = (duckdb::MaterializedQueryResult &)*result_data.result;
 	return materialized.RowCount();
 }
@@ -463,7 +471,10 @@ idx_t duckdb_result_chunk_count(duckdb_result result) {
 	if (result_data.result_set_type == duckdb::CAPIResultSetType::CAPI_RESULT_TYPE_DEPRECATED) {
 		return 0;
 	}
-	D_ASSERT(result_data.result->type == duckdb::QueryResultType::MATERIALIZED_RESULT);
+	if (result_data.result->type != duckdb::QueryResultType::MATERIALIZED_RESULT) {
+		// Can't know beforehand how many chunks are returned.
+		return 0;
+	}
 	auto &materialized = (duckdb::MaterializedQueryResult &)*result_data.result;
 	return materialized.Collection().ChunkCount();
 }
@@ -476,6 +487,10 @@ duckdb_data_chunk duckdb_result_get_chunk(duckdb_result result, idx_t chunk_idx)
 	if (result_data.result_set_type == duckdb::CAPIResultSetType::CAPI_RESULT_TYPE_DEPRECATED) {
 		return nullptr;
 	}
+	if (result_data.result->type != duckdb::QueryResultType::MATERIALIZED_RESULT) {
+		// This API is only supported for materialized query results
+		return nullptr;
+	}
 	result_data.result_set_type = duckdb::CAPIResultSetType::CAPI_RESULT_TYPE_MATERIALIZED;
 	auto &materialized = (duckdb::MaterializedQueryResult &)*result_data.result;
 	auto &collection = materialized.Collection();
@@ -487,3 +502,14 @@ duckdb_data_chunk duckdb_result_get_chunk(duckdb_result result, idx_t chunk_idx)
 	collection.FetchChunk(chunk_idx, *chunk);
 	return reinterpret_cast<duckdb_data_chunk>(chunk.release());
 }
+
+bool duckdb_result_is_streaming(duckdb_result result) {
+	if (!result.internal_data) {
+		return false;
+	}
+	if (duckdb_result_error(&result) != nullptr) {
+		return false;
+	}
+	auto &result_data = *((duckdb::DuckDBResultData *)result.internal_data);
+	return result_data.result->type == duckdb::QueryResultType::STREAM_RESULT;
+}

package/src/duckdb/src/main/capi/stream-c.cpp

@@ -0,0 +1,25 @@
+#include "duckdb/main/capi/capi_internal.hpp"
+#include "duckdb/common/types/timestamp.hpp"
+#include "duckdb/common/allocator.hpp"
+
+duckdb_data_chunk duckdb_stream_fetch_chunk(duckdb_result result) {
+	if (!result.internal_data) {
+		return nullptr;
+	}
+	auto &result_data = *((duckdb::DuckDBResultData *)result.internal_data);
+	if (result_data.result_set_type == duckdb::CAPIResultSetType::CAPI_RESULT_TYPE_DEPRECATED) {
+		return nullptr;
+	}
+	if (result_data.result->type != duckdb::QueryResultType::STREAM_RESULT) {
+		// We can only fetch from a StreamQueryResult
+		return nullptr;
+	}
+	result_data.result_set_type = duckdb::CAPIResultSetType::CAPI_RESULT_TYPE_STREAMING;
+	auto &streaming = (duckdb::StreamQueryResult &)*result_data.result;
+	if (!streaming.IsOpen()) {
+		return nullptr;
+	}
+	// FetchRaw ? Do we care about flattening them?
+	auto chunk = streaming.Fetch();
+	return reinterpret_cast<duckdb_data_chunk>(chunk.release());
+}

package/src/duckdb/src/main/capi/table_function-c.cpp

@@ -208,6 +208,16 @@ void duckdb_table_function_add_parameter(duckdb_table_function function, duckdb_
 	tf->arguments.push_back(*logical_type);
 }
 
+void duckdb_table_function_add_named_parameter(duckdb_table_function function, const char *name,
+                                               duckdb_logical_type type) {
+	if (!function || !type) {
+		return;
+	}
+	auto tf = (duckdb::TableFunction *)function;
+	auto logical_type = (duckdb::LogicalType *)type;
+	tf->named_parameters.insert({name, *logical_type});
+}
+
 void duckdb_table_function_set_extra_info(duckdb_table_function function, void *extra_info,
                                           duckdb_delete_callback_t destroy) {
 	if (!function) {
@@ -319,6 +329,19 @@ duckdb_value duckdb_bind_get_parameter(duckdb_bind_info info, idx_t index) {
 	return reinterpret_cast<duckdb_value>(new duckdb::Value(bind_info->input.inputs[index]));
 }
 
+duckdb_value duckdb_bind_get_named_parameter(duckdb_bind_info info, const char *name) {
+	if (!info || !name) {
+		return nullptr;
+	}
+	auto bind_info = (duckdb::CTableInternalBindInfo *)info;
+	auto t = bind_info->input.named_parameters.find(name);
+	if (t == bind_info->input.named_parameters.end()) {
+		return nullptr;
+	} else {
+		return reinterpret_cast<duckdb_value>(new duckdb::Value(t->second));
+	}
+}
+
 void duckdb_bind_set_bind_data(duckdb_bind_info info, void *bind_data, duckdb_delete_callback_t destroy) {
 	if (!info) {
 		return;

package/src/duckdb/src/main/client_context.cpp

@@ -1,52 +1,52 @@
 #include "duckdb/main/client_context.hpp"
 
-#include "duckdb/main/client_context_file_opener.hpp"
-#include "duckdb/main/query_profiler.hpp"
-#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_entry/scalar_function_catalog_entry.hpp"
+#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_search_path.hpp"
+#include "duckdb/common/file_system.hpp"
+#include "duckdb/common/http_state.hpp"
+#include "duckdb/common/preserved_error.hpp"
+#include "duckdb/common/progress_bar/progress_bar.hpp"
 #include "duckdb/common/serializer/buffered_deserializer.hpp"
+#include "duckdb/common/serializer/buffered_file_writer.hpp"
 #include "duckdb/common/serializer/buffered_serializer.hpp"
+#include "duckdb/common/types/column_data_collection.hpp"
+#include "duckdb/execution/column_binding_resolver.hpp"
+#include "duckdb/execution/operator/helper/physical_result_collector.hpp"
 #include "duckdb/execution/physical_plan_generator.hpp"
+#include "duckdb/main/appender.hpp"
+#include "duckdb/main/attached_database.hpp"
+#include "duckdb/main/client_context_file_opener.hpp"
+#include "duckdb/main/client_data.hpp"
 #include "duckdb/main/database.hpp"
+#include "duckdb/main/database_manager.hpp"
+#include "duckdb/main/error_manager.hpp"
 #include "duckdb/main/materialized_query_result.hpp"
-#include "duckdb/main/client_data.hpp"
+#include "duckdb/main/query_profiler.hpp"
 #include "duckdb/main/query_result.hpp"
+#include "duckdb/main/relation.hpp"
 #include "duckdb/main/stream_query_result.hpp"
 #include "duckdb/optimizer/optimizer.hpp"
-#include "duckdb/parser/parser.hpp"
+#include "duckdb/parallel/task_scheduler.hpp"
 #include "duckdb/parser/expression/constant_expression.hpp"
 #include "duckdb/parser/expression/parameter_expression.hpp"
 #include "duckdb/parser/parsed_data/create_function_info.hpp"
+#include "duckdb/parser/parsed_expression_iterator.hpp"
+#include "duckdb/parser/parser.hpp"
+#include "duckdb/parser/query_node/select_node.hpp"
 #include "duckdb/parser/statement/drop_statement.hpp"
+#include "duckdb/parser/statement/execute_statement.hpp"
 #include "duckdb/parser/statement/explain_statement.hpp"
+#include "duckdb/parser/statement/prepare_statement.hpp"
+#include "duckdb/parser/statement/relation_statement.hpp"
 #include "duckdb/parser/statement/select_statement.hpp"
 #include "duckdb/planner/operator/logical_execute.hpp"
 #include "duckdb/planner/planner.hpp"
-#include "duckdb/transaction/transaction_manager.hpp"
-#include "duckdb/transaction/transaction.hpp"
-#include "duckdb/storage/data_table.hpp"
-#include "duckdb/main/appender.hpp"
-#include "duckdb/main/relation.hpp"
-#include "duckdb/parser/statement/relation_statement.hpp"
-#include "duckdb/parallel/task_scheduler.hpp"
-#include "duckdb/common/serializer/buffered_file_writer.hpp"
 #include "duckdb/planner/pragma_handler.hpp"
-#include "duckdb/common/file_system.hpp"
-#include "duckdb/execution/column_binding_resolver.hpp"
-#include "duckdb/execution/operator/helper/physical_result_collector.hpp"
-#include "duckdb/parser/query_node/select_node.hpp"
-#include "duckdb/parser/parsed_expression_iterator.hpp"
-#include "duckdb/parser/statement/prepare_statement.hpp"
-#include "duckdb/parser/statement/execute_statement.hpp"
-#include "duckdb/common/types/column_data_collection.hpp"
-#include "duckdb/common/preserved_error.hpp"
-#include "duckdb/common/progress_bar/progress_bar.hpp"
-#include "duckdb/main/error_manager.hpp"
-#include "duckdb/main/database_manager.hpp"
+#include "duckdb/storage/data_table.hpp"
 #include "duckdb/transaction/meta_transaction.hpp"
-#include "duckdb/common/http_stats.hpp"
-#include "duckdb/main/attached_database.hpp"
+#include "duckdb/transaction/transaction.hpp"
+#include "duckdb/transaction/transaction_manager.hpp"
 
 namespace duckdb {
 
@@ -156,8 +156,8 @@ void ClientContext::BeginQueryInternal(ClientContextLock &lock, const string &qu
 PreservedError ClientContext::EndQueryInternal(ClientContextLock &lock, bool success, bool invalidate_transaction) {
 	client_data->profiler->EndQuery();
 
-	if (client_data->http_stats) {
-		client_data->http_stats->Reset();
+	if (client_data->http_state) {
+		client_data->http_state->Reset();
 	}
 
 	// Notify any registered state of query end
@@ -209,6 +209,7 @@ PreservedError ClientContext::EndQueryInternal(ClientContextLock &lock, bool suc
 }
 
 void ClientContext::CleanupInternal(ClientContextLock &lock, BaseQueryResult *result, bool invalidate_transaction) {
+	client_data->http_state = make_unique<HTTPState>();
 	if (!active_query) {
 		// no query currently active
 		return;
@@ -717,10 +718,6 @@ unique_ptr<PendingQueryResult> ClientContext::PendingStatementOrPreparedStatemen
 	auto &profiler = QueryProfiler::Get(*this);
 	profiler.StartQuery(query, IsExplainAnalyze(statement ? statement.get() : prepared->unbound_statement.get()));
 
-	if (IsExplainAnalyze(statement ? statement.get() : prepared->unbound_statement.get())) {
-		client_data->http_stats = make_unique<HTTPStats>();
-	}
-
 	bool invalidate_query = true;
 	try {
 		if (statement) {
@@ -813,12 +810,14 @@ unique_ptr<QueryResult> ClientContext::Query(const string &query, bool allow_str
 
 	unique_ptr<QueryResult> result;
 	QueryResult *last_result = nullptr;
+	bool last_had_result = false;
 	for (idx_t i = 0; i < statements.size(); i++) {
 		auto &statement = statements[i];
 		bool is_last_statement = i + 1 == statements.size();
 		PendingQueryParameters parameters;
 		parameters.allow_stream_result = allow_stream_result && is_last_statement;
 		auto pending_query = PendingQueryInternal(*lock, std::move(statement), parameters);
+		auto has_result = pending_query->properties.return_type == StatementReturnType::QUERY_RESULT;
 		unique_ptr<QueryResult> current_result;
 		if (pending_query->HasError()) {
 			current_result = make_unique<MaterializedQueryResult>(pending_query->GetErrorObject());
@@ -826,12 +825,17 @@ unique_ptr<QueryResult> ClientContext::Query(const string &query, bool allow_str
 			current_result = ExecutePendingQueryInternal(*lock, *pending_query);
 		}
 		// now append the result to the list of results
-		if (!last_result) {
+		if (!last_result || !last_had_result) {
 			// first result of the query
 			result = std::move(current_result);
 			last_result = result.get();
+			last_had_result = has_result;
 		} else {
 			// later results; attach to the result chain
+			// but only if there is a result
+			if (!has_result) {
+				continue;
+			}
 			last_result->next = std::move(current_result);
 			last_result = last_result->next.get();
 		}

package/src/duckdb/src/main/client_data.cpp

@@ -1,15 +1,16 @@
 #include "duckdb/main/client_data.hpp"
-#include "duckdb/main/client_context.hpp"
-#include "duckdb/main/client_context_file_opener.hpp"
-#include "duckdb/main/query_profiler.hpp"
-#include "duckdb/common/http_stats.hpp"
-#include "duckdb/common/random_engine.hpp"
-#include "duckdb/catalog/catalog_search_path.hpp"
+
 #include "duckdb/catalog/catalog.hpp"
+#include "duckdb/catalog/catalog_search_path.hpp"
+#include "duckdb/common/http_state.hpp"
+#include "duckdb/common/random_engine.hpp"
 #include "duckdb/common/serializer/buffered_file_writer.hpp"
 #include "duckdb/main/attached_database.hpp"
+#include "duckdb/main/client_context.hpp"
+#include "duckdb/main/client_context_file_opener.hpp"
 #include "duckdb/main/database.hpp"
 #include "duckdb/main/database_manager.hpp"
+#include "duckdb/main/query_profiler.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/main/config.cpp

@@ -9,6 +9,9 @@
 #include "duckdb/common/thread.hpp"
 #endif
 
+#include <cstdio>
+#include <inttypes.h>
+
 namespace duckdb {
 
 #define DUCKDB_GLOBAL(_PARAM)                                                                                          \
@@ -214,9 +217,75 @@ void DBConfig::SetDefaultMaxMemory() {
 	}
 }
 
+idx_t CGroupBandwidthQuota(idx_t physical_cores, FileSystem &fs) {
+	static constexpr const char *CPU_MAX = "/sys/fs/cgroup/cpu.max";
+	static constexpr const char *CFS_QUOTA = "/sys/fs/cgroup/cpu/cpu.cfs_quota_us";
+	static constexpr const char *CFS_PERIOD = "/sys/fs/cgroup/cpu/cpu.cfs_period_us";
+
+	int64_t quota, period;
+	char byte_buffer[1000];
+	unique_ptr<FileHandle> handle;
+	int64_t read_bytes;
+
+	if (fs.FileExists(CPU_MAX)) {
+		// cgroup v2
+		// https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html
+		handle =
+		    fs.OpenFile(CPU_MAX, FileFlags::FILE_FLAGS_READ, FileSystem::DEFAULT_LOCK, FileSystem::DEFAULT_COMPRESSION);
+		read_bytes = fs.Read(*handle, (void *)byte_buffer, 999);
+		byte_buffer[read_bytes] = '\0';
+		if (std::sscanf(byte_buffer, "%" SCNd64 " %" SCNd64 "", &quota, &period) != 2) {
+			return physical_cores;
+		}
+	} else if (fs.FileExists(CFS_QUOTA) && fs.FileExists(CFS_PERIOD)) {
+		// cgroup v1
+		// https://www.kernel.org/doc/html/latest/scheduler/sched-bwc.html#management
+
+		// Read the quota, this indicates how many microseconds the CPU can be utilized by this cgroup per period
+		handle = fs.OpenFile(CFS_QUOTA, FileFlags::FILE_FLAGS_READ, FileSystem::DEFAULT_LOCK,
+		                     FileSystem::DEFAULT_COMPRESSION);
+		read_bytes = fs.Read(*handle, (void *)byte_buffer, 999);
+		byte_buffer[read_bytes] = '\0';
+		if (std::sscanf(byte_buffer, "%" SCNd64 "", &quota) != 1) {
+			return physical_cores;
+		}
+
+		// Read the time period, a cgroup can utilize the CPU up to quota microseconds every period
+		handle = fs.OpenFile(CFS_PERIOD, FileFlags::FILE_FLAGS_READ, FileSystem::DEFAULT_LOCK,
+		                     FileSystem::DEFAULT_COMPRESSION);
+		read_bytes = fs.Read(*handle, (void *)byte_buffer, 999);
+		byte_buffer[read_bytes] = '\0';
+		if (std::sscanf(byte_buffer, "%" SCNd64 "", &period) != 1) {
+			return physical_cores;
+		}
+	} else {
+		// No cgroup quota
+		return physical_cores;
+	}
+	if (quota > 0 && period > 0) {
+		return idx_t(std::ceil((double)quota / (double)period));
+	} else {
+		return physical_cores;
+	}
+}
+
+idx_t GetSystemMaxThreadsInternal(FileSystem &fs) {
+#ifndef DUCKDB_NO_THREADS
+	idx_t physical_cores = std::thread::hardware_concurrency();
+#ifdef __linux__
+	auto cores_available_per_period = CGroupBandwidthQuota(physical_cores, fs);
+	return MaxValue<idx_t>(cores_available_per_period, 1);
+#else
+	return physical_cores;
+#endif
+#else
+	return 1;
+#endif
+}
+
 void DBConfig::SetDefaultMaxThreads() {
 #ifndef DUCKDB_NO_THREADS
-	options.maximum_threads = std::thread::hardware_concurrency();
+	options.maximum_threads = GetSystemMaxThreadsInternal(*file_system);
 #else
 	options.maximum_threads = 1;
 #endif

package/src/duckdb/src/main/database.cpp

@@ -115,6 +115,15 @@ ConnectionManager &ConnectionManager::Get(DatabaseInstance &db) {
 	return db.GetConnectionManager();
 }
 
+ClientContext *ConnectionManager::GetConnection(DatabaseInstance *db) {
+	for (auto &conn : connections) {
+		if (conn.first->db.get() == db) {
+			return conn.first;
+		}
+	}
+	return nullptr;
+}
+
 ConnectionManager &ConnectionManager::Get(ClientContext &context) {
 	return ConnectionManager::Get(DatabaseInstance::GetDatabase(context));
 }
@@ -191,8 +200,7 @@ void DatabaseInstance::Initialize(const char *database_path, DBConfig *user_conf
 	}
 
 	db_manager = make_unique<DatabaseManager>(*this);
-	buffer_manager =
-	    make_unique<BufferManager>(*this, config.options.temporary_directory, config.options.maximum_memory);
+	buffer_manager = make_unique<BufferManager>(*this, config.options.temporary_directory);
 	scheduler = make_unique<TaskScheduler>(*this);
 	object_cache = make_unique<ObjectCache>();
 	connection_manager = make_unique<ConnectionManager>();
@@ -271,6 +279,10 @@ BufferManager &DatabaseInstance::GetBufferManager() {
 	return *buffer_manager;
 }
 
+BufferPool &DatabaseInstance::GetBufferPool() {
+	return *config.buffer_pool;
+}
+
 DatabaseManager &DatabaseManager::Get(DatabaseInstance &db) {
 	return db.GetDatabaseManager();
 }
@@ -340,6 +352,11 @@ void DatabaseInstance::Configure(DBConfig &new_config) {
 	if (!config.default_allocator) {
 		config.default_allocator = Allocator::DefaultAllocatorReference();
 	}
+	if (new_config.buffer_pool) {
+		config.buffer_pool = std::move(new_config.buffer_pool);
+	} else {
+		config.buffer_pool = make_shared<BufferPool>(config.options.maximum_memory);
+	}
 }
 
 DBConfig &DBConfig::GetConfig(ClientContext &context) {

package/src/duckdb/src/main/extension/extension_install.cpp

@@ -206,8 +206,13 @@ void ExtensionHelper::InstallExtensionInternal(DBConfig &config, ClientConfig *c
 		if (exact_match) {
 			message += "\nAre you using a development build? In this case, extensions might not (yet) be uploaded.";
 		}
-		throw IOException("Failed to download extension \"%s\" at URL \"%s%s\"\n%s", extension_name, url_base,
-		                  url_local_part, message);
+		if (res.error() == duckdb_httplib::Error::Success) {
+			throw HTTPException(res.value(), "Failed to download extension \"%s\" at URL \"%s%s\"\n%s", extension_name,
+			                    url_base, url_local_part, message);
+		} else {
+			throw IOException("Failed to download extension \"%s\" at URL \"%s%s\"\n%s (ERROR %s)", extension_name,
+			                  url_base, url_local_part, message, to_string(res.error()));
+		}
 	}
 	auto decompressed_body = GZipFileSystem::UncompressGZIPString(res->body);
 	std::ofstream out(temp_path, std::ios::binary);

package/src/duckdb/src/main/prepared_statement.cpp

@@ -23,6 +23,10 @@ const string &PreparedStatement::GetError() {
 	return error.Message();
 }
 
+PreservedError &PreparedStatement::GetErrorObject() {
+	return error;
+}
+
 bool PreparedStatement::HasError() const {
 	return !success;
 }