duckdb 1.3.2 → 1.3.3-dev4.0

package/src/duckdb/src/include/duckdb/main/attached_database.hpp

@@ -13,7 +13,6 @@
 #include "duckdb/common/mutex.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/catalog/catalog_entry.hpp"
-#include "duckdb/storage/storage_options.hpp"
 
 namespace duckdb {
 class Catalog;
@@ -24,6 +23,7 @@ class StorageExtension;
 class DatabaseManager;
 
 struct AttachInfo;
+struct StoredDatabasePath;
 
 enum class AttachedDatabaseType {
 	READ_WRITE_DATABASE,
@@ -32,13 +32,23 @@ enum class AttachedDatabaseType {
 	TEMP_DATABASE,
 };
 
+class DatabaseFilePathManager;
+
+struct StoredDatabasePath {
+	StoredDatabasePath(DatabaseFilePathManager &manager, string path, const string &name);
+	~StoredDatabasePath();
+
+	DatabaseFilePathManager &manager;
+	string path;
+};
+
 //! AttachOptions holds information about a database we plan to attach. These options are generalized, i.e.,
 //! they have to apply to any database file type (duckdb, sqlite, etc.).
 struct AttachOptions {
 	//! Constructor for databases we attach outside of the ATTACH DATABASE statement.
 	explicit AttachOptions(const DBConfigOptions &options);
 	//! Constructor for databases we attach when using ATTACH DATABASE.
-	AttachOptions(const unique_ptr<AttachInfo> &info, const AccessMode default_access_mode);
+	AttachOptions(const unordered_map<string, Value> &options, const AccessMode default_access_mode);
 
 	//! Defaults to the access mode configured in the DBConfig, unless specified otherwise.
 	AccessMode access_mode;
@@ -48,10 +58,12 @@ struct AttachOptions {
 	unordered_map<string, Value> options;
 	//! (optionally) a catalog can be provided with a default table
 	QualifiedName default_table;
+	//! The stored database path (in the path manager)
+	unique_ptr<StoredDatabasePath> stored_database_path;
 };
 
 //! The AttachedDatabase represents an attached database instance.
-class AttachedDatabase : public CatalogEntry {
+class AttachedDatabase : public CatalogEntry, public enable_shared_from_this<AttachedDatabase> {
 public:
 	//! Create the built-in system database (without storage).
 	explicit AttachedDatabase(DatabaseInstance &db, AttachedDatabaseType type = AttachedDatabaseType::SYSTEM_DATABASE);
@@ -63,12 +75,13 @@ public:
 	~AttachedDatabase() override;
 
 	//! Initializes the catalog and storage of the attached database.
-	void Initialize(optional_ptr<ClientContext> context = nullptr, StorageOptions options = StorageOptions());
+	void Initialize(optional_ptr<ClientContext> context = nullptr);
 	void FinalizeLoad(optional_ptr<ClientContext> context);
 	void Close();
 
 	Catalog &ParentCatalog() override;
 	const Catalog &ParentCatalog() const override;
+	bool HasStorageManager() const;
 	StorageManager &GetStorageManager();
 	Catalog &GetCatalog();
 	TransactionManager &GetTransactionManager();
@@ -96,6 +109,7 @@ public:
 
 private:
 	DatabaseInstance &db;
+	unique_ptr<StoredDatabasePath> stored_database_path;
 	unique_ptr<StorageManager> storage;
 	unique_ptr<Catalog> catalog;
 	unique_ptr<TransactionManager> transaction_manager;

package/src/duckdb/src/include/duckdb/main/capi/capi_internal.hpp

@@ -42,6 +42,11 @@ struct CClientContextWrapper {
 	ClientContext &context;
 };
 
+struct CClientArrowOptionsWrapper {
+	explicit CClientArrowOptionsWrapper(ClientProperties &properties) : properties(properties) {};
+	ClientProperties properties;
+};
+
 struct PreparedStatementWrapper {
 	//! Map of name -> values
 	case_insensitive_map_t<BoundParameterData> values;
@@ -64,8 +69,8 @@ struct ArrowResultWrapper {
 };
 
 struct AppenderWrapper {
-	unique_ptr<Appender> appender;
-	string error;
+	unique_ptr<BaseAppender> appender;
+	ErrorData error_data;
 };
 
 struct TableDescriptionWrapper {
@@ -73,6 +78,14 @@ struct TableDescriptionWrapper {
 	string error;
 };
 
+struct ErrorDataWrapper {
+	ErrorData error_data;
+};
+
+struct ExpressionWrapper {
+	unique_ptr<Expression> expr;
+};
+
 enum class CAPIResultSetType : uint8_t {
 	CAPI_RESULT_TYPE_NONE = 0,
 	CAPI_RESULT_TYPE_MATERIALIZED,
@@ -88,10 +101,14 @@ struct DuckDBResultData {
 	CAPIResultSetType result_set_type;
 };
 
-duckdb_type ConvertCPPTypeToC(const LogicalType &type);
-LogicalTypeId ConvertCTypeToCPP(duckdb_type c_type);
-idx_t GetCTypeSize(duckdb_type type);
+duckdb_type LogicalTypeIdToC(const LogicalTypeId type);
+LogicalTypeId LogicalTypeIdFromC(const duckdb_type type);
+idx_t GetCTypeSize(const duckdb_type type);
+duckdb_statement_type StatementTypeToC(const StatementType type);
+duckdb_error_type ErrorTypeToC(const ExceptionType type);
+ExceptionType ErrorTypeFromC(const duckdb_error_type type);
+
 duckdb_state DuckDBTranslateResult(unique_ptr<QueryResult> result, duckdb_result *out);
 bool DeprecatedMaterializeResult(duckdb_result *result);
-duckdb_statement_type StatementTypeToC(duckdb::StatementType statement_type);
+
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/main/capi/extension_api.hpp

@@ -136,7 +136,7 @@ typedef struct {
 	duckdb_value (*duckdb_create_timestamp)(duckdb_timestamp input);
 	duckdb_value (*duckdb_create_interval)(duckdb_interval input);
 	duckdb_value (*duckdb_create_blob)(const uint8_t *data, idx_t length);
-	duckdb_value (*duckdb_create_varint)(duckdb_varint input);
+	duckdb_value (*duckdb_create_bignum)(duckdb_bignum input);
 	duckdb_value (*duckdb_create_decimal)(duckdb_decimal input);
 	duckdb_value (*duckdb_create_bit)(duckdb_bit input);
 	duckdb_value (*duckdb_create_uuid)(duckdb_uhugeint input);
@@ -160,7 +160,7 @@ typedef struct {
 	duckdb_interval (*duckdb_get_interval)(duckdb_value val);
 	duckdb_logical_type (*duckdb_get_value_type)(duckdb_value val);
 	duckdb_blob (*duckdb_get_blob)(duckdb_value val);
-	duckdb_varint (*duckdb_get_varint)(duckdb_value val);
+	duckdb_bignum (*duckdb_get_bignum)(duckdb_value val);
 	duckdb_decimal (*duckdb_get_decimal)(duckdb_value val);
 	duckdb_bit (*duckdb_get_bit)(duckdb_value val);
 	duckdb_uhugeint (*duckdb_get_uuid)(duckdb_value val);
@@ -471,12 +471,54 @@ typedef struct {
 
 	duckdb_state (*duckdb_append_default_to_chunk)(duckdb_appender appender, duckdb_data_chunk chunk, idx_t col,
 	                                               idx_t row);
+	duckdb_error_data (*duckdb_appender_error_data)(duckdb_appender appender);
+	duckdb_state (*duckdb_appender_create_query)(duckdb_connection connection, const char *query, idx_t column_count,
+	                                             duckdb_logical_type *types, const char *table_name,
+	                                             const char **column_names, duckdb_appender *out_appender);
+	// New arrow interface functions
+
+	duckdb_error_data (*duckdb_to_arrow_schema)(duckdb_arrow_options arrow_options, duckdb_logical_type *types,
+	                                            const char **names, idx_t column_count, struct ArrowSchema *out_schema);
+	duckdb_error_data (*duckdb_data_chunk_to_arrow)(duckdb_arrow_options arrow_options, duckdb_data_chunk chunk,
+	                                                struct ArrowArray *out_arrow_array);
+	duckdb_error_data (*duckdb_schema_from_arrow)(duckdb_connection connection, struct ArrowSchema *schema,
+	                                              duckdb_arrow_converted_schema *out_types);
+	duckdb_error_data (*duckdb_data_chunk_from_arrow)(duckdb_connection connection, struct ArrowArray *arrow_array,
+	                                                  duckdb_arrow_converted_schema converted_schema,
+	                                                  duckdb_data_chunk *out_chunk);
+	void (*duckdb_destroy_arrow_converted_schema)(duckdb_arrow_converted_schema *arrow_converted_schema);
+	// New functions for duckdb error data
+
+	duckdb_error_data (*duckdb_create_error_data)(duckdb_error_type type, const char *message);
+	void (*duckdb_destroy_error_data)(duckdb_error_data *error_data);
+	duckdb_error_type (*duckdb_error_data_error_type)(duckdb_error_data error_data);
+	const char *(*duckdb_error_data_message)(duckdb_error_data error_data);
+	bool (*duckdb_error_data_has_error)(duckdb_error_data error_data);
+	// API to create and manipulate expressions
+
+	void (*duckdb_destroy_expression)(duckdb_expression *expr);
+	duckdb_logical_type (*duckdb_expression_return_type)(duckdb_expression expr);
+	bool (*duckdb_expression_is_foldable)(duckdb_expression expr);
+	duckdb_error_data (*duckdb_expression_fold)(duckdb_client_context context, duckdb_expression expr,
+	                                            duckdb_value *out_value);
 	// New functions around the client context
 
 	idx_t (*duckdb_client_context_get_connection_id)(duckdb_client_context context);
 	void (*duckdb_destroy_client_context)(duckdb_client_context *context);
 	void (*duckdb_connection_get_client_context)(duckdb_connection connection, duckdb_client_context *out_context);
 	duckdb_value (*duckdb_get_table_names)(duckdb_connection connection, const char *query, bool qualified);
+	void (*duckdb_connection_get_arrow_options)(duckdb_connection connection, duckdb_arrow_options *out_arrow_options);
+	void (*duckdb_destroy_arrow_options)(duckdb_arrow_options *arrow_options);
+	// API to get information about the results of a prepared statement
+
+	idx_t (*duckdb_prepared_statement_column_count)(duckdb_prepared_statement prepared_statement);
+	const char *(*duckdb_prepared_statement_column_name)(duckdb_prepared_statement prepared_statement, idx_t col_idx);
+	duckdb_logical_type (*duckdb_prepared_statement_column_logical_type)(duckdb_prepared_statement prepared_statement,
+	                                                                     idx_t col_idx);
+	duckdb_type (*duckdb_prepared_statement_column_type)(duckdb_prepared_statement prepared_statement, idx_t col_idx);
+	// New query execution functions
+
+	duckdb_arrow_options (*duckdb_result_get_arrow_options)(duckdb_result *result);
 	// New functions around scalar function binding
 
 	void (*duckdb_scalar_function_set_bind)(duckdb_scalar_function scalar_function, duckdb_scalar_function_bind_t bind);
@@ -485,24 +527,35 @@ typedef struct {
 	void (*duckdb_scalar_function_set_bind_data)(duckdb_bind_info info, void *bind_data,
 	                                             duckdb_delete_callback_t destroy);
 	void *(*duckdb_scalar_function_get_bind_data)(duckdb_function_info info);
+	void *(*duckdb_scalar_function_bind_get_extra_info)(duckdb_bind_info info);
+	idx_t (*duckdb_scalar_function_bind_get_argument_count)(duckdb_bind_info info);
+	duckdb_expression (*duckdb_scalar_function_bind_get_argument)(duckdb_bind_info info, idx_t index);
+	void (*duckdb_scalar_function_set_bind_data_copy)(duckdb_bind_info info, duckdb_copy_callback_t copy);
 	// New string functions that are added
 
 	char *(*duckdb_value_to_string)(duckdb_value value);
+	// New functions around table function binding
+
+	void (*duckdb_table_function_get_client_context)(duckdb_bind_info info, duckdb_client_context *out_context);
 	// New value functions that are added
 
 	duckdb_value (*duckdb_create_map_value)(duckdb_logical_type map_type, duckdb_value *keys, duckdb_value *values,
 	                                        idx_t entry_count);
 	duckdb_value (*duckdb_create_union_value)(duckdb_logical_type union_type, idx_t tag_index, duckdb_value value);
-	// An API to create new vector types
+	duckdb_value (*duckdb_create_time_ns)(duckdb_time_ns input);
+	duckdb_time_ns (*duckdb_get_time_ns)(duckdb_value val);
+	// API to create and manipulate vector types
 
 	duckdb_vector (*duckdb_create_vector)(duckdb_logical_type type, idx_t capacity);
 	void (*duckdb_destroy_vector)(duckdb_vector *vector);
-	void (*duckdb_slice_vector)(duckdb_vector vector, duckdb_selection_vector selection, idx_t len);
+	void (*duckdb_slice_vector)(duckdb_vector vector, duckdb_selection_vector sel, idx_t len);
 	void (*duckdb_vector_reference_value)(duckdb_vector vector, duckdb_value value);
 	void (*duckdb_vector_reference_vector)(duckdb_vector to_vector, duckdb_vector from_vector);
 	duckdb_selection_vector (*duckdb_create_selection_vector)(idx_t size);
-	void (*duckdb_destroy_selection_vector)(duckdb_selection_vector vector);
-	sel_t *(*duckdb_selection_vector_get_data_ptr)(duckdb_selection_vector vector);
+	void (*duckdb_destroy_selection_vector)(duckdb_selection_vector sel);
+	sel_t *(*duckdb_selection_vector_get_data_ptr)(duckdb_selection_vector sel);
+	void (*duckdb_vector_copy_sel)(duckdb_vector src, duckdb_vector dst, duckdb_selection_vector sel, idx_t src_count,
+	                               idx_t src_offset, idx_t dst_offset);
 } duckdb_ext_api_v1;
 
 //===--------------------------------------------------------------------===//
@@ -624,7 +677,7 @@ inline duckdb_ext_api_v1 CreateAPIv1() {
 	result.duckdb_create_timestamp = duckdb_create_timestamp;
 	result.duckdb_create_interval = duckdb_create_interval;
 	result.duckdb_create_blob = duckdb_create_blob;
-	result.duckdb_create_varint = duckdb_create_varint;
+	result.duckdb_create_bignum = duckdb_create_bignum;
 	result.duckdb_create_decimal = duckdb_create_decimal;
 	result.duckdb_create_bit = duckdb_create_bit;
 	result.duckdb_create_uuid = duckdb_create_uuid;
@@ -648,7 +701,7 @@ inline duckdb_ext_api_v1 CreateAPIv1() {
 	result.duckdb_get_interval = duckdb_get_interval;
 	result.duckdb_get_value_type = duckdb_get_value_type;
 	result.duckdb_get_blob = duckdb_get_blob;
-	result.duckdb_get_varint = duckdb_get_varint;
+	result.duckdb_get_bignum = duckdb_get_bignum;
 	result.duckdb_get_decimal = duckdb_get_decimal;
 	result.duckdb_get_bit = duckdb_get_bit;
 	result.duckdb_get_uuid = duckdb_get_uuid;
@@ -918,18 +971,48 @@ inline duckdb_ext_api_v1 CreateAPIv1() {
 	result.duckdb_get_or_create_from_cache = duckdb_get_or_create_from_cache;
 	result.duckdb_destroy_instance_cache = duckdb_destroy_instance_cache;
 	result.duckdb_append_default_to_chunk = duckdb_append_default_to_chunk;
+	result.duckdb_appender_error_data = duckdb_appender_error_data;
+	result.duckdb_appender_create_query = duckdb_appender_create_query;
+	result.duckdb_to_arrow_schema = duckdb_to_arrow_schema;
+	result.duckdb_data_chunk_to_arrow = duckdb_data_chunk_to_arrow;
+	result.duckdb_schema_from_arrow = duckdb_schema_from_arrow;
+	result.duckdb_data_chunk_from_arrow = duckdb_data_chunk_from_arrow;
+	result.duckdb_destroy_arrow_converted_schema = duckdb_destroy_arrow_converted_schema;
+	result.duckdb_create_error_data = duckdb_create_error_data;
+	result.duckdb_destroy_error_data = duckdb_destroy_error_data;
+	result.duckdb_error_data_error_type = duckdb_error_data_error_type;
+	result.duckdb_error_data_message = duckdb_error_data_message;
+	result.duckdb_error_data_has_error = duckdb_error_data_has_error;
+	result.duckdb_destroy_expression = duckdb_destroy_expression;
+	result.duckdb_expression_return_type = duckdb_expression_return_type;
+	result.duckdb_expression_is_foldable = duckdb_expression_is_foldable;
+	result.duckdb_expression_fold = duckdb_expression_fold;
 	result.duckdb_client_context_get_connection_id = duckdb_client_context_get_connection_id;
 	result.duckdb_destroy_client_context = duckdb_destroy_client_context;
 	result.duckdb_connection_get_client_context = duckdb_connection_get_client_context;
 	result.duckdb_get_table_names = duckdb_get_table_names;
+	result.duckdb_connection_get_arrow_options = duckdb_connection_get_arrow_options;
+	result.duckdb_destroy_arrow_options = duckdb_destroy_arrow_options;
+	result.duckdb_prepared_statement_column_count = duckdb_prepared_statement_column_count;
+	result.duckdb_prepared_statement_column_name = duckdb_prepared_statement_column_name;
+	result.duckdb_prepared_statement_column_logical_type = duckdb_prepared_statement_column_logical_type;
+	result.duckdb_prepared_statement_column_type = duckdb_prepared_statement_column_type;
+	result.duckdb_result_get_arrow_options = duckdb_result_get_arrow_options;
 	result.duckdb_scalar_function_set_bind = duckdb_scalar_function_set_bind;
 	result.duckdb_scalar_function_bind_set_error = duckdb_scalar_function_bind_set_error;
 	result.duckdb_scalar_function_get_client_context = duckdb_scalar_function_get_client_context;
 	result.duckdb_scalar_function_set_bind_data = duckdb_scalar_function_set_bind_data;
 	result.duckdb_scalar_function_get_bind_data = duckdb_scalar_function_get_bind_data;
+	result.duckdb_scalar_function_bind_get_extra_info = duckdb_scalar_function_bind_get_extra_info;
+	result.duckdb_scalar_function_bind_get_argument_count = duckdb_scalar_function_bind_get_argument_count;
+	result.duckdb_scalar_function_bind_get_argument = duckdb_scalar_function_bind_get_argument;
+	result.duckdb_scalar_function_set_bind_data_copy = duckdb_scalar_function_set_bind_data_copy;
 	result.duckdb_value_to_string = duckdb_value_to_string;
+	result.duckdb_table_function_get_client_context = duckdb_table_function_get_client_context;
 	result.duckdb_create_map_value = duckdb_create_map_value;
 	result.duckdb_create_union_value = duckdb_create_union_value;
+	result.duckdb_create_time_ns = duckdb_create_time_ns;
+	result.duckdb_get_time_ns = duckdb_get_time_ns;
 	result.duckdb_create_vector = duckdb_create_vector;
 	result.duckdb_destroy_vector = duckdb_destroy_vector;
 	result.duckdb_slice_vector = duckdb_slice_vector;
@@ -938,6 +1021,7 @@ inline duckdb_ext_api_v1 CreateAPIv1() {
 	result.duckdb_create_selection_vector = duckdb_create_selection_vector;
 	result.duckdb_destroy_selection_vector = duckdb_destroy_selection_vector;
 	result.duckdb_selection_vector_get_data_ptr = duckdb_selection_vector_get_data_ptr;
+	result.duckdb_vector_copy_sel = duckdb_vector_copy_sel;
 	return result;
 }
 

package/src/duckdb/src/include/duckdb/main/client_config.hpp

@@ -16,6 +16,7 @@
 #include "duckdb/common/types/value.hpp"
 #include "duckdb/main/profiling_info.hpp"
 #include "duckdb/parser/expression/lambda_expression.hpp"
+#include "duckdb/main/query_profiler.hpp"
 
 namespace duckdb {
 
@@ -23,8 +24,7 @@ class ClientContext;
 class PhysicalResultCollector;
 class PreparedStatementData;
 
-typedef std::function<unique_ptr<PhysicalResultCollector>(ClientContext &context, PreparedStatementData &data)>
-    get_result_collector_t;
+typedef std::function<PhysicalOperator &(ClientContext &context, PreparedStatementData &data)> get_result_collector_t;
 
 struct ClientConfig {
 	//! The home directory used by the system (if any)
@@ -55,9 +55,6 @@ struct ClientConfig {
 	//! The wait time before showing the progress bar
 	int wait_time = 2000;
 
-	//! Preserve identifier case while parsing.
-	//! If false, all unquoted identifiers are lower-cased (e.g. "MyTable" -> "mytable").
-	bool preserve_identifier_case = true;
 	//! The maximum expression depth limit in the parser
 	idx_t max_expression_depth = 1000;
 
@@ -77,32 +74,11 @@ struct ClientConfig {
 	bool verify_parallelism = false;
 	//! Force out-of-core computation for operators that support it, used for testing
 	bool force_external = false;
-	//! Force disable cross product generation when hyper graph isn't connected, used for testing
-	bool force_no_cross_product = false;
-	//! Force use of IEJoin to implement AsOfJoin, used for testing
-	bool force_asof_iejoin = false;
 	//! Force use of fetch row instead of scan, used for testing
 	bool force_fetch_row = false;
-	//! Use range joins for inequalities, even if there are equality predicates
-	bool prefer_range_joins = false;
 	//! If this context should also try to use the available replacement scans
 	//! True by default
 	bool use_replacement_scans = true;
-	//! Maximum bits allowed for using a perfect hash table (i.e. the perfect HT can hold up to 2^perfect_ht_threshold
-	//! elements)
-	idx_t perfect_ht_threshold = 12;
-	//! The maximum number of rows to accumulate before sorting ordered aggregates.
-	idx_t ordered_aggregate_threshold = (idx_t(1) << 18);
-	//! The number of rows to accumulate before flushing during a partitioned write
-	idx_t partitioned_write_flush_threshold = idx_t(1) << idx_t(19);
-	//! The amount of rows we can keep open before we close and flush them during a partitioned write
-	idx_t partitioned_write_max_open_files = idx_t(100);
-	//! The number of rows we need on either table to choose a nested loop join
-	idx_t nested_loop_join_threshold = 5;
-	//! The number of rows we need on either table to choose a merge join over an IE join
-	idx_t merge_join_threshold = 1000;
-	//! The maximum number of rows to use the nested loop join implementation
-	idx_t asof_loop_join_threshold = 64;
 
 	//! The maximum amount of memory to keep buffered in a streaming query result. Default: 1mb.
 	idx_t streaming_buffer_size = 1000000;
@@ -113,31 +89,12 @@ struct ClientConfig {
 	//! The explain output type used when none is specified (default: PHYSICAL_ONLY)
 	ExplainOutputType explain_output_type = ExplainOutputType::PHYSICAL_ONLY;
 
-	//! The maximum amount of pivot columns
-	idx_t pivot_limit = 100000;
-
-	//! The threshold at which we switch from using filtered aggregates to LIST with a dedicated pivot operator
-	idx_t pivot_filter_threshold = 20;
-
-	//! The maximum amount of OR filters we generate dynamically from a hash join
-	idx_t dynamic_or_filter_threshold = 50;
-
-	//! The maximum amount of rows in the LIMIT/SAMPLE for which we trigger late materialization
-	idx_t late_materialization_max_rows = 50;
-
-	//! Whether the "/" division operator defaults to integer division or floating point division
-	bool integer_division = false;
-	//! When a scalar subquery returns multiple rows - return a random row instead of returning an error
-	bool scalar_subquery_error_on_multiple_rows = true;
-	//! Use IEE754-compliant floating point operations (returning NAN instead of errors/NULL)
-	bool ieee_floating_point_ops = true;
-	//! Allow ordering by non-integer literals - ordering by such literals has no effect
-	bool order_by_non_integer_literal = false;
-	//! Disable casting from timestamp => timestamptz (naïve timestamps)
-	bool disable_timestamptz_casts = false;
 	//! If DEFAULT or ENABLE_SINGLE_ARROW, it is possible to use the deprecated single arrow operator (->) for lambda
 	//! functions. Otherwise, DISABLE_SINGLE_ARROW.
 	LambdaSyntax lambda_syntax = LambdaSyntax::DEFAULT;
+	//! The profiling coverage. SELECT is the default behavior, and ALL emits profiling information for all operator
+	//! types.
+	ProfilingCoverage profiling_coverage = ProfilingCoverage::SELECT;
 
 	//! Output error messages as structured JSON instead of as a raw string
 	bool errors_as_json = false;
@@ -148,9 +105,8 @@ struct ClientConfig {
 	//! Variables set by the user
 	case_insensitive_map_t<Value> user_variables;
 
-	//! Function that is used to create the result collector for a materialized result
-	//! Defaults to PhysicalMaterializedCollector
-	get_result_collector_t result_collector = nullptr;
+	//! Function that is used to create the result collector for a materialized result.
+	get_result_collector_t get_result_collector = nullptr;
 
 	//! If HTTP logging is enabled or not.
 	bool enable_http_logging = true;
@@ -163,26 +119,11 @@ public:
 	static ClientConfig &GetConfig(ClientContext &context);
 	static const ClientConfig &GetConfig(const ClientContext &context);
 
-	bool AnyVerification() {
-		return query_verification_enabled || verify_external || verify_serializer || verify_fetch_row;
-	}
+	bool AnyVerification() const;
 
-	void SetUserVariable(const string &name, Value value) {
-		user_variables[name] = std::move(value);
-	}
-
-	bool GetUserVariable(const string &name, Value &result) {
-		auto entry = user_variables.find(name);
-		if (entry == user_variables.end()) {
-			return false;
-		}
-		result = entry->second;
-		return true;
-	}
-
-	void ResetUserVariable(const string &name) {
-		user_variables.erase(name);
-	}
+	void SetUserVariable(const string &name, Value value);
+	bool GetUserVariable(const string &name, Value &result);
+	void ResetUserVariable(const String &name);
 
 	template <class OP>
 	static typename OP::RETURN_TYPE GetSetting(const ClientContext &context) {

package/src/duckdb/src/include/duckdb/main/client_context.hpp

@@ -24,13 +24,13 @@
 #include "duckdb/main/external_dependencies.hpp"
 #include "duckdb/main/pending_query_result.hpp"
 #include "duckdb/main/prepared_statement.hpp"
-#include "duckdb/main/settings.hpp"
 #include "duckdb/main/stream_query_result.hpp"
 #include "duckdb/main/table_description.hpp"
 #include "duckdb/planner/expression/bound_parameter_data.hpp"
 #include "duckdb/transaction/transaction_context.hpp"
 
 namespace duckdb {
+
 class Appender;
 class Catalog;
 class CatalogSearchPath;
@@ -131,6 +131,9 @@ public:
 	                                                  const string &table_name);
 	//! Get the table info of a specific table, or nullptr if it cannot be found. Uses INVALID_CATALOG.
 	DUCKDB_API unique_ptr<TableDescription> TableInfo(const string &schema_name, const string &table_name);
+	//! Execute a query with the given collection "attached" to the query using a CTE
+	DUCKDB_API void Append(ColumnDataCollection &collection, const string &query, const vector<string> &column_names,
+	                       const string &collection_name);
 	//! Appends a DataChunk and its default columns to the specified table.
 	DUCKDB_API void Append(TableDescription &description, ColumnDataCollection &collection,
 	                       optional_ptr<const vector<LogicalIndex>> column_ids = nullptr);
@@ -190,7 +193,7 @@ public:
 	                                                 bool requires_valid_transaction = true);
 
 	//! Equivalent to CURRENT_SETTING(key) SQL function.
-	DUCKDB_API SettingLookupResult TryGetCurrentSetting(const std::string &key, Value &result) const;
+	DUCKDB_API SettingLookupResult TryGetCurrentSetting(const string &key, Value &result) const;
 
 	//! Returns the parser options for this client context
 	DUCKDB_API ParserOptions GetParserOptions() const;
@@ -224,8 +227,7 @@ public:
 
 private:
 	//! Parse statements and resolve pragmas from a query
-	bool ParseStatements(ClientContextLock &lock, const string &query, vector<unique_ptr<SQLStatement>> &result,
-	                     ErrorData &error);
+	vector<unique_ptr<SQLStatement>> ParseStatements(ClientContextLock &lock, const string &query);
 	//! Issues a query to the database and returns a Pending Query Result
 	unique_ptr<PendingQueryResult> PendingQueryInternal(ClientContextLock &lock, unique_ptr<SQLStatement> statement,
 	                                                    const PendingQueryParameters &parameters, bool verify = true);
@@ -250,7 +252,7 @@ private:
 	                                                        shared_ptr<PreparedStatementData> statement_p,
 	                                                        const PendingQueryParameters &parameters);
 	unique_ptr<PendingQueryResult> PendingPreparedStatementInternal(ClientContextLock &lock,
-	                                                                shared_ptr<PreparedStatementData> statement_p,
+	                                                                shared_ptr<PreparedStatementData> statement_data_p,
 	                                                                const PendingQueryParameters &parameters);
 	void CheckIfPreparedStatementIsExecutable(PreparedStatementData &statement);
 
@@ -325,4 +327,25 @@ private:
 	lock_guard<mutex> client_guard;
 };
 
+//! The QueryContext wraps an optional client context.
+//! It makes query-related information available to operations.
+class QueryContext {
+public:
+	QueryContext() : context(nullptr) {
+	}
+	QueryContext(optional_ptr<ClientContext> context) : context(context) { // NOLINT: allow implicit construction
+	}
+
+public:
+	bool Valid() const {
+		return context != nullptr;
+	}
+	optional_ptr<ClientContext> GetClientContext() const {
+		return context;
+	}
+
+private:
+	optional_ptr<ClientContext> context;
+};
+
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/main/client_data.hpp

@@ -8,63 +8,61 @@
 
 #pragma once
 
-#include "duckdb/common/common.hpp"
-#include "duckdb/common/enums/output_type.hpp"
-#include "duckdb/common/types/value.hpp"
 #include "duckdb/common/case_insensitive_map.hpp"
-#include "duckdb/common/atomic.hpp"
-#include "duckdb/execution/operator/csv_scanner/csv_state_machine_cache.hpp"
+#include "duckdb/common/common.hpp"
 
 namespace duckdb {
+
 class AttachedDatabase;
 class BufferedFileWriter;
 class ClientContext;
 class CatalogSearchPath;
-class CachingFileSystem;
 class FileOpener;
 class FileSystem;
-class HTTPState;
 class QueryProfiler;
 class PreparedStatementData;
-class SchemaCatalogEntry;
 class RandomEngine;
+class BufferManager;
 
 struct ClientData {
+public:
 	explicit ClientData(ClientContext &context);
 	~ClientData();
 
-	//! Query profiler
+public:
+	//! The query profiler of this client.
 	shared_ptr<QueryProfiler> profiler;
 
-	//! The set of temporary objects that belong to this client
+	//! The set of temporary objects belonging to this client.
 	shared_ptr<AttachedDatabase> temporary_objects;
-	//! The set of bound prepared statements that belong to this client
+	//! The set of bound prepared statements belonging to this client.
 	case_insensitive_map_t<shared_ptr<PreparedStatementData>> prepared_statements;
 
-	//! The writer used to log queries (if logging is enabled) **Deprecated** queries are written to the generic log
-	//! now.
-	unique_ptr<BufferedFileWriter> log_query_writer;
-
-	//! The random generator used by random(). Its seed value can be set by setseed().
+	//! The random generator used by random().
+	//! Its seed value can be set by setseed().
 	unique_ptr<RandomEngine> random_engine;
 
-	//! The catalog search path
+	//! The catalog search path.
 	unique_ptr<CatalogSearchPath> catalog_search_path;
+	//! The file search path.
+	string file_search_path;
 
-	//! The file opener of the client context
+	//! The file opener of the client context.
 	unique_ptr<FileOpener> file_opener;
-
-	//! The clients' file system wrapper
+	//! The clients' file system wrapper.
 	unique_ptr<FileSystem> client_file_system;
-
-	//! The file search path
-	string file_search_path;
+	//! The clients' buffer manager wrapper.
+	unique_ptr<BufferManager> client_buffer_manager;
 
 	//! The Max Line Length Size of Last Query Executed on a CSV File. (Only used for testing)
 	//! FIXME: this should not be done like this
 	bool debug_set_max_line_length = false;
 	idx_t debug_max_line_length = 0;
 
+	//! The writer used to log queries, if logging is enabled.
+	//! DEPRECATED: Now, queries are written to the generic log.
+	unique_ptr<BufferedFileWriter> log_query_writer;
+
 public:
 	DUCKDB_API static ClientData &Get(ClientContext &context);
 	DUCKDB_API static const ClientData &Get(const ClientContext &context);

package/src/duckdb/src/include/duckdb/main/client_properties.hpp

@@ -10,27 +10,10 @@
 
 #include "duckdb/common/string.hpp"
 #include "duckdb/common/types.hpp"
+#include "duckdb/common/enums/arrow_format_version.hpp"
 
 namespace duckdb {
 
-enum class ArrowOffsetSize : uint8_t { REGULAR, LARGE };
-
-enum ArrowFormatVersion : uint8_t {
-	//! Base Version
-	V1_0 = 10,
-	//! Added 256-bit Decimal type.
-	V1_1 = 11,
-	//! Added MonthDayNano interval type.
-	V1_2 = 12,
-	//! Added Run-End Encoded Layout.
-	V1_3 = 13,
-	//! Added Variable-size Binary View Layout and the associated BinaryView and Utf8View types.
-	//! Added ListView Layout and the associated ListView and LargeListView types. Added Variadic buffers.
-	V1_4 = 14,
-	//! Expanded Decimal type bit widths to allow 32-bit and 64-bit types.
-	V1_5 = 15
-};
-
 //! A set of properties from the client context that can be used to interpret the query result
 struct ClientProperties {
 	ClientProperties(string time_zone_p, const ArrowOffsetSize arrow_offset_size_p, const bool arrow_use_list_view_p,
@@ -41,13 +24,14 @@ struct ClientProperties {
 	      arrow_lossless_conversion(lossless_conversion), arrow_output_version(arrow_output_version),
 	      client_context(client_context) {
 	}
+	ClientProperties() {};
 
 	string time_zone = "UTC";
 	ArrowOffsetSize arrow_offset_size = ArrowOffsetSize::REGULAR;
 	bool arrow_use_list_view = false;
 	bool produce_arrow_string_view = false;
 	bool arrow_lossless_conversion = false;
-	ArrowFormatVersion arrow_output_version = V1_0;
+	ArrowFormatVersion arrow_output_version = ArrowFormatVersion::V1_0;
 	optional_ptr<ClientContext> client_context;
 };
 } // namespace duckdb