duckdb 0.8.2-dev77.0 → 0.9.0

package/src/duckdb/src/include/duckdb/function/table/arrow.hpp

@@ -16,25 +16,9 @@
 #include "duckdb/common/thread.hpp"
 #include "duckdb/common/unordered_map.hpp"
 #include "duckdb/function/built_in_functions.hpp"
+#include "duckdb/function/table/arrow/arrow_duck_schema.hpp"
 
 namespace duckdb {
-//===--------------------------------------------------------------------===//
-// Arrow Variable Size Types
-//===--------------------------------------------------------------------===//
-enum class ArrowVariableSizeType : uint8_t { FIXED_SIZE = 0, NORMAL = 1, SUPER_SIZE = 2 };
-
-//===--------------------------------------------------------------------===//
-// Arrow Time/Date Types
-//===--------------------------------------------------------------------===//
-enum class ArrowDateTimeType : uint8_t {
-	MILLISECONDS = 0,
-	MICROSECONDS = 1,
-	NANOSECONDS = 2,
-	SECONDS = 3,
-	DAYS = 4,
-	MONTHS = 5,
-	MONTH_DAY_NANO = 6
-};
 
 struct ArrowInterval {
 	int32_t months;
@@ -46,18 +30,6 @@ struct ArrowInterval {
 	}
 };
 
-struct ArrowConvertData {
-	ArrowConvertData(LogicalType type) : dictionary_type(type) {};
-	ArrowConvertData() {};
-
-	//! Hold type of dictionary
-	LogicalType dictionary_type;
-	//! If its a variable size type (e.g., strings, blobs, lists) holds which type it is
-	vector<pair<ArrowVariableSizeType, idx_t>> variable_sz_type;
-	//! If this is a date/time holds its precision
-	vector<ArrowDateTimeType> date_time_precision;
-};
-
 struct ArrowProjectedColumns {
 	unordered_map<idx_t, string> projection_map;
 	vector<string> columns;
@@ -73,11 +45,10 @@ typedef unique_ptr<ArrowArrayStreamWrapper> (*stream_factory_produce_t)(uintptr_
 typedef void (*stream_factory_get_schema_t)(uintptr_t stream_factory_ptr, ArrowSchemaWrapper &schema);
 
 struct ArrowScanFunctionData : public PyTableFunctionData {
+public:
 	ArrowScanFunctionData(stream_factory_produce_t scanner_producer_p, uintptr_t stream_factory_ptr_p)
 	    : lines_read(0), stream_factory_ptr(stream_factory_ptr_p), scanner_producer(scanner_producer_p) {
 	}
-	//! This holds the original list type (col_idx, [ArrowListType,size])
-	unordered_map<idx_t, unique_ptr<ArrowConvertData>> arrow_convert_data;
 	vector<LogicalType> all_types;
 	atomic<idx_t> lines_read;
 	ArrowSchemaWrapper schema_root;
@@ -86,6 +57,8 @@ struct ArrowScanFunctionData : public PyTableFunctionData {
 	uintptr_t stream_factory_ptr;
 	//! Pointer to the scanner factory produce
 	stream_factory_produce_t scanner_producer;
+	//! Arrow table data
+	ArrowTableType arrow_table;
 };
 
 struct ArrowScanLocalState : public LocalTableFunctionState {
@@ -94,6 +67,9 @@ struct ArrowScanLocalState : public LocalTableFunctionState {
 
 	unique_ptr<ArrowArrayStreamWrapper> stream;
 	shared_ptr<ArrowArrayWrapper> chunk;
+	// This vector hold the Arrow Vectors owned by DuckDB to allow for zero-copy
+	// Note that only DuckDB can release these vectors
+	unordered_map<idx_t, shared_ptr<ArrowArrayWrapper>> arrow_owned_data;
 	idx_t chunk_offset = 0;
 	idx_t batch_index = 0;
 	vector<column_t> column_ids;
@@ -132,8 +108,7 @@ public:
 	static unique_ptr<FunctionData> ArrowScanBind(ClientContext &context, TableFunctionBindInput &input,
 	                                              vector<LogicalType> &return_types, vector<string> &names);
 	//! Actual conversion from Arrow to DuckDB
-	static void ArrowToDuckDB(ArrowScanLocalState &scan_state,
-	                          std::unordered_map<idx_t, unique_ptr<ArrowConvertData>> &arrow_convert_data,
+	static void ArrowToDuckDB(ArrowScanLocalState &scan_state, const arrow_column_map_t &arrow_convert_data,
 	                          DataChunk &output, idx_t start, bool arrow_scan_is_projected = true);
 
 	//! Get next scan state
@@ -154,6 +129,8 @@ public:
 
 	//! Scan Function
 	static void ArrowScanFunction(ClientContext &context, TableFunctionInput &data, DataChunk &output);
+	static void PopulateArrowTableType(ArrowTableType &arrow_table, ArrowSchemaWrapper &schema_p, vector<string> &names,
+	                                   vector<LogicalType> &return_types);
 
 protected:
 	//! Defines Maximum Number of Threads
@@ -172,9 +149,7 @@ protected:
 	//! Renames repeated columns and case sensitive columns
 	static void RenameArrowColumns(vector<string> &names);
 	//! Helper function to get the DuckDB logical type
-	static LogicalType GetArrowLogicalType(ArrowSchema &schema,
-	                                       std::unordered_map<idx_t, unique_ptr<ArrowConvertData>> &arrow_convert_data,
-	                                       idx_t col_idx);
+	static unique_ptr<ArrowType> GetArrowLogicalType(ArrowSchema &schema);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/function/table/read_csv.hpp

@@ -8,14 +8,16 @@
 
 #pragma once
 
-#include "duckdb/function/table_function.hpp"
-#include "duckdb/function/scalar/strftime_format.hpp"
-#include "duckdb/execution/operator/persistent/csv_reader_options.hpp"
-#include "duckdb/execution/operator/persistent/buffered_csv_reader.hpp"
-#include "duckdb/execution/operator/persistent/parallel_csv_reader.hpp"
-#include "duckdb/execution/operator/persistent/csv_file_handle.hpp"
-#include "duckdb/execution/operator/persistent/csv_buffer.hpp"
+#include "duckdb/execution/operator/scan/csv/buffered_csv_reader.hpp"
+#include "duckdb/execution/operator/scan/csv/csv_buffer.hpp"
+#include "duckdb/execution/operator/scan/csv/csv_buffer_manager.hpp"
+#include "duckdb/execution/operator/scan/csv/csv_file_handle.hpp"
+#include "duckdb/execution/operator/scan/csv/csv_reader_options.hpp"
+#include "duckdb/execution/operator/scan/csv/parallel_csv_reader.hpp"
 #include "duckdb/function/built_in_functions.hpp"
+#include "duckdb/function/scalar/strftime_format.hpp"
+#include "duckdb/function/table_function.hpp"
+#include "duckdb/execution/operator/scan/csv/csv_state_machine_cache.hpp"
 
 namespace duckdb {
 
@@ -31,7 +33,7 @@ struct BaseCSVData : public TableFunctionData {
 	//! The file path of the CSV file to read or write
 	vector<string> files;
 	//! The CSV reader options
-	BufferedCSVReaderOptions options;
+	CSVReaderOptions options;
 	//! Offsets for generated columns
 	idx_t filename_col_idx;
 	idx_t hive_partition_col_idx;
@@ -50,8 +52,6 @@ struct WriteCSVData : public BaseCSVData {
 	vector<LogicalType> sql_types;
 	//! The newline string to write
 	string newline = "\n";
-	//! Whether or not we are writing a simple CSV (delimiter, quote and escape are all 1 byte in length)
-	bool is_simple;
 	//! The size of the CSV file (in bytes) that we buffer before we flush it to disk
 	idx_t flush_size = 4096 * 8;
 	//! For each byte whether or not the CSV file requires quotes when containing the byte
@@ -65,17 +65,9 @@ struct ColumnInfo {
 		names = std::move(names_p);
 		types = std::move(types_p);
 	}
-	void Serialize(FieldWriter &writer) const {
-		writer.WriteList<string>(names);
-		writer.WriteRegularSerializableList<LogicalType>(types);
-	}
+	void Serialize(Serializer &serializer) const;
+	static ColumnInfo Deserialize(Deserializer &deserializer);
 
-	static ColumnInfo Deserialize(FieldReader &reader) {
-		ColumnInfo info;
-		info.names = reader.ReadRequiredList<string>();
-		info.types = reader.ReadRequiredSerializableList<LogicalType, LogicalType>();
-		return info;
-	}
 	vector<std::string> names;
 	vector<LogicalType> types;
 };
@@ -89,8 +81,9 @@ struct ReadCSVData : public BaseCSVData {
 	vector<LogicalType> return_types;
 	//! The expected SQL names to be returned from the read - including added constants (e.g. filename, hive partitions)
 	vector<string> return_names;
-	//! The initial reader (if any): this is used when automatic detection is used during binding.
-	//! In this case, the CSV reader is already created and might as well be re-used.
+	//! The buffer manager (if any): this is used when automatic detection is used during binding.
+	//! In this case, some CSV buffers have already been read and can be reused.
+	shared_ptr<CSVBufferManager> buffer_manager;
 	unique_ptr<BufferedCSVReader> initial_reader;
 	//! The union readers are created (when csv union_by_name option is on) during binding
 	//! Those readers can be re-used during ReadCSVFunction
@@ -100,11 +93,18 @@ struct ReadCSVData : public BaseCSVData {
 	//! Reader bind data
 	MultiFileReaderBindData reader_bind;
 	vector<ColumnInfo> column_info;
+	//! The CSVStateMachineCache caches state machines created for sniffing and parsing csv files
+	//! We cache them because when reading very small csv files, the cost of creating all the possible
+	//! State machines for sniffing becomes a major bottleneck.
+	CSVStateMachineCache state_machine_cache;
 
 	void Initialize(unique_ptr<BufferedCSVReader> &reader) {
 		this->initial_reader = std::move(reader);
 	}
 	void FinalizeRead(ClientContext &context);
+
+	void Serialize(Serializer &serializer) const;
+	static unique_ptr<ReadCSVData> Deserialize(Deserializer &deserializer);
 };
 
 struct CSVCopyFunction {

package/src/duckdb/src/include/duckdb/function/table/system_functions.hpp

@@ -25,6 +25,10 @@ struct PragmaStorageInfo {
 	static void RegisterFunction(BuiltinFunctions &set);
 };
 
+struct PragmaMetadataInfo {
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
 struct PragmaLastProfilingOutput {
 	static void RegisterFunction(BuiltinFunctions &set);
 };
@@ -37,6 +41,10 @@ struct PragmaVersion {
 	static void RegisterFunction(BuiltinFunctions &set);
 };
 
+struct PragmaPlatform {
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
 struct PragmaDatabaseSize {
 	static void RegisterFunction(BuiltinFunctions &set);
 };
@@ -118,7 +126,7 @@ struct TestType {
 
 struct TestAllTypesFun {
 	static void RegisterFunction(BuiltinFunctions &set);
-	static vector<TestType> GetTestTypes();
+	static vector<TestType> GetTestTypes(bool large_enum = false);
 };
 
 struct TestVectorTypesFun {

package/src/duckdb/src/include/duckdb/function/table_function.hpp

@@ -202,10 +202,9 @@ typedef void (*table_function_pushdown_complex_filter_t)(ClientContext &context,
                                                          vector<unique_ptr<Expression>> &filters);
 typedef string (*table_function_to_string_t)(const FunctionData *bind_data);
 
-typedef void (*table_function_serialize_t)(FieldWriter &writer, const FunctionData *bind_data,
+typedef void (*table_function_serialize_t)(Serializer &serializer, const optional_ptr<FunctionData> bind_data,
                                            const TableFunction &function);
-typedef unique_ptr<FunctionData> (*table_function_deserialize_t)(PlanDeserializationState &context, FieldReader &reader,
-                                                                 TableFunction &function);
+typedef unique_ptr<FunctionData> (*table_function_deserialize_t)(Deserializer &deserializer, TableFunction &function);
 
 class TableFunction : public SimpleNamedParameterFunction {
 public:

package/src/duckdb/src/include/duckdb/function/table_macro_function.hpp

@@ -34,10 +34,8 @@ public:
 
 	string ToSQL(const string &schema, const string &name) const override;
 
-	static unique_ptr<MacroFunction> Deserialize(FieldReader &reader);
-
-protected:
-	void SerializeInternal(FieldWriter &writer) const override;
+	void Serialize(Serializer &serializer) const override;
+	static unique_ptr<MacroFunction> Deserialize(Deserializer &deserializer);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/function/udf_function.hpp

@@ -318,8 +318,9 @@ private:
 		case LogicalTypeId::DATE:
 			return std::is_same<T, date_t>();
 		case LogicalTypeId::TIME:
-		case LogicalTypeId::TIME_TZ:
 			return std::is_same<T, dtime_t>();
+		case LogicalTypeId::TIME_TZ:
+			return std::is_same<T, dtime_tz_t>();
 		case LogicalTypeId::TIMESTAMP:
 		case LogicalTypeId::TIMESTAMP_MS:
 		case LogicalTypeId::TIMESTAMP_NS:

package/src/duckdb/src/include/duckdb/main/attached_database.hpp

@@ -60,7 +60,7 @@ public:
 	bool IsInitialDatabase() const;
 	void SetInitialDatabase();
 
-	static string ExtractDatabaseName(const string &dbpath);
+	static string ExtractDatabaseName(const string &dbpath, FileSystem &fs);
 
 private:
 	DatabaseInstance &db;

package/src/duckdb/src/include/duckdb/main/capi/capi_internal.hpp

@@ -13,7 +13,7 @@
 #include "duckdb/common/types.hpp"
 #include "duckdb/common/types/data_chunk.hpp"
 #include "duckdb/main/appender.hpp"
-#include "duckdb/common/arrow/arrow_options.hpp"
+#include "duckdb/common/case_insensitive_map.hpp"
 
 #include <cstring>
 #include <cassert>
@@ -31,8 +31,9 @@ struct DatabaseData {
 };
 
 struct PreparedStatementWrapper {
+	//! Map of name -> values
+	case_insensitive_map_t<Value> values;
 	unique_ptr<PreparedStatement> statement;
-	vector<Value> values;
 };
 
 struct ExtractStatementsWrapper {
@@ -48,7 +49,7 @@ struct PendingStatementWrapper {
 struct ArrowResultWrapper {
 	unique_ptr<MaterializedQueryResult> result;
 	unique_ptr<DataChunk> current_chunk;
-	ArrowOptions options;
+	ClientProperties options;
 };
 
 struct AppenderWrapper {

package/src/duckdb/src/include/duckdb/main/chunk_scan_state/query_result.hpp

@@ -0,0 +1,29 @@
+#pragma once
+
+#include "duckdb/main/chunk_scan_state.hpp"
+#include "duckdb/common/preserved_error.hpp"
+
+namespace duckdb {
+
+class QueryResult;
+
+class QueryResultChunkScanState : public ChunkScanState {
+public:
+	QueryResultChunkScanState(QueryResult &result);
+	~QueryResultChunkScanState();
+
+public:
+	bool LoadNextChunk(PreservedError &error) override;
+	bool HasError() const override;
+	PreservedError &GetError() override;
+	const vector<LogicalType> &Types() const override;
+	const vector<string> &Names() const override;
+
+private:
+	bool InternalLoad(PreservedError &error);
+
+private:
+	QueryResult &result;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/main/chunk_scan_state.hpp

@@ -0,0 +1,43 @@
+#pragma once
+
+#include "duckdb/common/vector.hpp"
+#include "duckdb/common/unique_ptr.hpp"
+#include "duckdb/common/preserved_error.hpp"
+
+namespace duckdb {
+
+class DataChunk;
+
+//! Abstract chunk fetcher
+class ChunkScanState {
+public:
+	explicit ChunkScanState();
+	virtual ~ChunkScanState();
+
+public:
+	ChunkScanState(const ChunkScanState &other) = delete;
+	ChunkScanState(ChunkScanState &&other) = default;
+	ChunkScanState &operator=(const ChunkScanState &other) = delete;
+	ChunkScanState &operator=(ChunkScanState &&other) = default;
+
+public:
+	virtual bool LoadNextChunk(PreservedError &error) = 0;
+	virtual bool HasError() const = 0;
+	virtual PreservedError &GetError() = 0;
+	virtual const vector<LogicalType> &Types() const = 0;
+	virtual const vector<string> &Names() const = 0;
+	idx_t CurrentOffset() const;
+	idx_t RemainingInChunk() const;
+	DataChunk &CurrentChunk();
+	bool ChunkIsEmpty() const;
+	bool Finished() const;
+	bool ScanStarted() const;
+	void IncreaseOffset(idx_t increment, bool unsafe = false);
+
+protected:
+	idx_t offset = 0;
+	bool finished = false;
+	unique_ptr<DataChunk> current_chunk;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/main/client_config.hpp

@@ -67,6 +67,8 @@ struct ClientConfig {
 	bool enable_caching_operators = true;
 	//! Force parallelism of small tables, used for testing
 	bool verify_parallelism = false;
+	//! Enable the optimizer to consider index joins, which are disabled on default
+	bool enable_index_join = false;
 	//! Force index join independent of table cardinality, used for testing
 	bool force_index_join = false;
 	//! Force out-of-core computation for operators that support it, used for testing
@@ -75,6 +77,8 @@ struct ClientConfig {
 	bool force_no_cross_product = false;
 	//! Force use of IEJoin to implement AsOfJoin, used for testing
 	bool force_asof_iejoin = false;
+	//! Use range joins for inequalities, even if there are equality predicates
+	bool prefer_range_joins = false;
 	//! If this context should also try to use the available replacement scans
 	//! True by default
 	bool use_replacement_scans = true;
@@ -89,6 +93,8 @@ struct ClientConfig {
 
 	//! Override for the default extension repository
 	string custom_extension_repo = "";
+	//! Override for the default autoload extensoin repository
+	string autoinstall_extension_repo = "";
 
 	//! The explain output type used when none is specified (default: PHYSICAL_ONLY)
 	ExplainOutputType explain_output_type = ExplainOutputType::PHYSICAL_ONLY;
@@ -96,6 +102,9 @@ struct ClientConfig {
 	//! The maximum amount of pivot columns
 	idx_t pivot_limit = 100000;
 
+	//! The threshold at which we switch from using filtered aggregates to LIST with a dedicated pivot operator
+	idx_t pivot_filter_threshold = 10;
+
 	//! Whether or not the "/" division operator defaults to integer division or floating point division
 	bool integer_division = false;
 
@@ -110,8 +119,6 @@ public:
 	static ClientConfig &GetConfig(ClientContext &context);
 	static const ClientConfig &GetConfig(const ClientContext &context);
 
-	string ExtractTimezone() const;
-
 	bool AnyVerification() {
 		return query_verification_enabled || verify_external || verify_serializer;
 	}

package/src/duckdb/src/include/duckdb/main/client_context.hpp

@@ -24,6 +24,7 @@
 #include "duckdb/main/client_config.hpp"
 #include "duckdb/main/external_dependencies.hpp"
 #include "duckdb/common/preserved_error.hpp"
+#include "duckdb/main/client_properties.hpp"
 
 namespace duckdb {
 class Appender;
@@ -46,7 +47,7 @@ struct ClientData;
 
 struct PendingQueryParameters {
 	//! Prepared statement parameters (if any)
-	vector<Value> *parameters = nullptr;
+	optional_ptr<case_insensitive_map_t<Value>> parameters;
 	//! Whether or not a stream result should be allowed
 	bool allow_stream_result = false;
 };
@@ -134,16 +135,17 @@ public:
 	//! Create a pending query result from a prepared statement with the given name and set of parameters
 	//! It is possible that the prepared statement will be re-bound. This will generally happen if the catalog is
 	//! modified in between the prepared statement being bound and the prepared statement being run.
-	DUCKDB_API unique_ptr<PendingQueryResult>
-	PendingQuery(const string &query, shared_ptr<PreparedStatementData> &prepared, PendingQueryParameters parameters);
+	DUCKDB_API unique_ptr<PendingQueryResult> PendingQuery(const string &query,
+	                                                       shared_ptr<PreparedStatementData> &prepared,
+	                                                       const PendingQueryParameters &parameters);
 
 	//! Execute a prepared statement with the given name and set of parameters
 	//! It is possible that the prepared statement will be re-bound. This will generally happen if the catalog is
 	//! modified in between the prepared statement being bound and the prepared statement being run.
 	DUCKDB_API unique_ptr<QueryResult> Execute(const string &query, shared_ptr<PreparedStatementData> &prepared,
-	                                           vector<Value> &values, bool allow_stream_result = true);
+	                                           case_insensitive_map_t<Value> &values, bool allow_stream_result = true);
 	DUCKDB_API unique_ptr<QueryResult> Execute(const string &query, shared_ptr<PreparedStatementData> &prepared,
-	                                           PendingQueryParameters parameters);
+	                                           const PendingQueryParameters &parameters);
 
 	//! Gets current percentage of the query's progress, returns 0 in case the progress bar is disabled.
 	DUCKDB_API double GetProgress();
@@ -197,7 +199,7 @@ private:
 	                     PreservedError &error);
 	//! Issues a query to the database and returns a Pending Query Result
 	unique_ptr<PendingQueryResult> PendingQueryInternal(ClientContextLock &lock, unique_ptr<SQLStatement> statement,
-	                                                    PendingQueryParameters parameters, bool verify = true);
+	                                                    const PendingQueryParameters &parameters, bool verify = true);
 	unique_ptr<QueryResult> ExecutePendingQueryInternal(ClientContextLock &lock, PendingQueryResult &query);
 
 	//! Parse statements from a query
@@ -213,18 +215,18 @@ private:
 	unique_ptr<PendingQueryResult> PendingStatementOrPreparedStatement(ClientContextLock &lock, const string &query,
 	                                                                   unique_ptr<SQLStatement> statement,
 	                                                                   shared_ptr<PreparedStatementData> &prepared,
-	                                                                   PendingQueryParameters parameters);
+	                                                                   const PendingQueryParameters &parameters);
 	unique_ptr<PendingQueryResult> PendingPreparedStatement(ClientContextLock &lock,
 	                                                        shared_ptr<PreparedStatementData> statement_p,
-	                                                        PendingQueryParameters parameters);
+	                                                        const PendingQueryParameters &parameters);
 
 	//! Internally prepare a SQL statement. Caller must hold the context_lock.
-	shared_ptr<PreparedStatementData> CreatePreparedStatement(ClientContextLock &lock, const string &query,
-	                                                          unique_ptr<SQLStatement> statement,
-	                                                          vector<Value> *values = nullptr);
+	shared_ptr<PreparedStatementData>
+	CreatePreparedStatement(ClientContextLock &lock, const string &query, unique_ptr<SQLStatement> statement,
+	                        optional_ptr<case_insensitive_map_t<Value>> values = nullptr);
 	unique_ptr<PendingQueryResult> PendingStatementInternal(ClientContextLock &lock, const string &query,
 	                                                        unique_ptr<SQLStatement> statement,
-	                                                        PendingQueryParameters parameters);
+	                                                        const PendingQueryParameters &parameters);
 	unique_ptr<QueryResult> RunStatementInternal(ClientContextLock &lock, const string &query,
 	                                             unique_ptr<SQLStatement> statement, bool allow_stream_result,
 	                                             bool verify = true);
@@ -244,11 +246,11 @@ private:
 
 	unique_ptr<PendingQueryResult> PendingStatementOrPreparedStatementInternal(
 	    ClientContextLock &lock, const string &query, unique_ptr<SQLStatement> statement,
-	    shared_ptr<PreparedStatementData> &prepared, PendingQueryParameters parameters);
+	    shared_ptr<PreparedStatementData> &prepared, const PendingQueryParameters &parameters);
 
 	unique_ptr<PendingQueryResult> PendingQueryPreparedInternal(ClientContextLock &lock, const string &query,
 	                                                            shared_ptr<PreparedStatementData> &prepared,
-	                                                            PendingQueryParameters parameters);
+	                                                            const PendingQueryParameters &parameters);
 
 	unique_ptr<PendingQueryResult> PendingQueryInternal(ClientContextLock &, const shared_ptr<Relation> &relation,
 	                                                    bool allow_stream_result);

package/src/duckdb/src/include/duckdb/main/client_context_file_opener.hpp

@@ -21,6 +21,7 @@ public:
 	explicit ClientContextFileOpener(ClientContext &context_p) : context(context_p) {
 	}
 
+	bool TryGetCurrentSetting(const string &key, Value &result, FileOpenerInfo &info) override;
 	bool TryGetCurrentSetting(const string &key, Value &result) override;
 
 	ClientContext *TryGetClientContext() override {

package/src/duckdb/src/include/duckdb/main/client_data.hpp

@@ -13,6 +13,7 @@
 #include "duckdb/common/types/value.hpp"
 #include "duckdb/common/case_insensitive_map.hpp"
 #include "duckdb/common/atomic.hpp"
+#include "duckdb/execution/operator/scan/csv/csv_state_machine_cache.hpp"
 
 namespace duckdb {
 class AttachedDatabase;
@@ -29,7 +30,7 @@ class SchemaCatalogEntry;
 struct RandomEngine;
 
 struct ClientData {
-	ClientData(ClientContext &context);
+	explicit ClientData(ClientContext &context);
 	~ClientData();
 
 	//! Query profiler

package/src/duckdb/src/include/duckdb/main/client_properties.hpp

@@ -0,0 +1,25 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/main/client_properties.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include <string>
+
+namespace duckdb {
+enum ArrowOffsetSize { REGULAR, LARGE };
+
+//! A set of properties from the client context that can be used to interpret the query result
+struct ClientProperties {
+	ClientProperties(string time_zone_p, ArrowOffsetSize arrow_offset_size_p)
+	    : time_zone(std::move(time_zone_p)), arrow_offset_size(arrow_offset_size_p) {
+	}
+	ClientProperties() {};
+	string time_zone = "UTC";
+	ArrowOffsetSize arrow_offset_size = ArrowOffsetSize::REGULAR;
+};
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/main/config.hpp

@@ -28,7 +28,7 @@
 #include "duckdb/optimizer/optimizer_extension.hpp"
 #include "duckdb/parser/parser_extension.hpp"
 #include "duckdb/planner/operator_extension.hpp"
-#include "duckdb/common/arrow/arrow_options.hpp"
+#include "duckdb/main/client_properties.hpp"
 
 namespace duckdb {
 class BufferPool;
@@ -39,6 +39,7 @@ class CompressionFunction;
 class TableFunctionRef;
 class OperatorExtension;
 class StorageExtension;
+class ExtensionCallback;
 
 struct CompressionFunctionSet;
 struct DBConfig;
@@ -95,6 +96,18 @@ struct DBConfigOptions {
 	bool use_direct_io = false;
 	//! Whether extensions should be loaded on start-up
 	bool load_extensions = true;
+#ifdef DUCKDB_EXTENSION_AUTOLOAD_DEFAULT
+	//! Whether known extensions are allowed to be automatically loaded when a query depends on them
+	bool autoload_known_extensions = DUCKDB_EXTENSION_AUTOLOAD_DEFAULT;
+#else
+	bool autoload_known_extensions = false;
+#endif
+#ifdef DUCKDB_EXTENSION_AUTOINSTALL_DEFAULT
+	//! Whether known extensions are allowed to be automatically installed when a query depends on them
+	bool autoinstall_known_extensions = DUCKDB_EXTENSION_AUTOINSTALL_DEFAULT;
+#else
+	bool autoinstall_known_extensions = false;
+#endif
 	//! The maximum memory used by the database system (in bytes). Default: 80% of System available memory
 	idx_t maximum_memory = (idx_t)-1;
 	//! The maximum amount of CPU threads used by the database system. Default: all available.
@@ -158,6 +171,8 @@ struct DBConfigOptions {
 	bool lock_configuration = false;
 	//! Whether to print bindings when printing the plan (debug mode only)
 	static bool debug_print_bindings;
+	//! The peak allocation threshold at which to flush the allocator after completing a task (1 << 27, ~128MB)
+	idx_t allocator_flush_threshold = 134217728;
 
 	bool operator==(const DBConfigOptions &other) const;
 };
@@ -198,6 +213,8 @@ public:
 	case_insensitive_map_t<duckdb::unique_ptr<StorageExtension>> storage_extensions;
 	//! A buffer pool can be shared across multiple databases (if desired).
 	shared_ptr<BufferPool> buffer_pool;
+	//! Set of callbacks that can be installed by extensions
+	vector<unique_ptr<ExtensionCallback>> extension_callbacks;
 
 public:
 	DUCKDB_API static DBConfig &GetConfig(ClientContext &context);
@@ -234,6 +251,7 @@ public:
 	bool operator!=(const DBConfig &other);
 
 	DUCKDB_API CastFunctionSet &GetCastFunctions();
+	static idx_t GetSystemMaxThreads(FileSystem &fs);
 	void SetDefaultMaxThreads();
 	void SetDefaultMaxMemory();
 

package/src/duckdb/src/include/duckdb/main/connection.hpp

@@ -30,7 +30,7 @@ class DatabaseInstance;
 class DuckDB;
 class LogicalOperator;
 class SelectStatement;
-struct BufferedCSVReaderOptions;
+struct CSVReaderOptions;
 
 typedef void (*warning_callback)(std::string);
 
@@ -131,7 +131,7 @@ public:
 
 	//! Reads CSV file
 	DUCKDB_API shared_ptr<Relation> ReadCSV(const string &csv_file);
-	DUCKDB_API shared_ptr<Relation> ReadCSV(const string &csv_file, BufferedCSVReaderOptions &options);
+	DUCKDB_API shared_ptr<Relation> ReadCSV(const string &csv_file, named_parameter_map_t &&options);
 	DUCKDB_API shared_ptr<Relation> ReadCSV(const string &csv_file, const vector<string> &columns);
 
 	//! Reads Parquet file
@@ -169,8 +169,7 @@ public:
 	template <typename TR, typename... Args>
 	void CreateScalarFunction(const string &name, vector<LogicalType> args, LogicalType ret_type,
 	                          TR (*udf_func)(Args...)) {
-		scalar_function_t function =
-		    UDFWrapper::CreateScalarFunction<TR, Args...>(name, args, std::move(ret_type), udf_func);
+		scalar_function_t function = UDFWrapper::CreateScalarFunction<TR, Args...>(name, args, ret_type, udf_func);
 		UDFWrapper::RegisterFunction(name, args, ret_type, function, *context);
 	}