duckdb 0.10.1-dev9.0 → 0.10.2-dev0.0

package/src/duckdb/src/include/duckdb/function/function.hpp

@@ -53,7 +53,7 @@ struct FunctionData {
 
 	template <class TARGET>
 	TARGET &Cast() {
-		D_ASSERT(dynamic_cast<TARGET *>(this));
+		DynamicCastCheck<TARGET>(this);
 		return reinterpret_cast<TARGET &>(*this);
 	}
 	template <class TARGET>

package/src/duckdb/src/include/duckdb/function/scalar/list/contains_or_position.hpp

@@ -130,6 +130,7 @@ void ListContainsOrPosition(DataChunk &args, Vector &result) {
 		break;
 	case PhysicalType::STRUCT:
 	case PhysicalType::LIST:
+	case PhysicalType::ARRAY:
 		TemplatedContainsOrPosition<int8_t, T, OP, LIST_ACCESSOR>(args, result, true);
 		break;
 	default:

package/src/duckdb/src/include/duckdb/function/scalar/nested_functions.hpp

@@ -45,7 +45,7 @@ struct PositionFunctor {
 		return 0;
 	}
 	static inline int32_t UpdateResultEntries(idx_t child_idx) {
-		return child_idx + 1;
+		return UnsafeNumericCast<int32_t>(child_idx + 1);
 	}
 };
 

package/src/duckdb/src/include/duckdb/function/scalar_function.hpp

@@ -24,7 +24,7 @@ struct FunctionLocalState {
 
 	template <class TARGET>
 	TARGET &Cast() {
-		D_ASSERT(dynamic_cast<TARGET *>(this));
+		DynamicCastCheck<TARGET>(this);
 		return reinterpret_cast<TARGET &>(*this);
 	}
 	template <class TARGET>

package/src/duckdb/src/include/duckdb/function/table/arrow.hpp

@@ -86,7 +86,11 @@ public:
 
 public:
 	ArrowScanLocalState &state;
+	// Hold ownership over the Arrow Arrays owned by DuckDB to allow for zero-copy
+	shared_ptr<ArrowArrayWrapper> owned_data;
 	unordered_map<idx_t, unique_ptr<ArrowArrayScanState>> children;
+	// Optionally holds the pointer that was used to create the cached dictionary
+	optional_ptr<ArrowArray> arrow_dictionary = nullptr;
 	// Cache the (optional) dictionary of this array
 	unique_ptr<Vector> dictionary;
 	//! Run-end-encoding state
@@ -94,8 +98,9 @@ public:
 
 public:
 	ArrowArrayScanState &GetChild(idx_t child_idx);
-	void AddDictionary(unique_ptr<Vector> dictionary_p);
+	void AddDictionary(unique_ptr<Vector> dictionary_p, ArrowArray *arrow_dict);
 	bool HasDictionary() const;
+	bool CacheOutdated(ArrowArray *dictionary) const;
 	Vector &GetDictionary();
 	ArrowRunEndEncodingState &RunEndEncoding() {
 		return run_end_encoding;
@@ -106,6 +111,10 @@ public:
 		// Note: dictionary is not reset
 		// the dictionary should be the same for every array scanned of this column
 		run_end_encoding.Reset();
+		for (auto &child : children) {
+			child.second->Reset();
+		}
+		owned_data.reset();
 	}
 };
 
@@ -117,9 +126,6 @@ public:
 public:
 	unique_ptr<ArrowArrayStreamWrapper> stream;
 	shared_ptr<ArrowArrayWrapper> chunk;
-	// This vector hold the Arrow Vectors owned by DuckDB to allow for zero-copy
-	// Note that only DuckDB can release these vectors
-	unordered_map<idx_t, shared_ptr<ArrowArrayWrapper>> arrow_owned_data;
 	idx_t chunk_offset = 0;
 	idx_t batch_index = 0;
 	vector<column_t> column_ids;

package/src/duckdb/src/include/duckdb/function/table/read_csv.hpp

@@ -9,15 +9,15 @@
 #pragma once
 
 #include "duckdb/common/multi_file_reader.hpp"
-#include "duckdb/execution/operator/csv_scanner/buffer_manager/csv_buffer.hpp"
-#include "duckdb/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.hpp"
-#include "duckdb/execution/operator/csv_scanner/buffer_manager/csv_file_handle.hpp"
-#include "duckdb/execution/operator/csv_scanner/options/csv_reader_options.hpp"
-#include "duckdb/execution/operator/csv_scanner/state_machine/csv_state_machine_cache.hpp"
+#include "duckdb/execution/operator/csv_scanner/csv_buffer.hpp"
+#include "duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp"
+#include "duckdb/execution/operator/csv_scanner/csv_file_handle.hpp"
+#include "duckdb/execution/operator/csv_scanner/csv_reader_options.hpp"
+#include "duckdb/execution/operator/csv_scanner/csv_state_machine_cache.hpp"
 #include "duckdb/function/built_in_functions.hpp"
 #include "duckdb/function/scalar/strftime_format.hpp"
 #include "duckdb/function/table_function.hpp"
-#include "duckdb/execution/operator/csv_scanner/table_function/csv_file_scanner.hpp"
+#include "duckdb/execution/operator/csv_scanner/csv_file_scanner.hpp"
 
 namespace duckdb {
 class BaseScanner;
@@ -80,6 +80,8 @@ struct ReadCSVData : public BaseCSVData {
 	vector<LogicalType> csv_types;
 	//! The expected SQL names to be read from the file
 	vector<string> csv_names;
+	//! If the sql types from the file were manually set
+	vector<bool> manually_set;
 	//! The expected SQL types to be returned from the read - including added constants (e.g. filename, hive partitions)
 	vector<LogicalType> return_types;
 	//! The expected SQL names to be returned from the read - including added constants (e.g. filename, hive partitions)

package/src/duckdb/src/include/duckdb/function/table_function.hpp

@@ -31,7 +31,7 @@ struct TableFunctionInfo {
 
 	template <class TARGET>
 	TARGET &Cast() {
-		D_ASSERT(dynamic_cast<TARGET *>(this));
+		DynamicCastCheck<TARGET>(this);
 		return reinterpret_cast<TARGET &>(*this);
 	}
 	template <class TARGET>
@@ -55,7 +55,7 @@ public:
 
 	template <class TARGET>
 	TARGET &Cast() {
-		D_ASSERT(dynamic_cast<TARGET *>(this));
+		DynamicCastCheck<TARGET>(this);
 		return reinterpret_cast<TARGET &>(*this);
 	}
 	template <class TARGET>
@@ -70,7 +70,7 @@ struct LocalTableFunctionState {
 
 	template <class TARGET>
 	TARGET &Cast() {
-		D_ASSERT(dynamic_cast<TARGET *>(this));
+		DynamicCastCheck<TARGET>(this);
 		return reinterpret_cast<TARGET &>(*this);
 	}
 	template <class TARGET>

package/src/duckdb/src/include/duckdb/main/attached_database.hpp

@@ -44,6 +44,7 @@ public:
 	~AttachedDatabase() override;
 
 	void Initialize();
+	void Close();
 
 	Catalog &ParentCatalog() override;
 	StorageManager &GetStorageManager();
@@ -61,6 +62,7 @@ public:
 	bool IsInitialDatabase() const;
 	void SetInitialDatabase();
 
+	static bool NameIsReserved(const string &name);
 	static string ExtractDatabaseName(const string &dbpath, FileSystem &fs);
 
 private:
@@ -71,6 +73,7 @@ private:
 	AttachedDatabaseType type;
 	optional_ptr<Catalog> parent_catalog;
 	bool is_initial_database = false;
+	bool is_closed = false;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/main/capi/cast/from_decimal.hpp

@@ -21,19 +21,21 @@ bool CastDecimalCInternal(duckdb_result *source, RESULT_TYPE &result, idx_t col,
 	auto width = duckdb::DecimalType::GetWidth(source_type);
 	auto scale = duckdb::DecimalType::GetScale(source_type);
 	void *source_address = UnsafeFetchPtr<hugeint_t>(source, col, row);
+
+	CastParameters parameters;
 	switch (source_type.InternalType()) {
 	case duckdb::PhysicalType::INT16:
 		return duckdb::TryCastFromDecimal::Operation<int16_t, RESULT_TYPE>(UnsafeFetchFromPtr<int16_t>(source_address),
-		                                                                   result, nullptr, width, scale);
+		                                                                   result, parameters, width, scale);
 	case duckdb::PhysicalType::INT32:
 		return duckdb::TryCastFromDecimal::Operation<int32_t, RESULT_TYPE>(UnsafeFetchFromPtr<int32_t>(source_address),
-		                                                                   result, nullptr, width, scale);
+		                                                                   result, parameters, width, scale);
 	case duckdb::PhysicalType::INT64:
 		return duckdb::TryCastFromDecimal::Operation<int64_t, RESULT_TYPE>(UnsafeFetchFromPtr<int64_t>(source_address),
-		                                                                   result, nullptr, width, scale);
+		                                                                   result, parameters, width, scale);
 	case duckdb::PhysicalType::INT128:
 		return duckdb::TryCastFromDecimal::Operation<hugeint_t, RESULT_TYPE>(
-		    UnsafeFetchFromPtr<hugeint_t>(source_address), result, nullptr, width, scale);
+		    UnsafeFetchFromPtr<hugeint_t>(source_address), result, parameters, width, scale);
 	default:
 		throw duckdb::InternalException("Unimplemented internal type for decimal");
 	}

package/src/duckdb/src/include/duckdb/main/capi/cast/to_decimal.hpp

@@ -15,7 +15,8 @@ namespace duckdb {
 template <class INTERNAL_TYPE>
 struct ToCDecimalCastWrapper {
 	template <class SOURCE_TYPE>
-	static bool Operation(SOURCE_TYPE input, duckdb_decimal &result, std::string *error, uint8_t width, uint8_t scale) {
+	static bool Operation(SOURCE_TYPE input, duckdb_decimal &result, CastParameters &parameters, uint8_t width,
+	                      uint8_t scale) {
 		throw NotImplementedException("Type not implemented for CDecimalCastWrapper");
 	}
 };
@@ -24,10 +25,12 @@ struct ToCDecimalCastWrapper {
 template <>
 struct ToCDecimalCastWrapper<hugeint_t> {
 	template <class SOURCE_TYPE>
-	static bool Operation(SOURCE_TYPE input, duckdb_decimal &result, std::string *error, uint8_t width, uint8_t scale) {
+	static bool Operation(SOURCE_TYPE input, duckdb_decimal &result, CastParameters &parameters, uint8_t width,
+	                      uint8_t scale) {
 		hugeint_t intermediate_result;
 
-		if (!TryCastToDecimal::Operation<SOURCE_TYPE, hugeint_t>(input, intermediate_result, error, width, scale)) {
+		if (!TryCastToDecimal::Operation<SOURCE_TYPE, hugeint_t>(input, intermediate_result, parameters, width,
+		                                                         scale)) {
 			result = FetchDefaultValue::Operation<duckdb_decimal>();
 			return false;
 		}
@@ -47,10 +50,11 @@ struct ToCDecimalCastWrapper<hugeint_t> {
 template <>
 struct ToCDecimalCastWrapper<int16_t> {
 	template <class SOURCE_TYPE>
-	static bool Operation(SOURCE_TYPE input, duckdb_decimal &result, std::string *error, uint8_t width, uint8_t scale) {
+	static bool Operation(SOURCE_TYPE input, duckdb_decimal &result, CastParameters &parameters, uint8_t width,
+	                      uint8_t scale) {
 		int16_t intermediate_result;
 
-		if (!TryCastToDecimal::Operation<SOURCE_TYPE, int16_t>(input, intermediate_result, error, width, scale)) {
+		if (!TryCastToDecimal::Operation<SOURCE_TYPE, int16_t>(input, intermediate_result, parameters, width, scale)) {
 			result = FetchDefaultValue::Operation<duckdb_decimal>();
 			return false;
 		}
@@ -70,10 +74,11 @@ struct ToCDecimalCastWrapper<int16_t> {
 template <>
 struct ToCDecimalCastWrapper<int32_t> {
 	template <class SOURCE_TYPE>
-	static bool Operation(SOURCE_TYPE input, duckdb_decimal &result, std::string *error, uint8_t width, uint8_t scale) {
+	static bool Operation(SOURCE_TYPE input, duckdb_decimal &result, CastParameters &parameters, uint8_t width,
+	                      uint8_t scale) {
 		int32_t intermediate_result;
 
-		if (!TryCastToDecimal::Operation<SOURCE_TYPE, int32_t>(input, intermediate_result, error, width, scale)) {
+		if (!TryCastToDecimal::Operation<SOURCE_TYPE, int32_t>(input, intermediate_result, parameters, width, scale)) {
 			result = FetchDefaultValue::Operation<duckdb_decimal>();
 			return false;
 		}
@@ -93,10 +98,11 @@ struct ToCDecimalCastWrapper<int32_t> {
 template <>
 struct ToCDecimalCastWrapper<int64_t> {
 	template <class SOURCE_TYPE>
-	static bool Operation(SOURCE_TYPE input, duckdb_decimal &result, std::string *error, uint8_t width, uint8_t scale) {
+	static bool Operation(SOURCE_TYPE input, duckdb_decimal &result, CastParameters &parameters, uint8_t width,
+	                      uint8_t scale) {
 		int64_t intermediate_result;
 
-		if (!TryCastToDecimal::Operation<SOURCE_TYPE, int64_t>(input, intermediate_result, error, width, scale)) {
+		if (!TryCastToDecimal::Operation<SOURCE_TYPE, int64_t>(input, intermediate_result, parameters, width, scale)) {
 			result = FetchDefaultValue::Operation<duckdb_decimal>();
 			return false;
 		}
@@ -117,7 +123,8 @@ template <class SOURCE_TYPE, class OP>
 duckdb_decimal TryCastToDecimalCInternal(SOURCE_TYPE source, uint8_t width, uint8_t scale) {
 	duckdb_decimal result;
 	try {
-		if (!OP::template Operation<SOURCE_TYPE>(source, result, nullptr, width, scale)) {
+		CastParameters parameters;
+		if (!OP::template Operation<SOURCE_TYPE>(source, result, parameters, width, scale)) {
 			return FetchDefaultValue::Operation<duckdb_decimal>();
 		}
 	} catch (...) {

package/src/duckdb/src/include/duckdb/main/client_config.hpp

@@ -87,6 +87,8 @@ struct ClientConfig {
 	idx_t perfect_ht_threshold = 12;
 	//! The maximum number of rows to accumulate before sorting ordered aggregates.
 	idx_t ordered_aggregate_threshold = (idx_t(1) << 18);
+	//! The number of rows to accumulate before flushing during a partitioned write
+	idx_t partitioned_write_flush_threshold = idx_t(1) << idx_t(19);
 
 	//! Callback to create a progress bar display
 	progress_bar_display_create_func_t display_create_func = nullptr;

package/src/duckdb/src/include/duckdb/main/client_context.hpp

@@ -24,8 +24,10 @@
 #include "duckdb/main/client_config.hpp"
 #include "duckdb/main/external_dependencies.hpp"
 #include "duckdb/common/error_data.hpp"
+#include "duckdb/common/enums/prepared_statement_mode.hpp"
 #include "duckdb/main/client_properties.hpp"
 #include "duckdb/main/client_context_state.hpp"
+#include "duckdb/main/settings.hpp"
 
 namespace duckdb {
 class Appender;
@@ -165,7 +167,7 @@ public:
 	                                                 bool requires_valid_transaction = true);
 
 	//! Equivalent to CURRENT_SETTING(key) SQL function.
-	DUCKDB_API bool TryGetCurrentSetting(const std::string &key, Value &result);
+	DUCKDB_API SettingLookupResult TryGetCurrentSetting(const std::string &key, Value &result);
 
 	//! Returns the parser options for this client context
 	DUCKDB_API ParserOptions GetParserOptions() const;
@@ -214,14 +216,19 @@ private:
 	                                                                   unique_ptr<SQLStatement> statement,
 	                                                                   shared_ptr<PreparedStatementData> &prepared,
 	                                                                   const PendingQueryParameters &parameters);
-	unique_ptr<PendingQueryResult> PendingPreparedStatement(ClientContextLock &lock,
+	unique_ptr<PendingQueryResult> PendingPreparedStatement(ClientContextLock &lock, const string &query,
 	                                                        shared_ptr<PreparedStatementData> statement_p,
 	                                                        const PendingQueryParameters &parameters);
+	unique_ptr<PendingQueryResult> PendingPreparedStatementInternal(ClientContextLock &lock,
+	                                                                shared_ptr<PreparedStatementData> statement_p,
+	                                                                const PendingQueryParameters &parameters);
+	void CheckIfPreparedStatementIsExecutable(PreparedStatementData &statement);
 
 	//! Internally prepare a SQL statement. Caller must hold the context_lock.
 	shared_ptr<PreparedStatementData>
 	CreatePreparedStatement(ClientContextLock &lock, const string &query, unique_ptr<SQLStatement> statement,
-	                        optional_ptr<case_insensitive_map_t<Value>> values = nullptr);
+	                        optional_ptr<case_insensitive_map_t<Value>> values = nullptr,
+	                        PreparedStatementMode mode = PreparedStatementMode::PREPARE_ONLY);
 	unique_ptr<PendingQueryResult> PendingStatementInternal(ClientContextLock &lock, const string &query,
 	                                                        unique_ptr<SQLStatement> statement,
 	                                                        const PendingQueryParameters &parameters);
@@ -251,9 +258,16 @@ private:
 	unique_ptr<PendingQueryResult> PendingQueryInternal(ClientContextLock &, const shared_ptr<Relation> &relation,
 	                                                    bool allow_stream_result);
 
+	void RebindPreparedStatement(ClientContextLock &lock, const string &query,
+	                             shared_ptr<PreparedStatementData> &prepared, const PendingQueryParameters &parameters);
+
 	template <class T>
 	unique_ptr<T> ErrorResult(ErrorData error, const string &query = string());
 
+	shared_ptr<PreparedStatementData>
+	CreatePreparedStatementInternal(ClientContextLock &lock, const string &query, unique_ptr<SQLStatement> statement,
+	                                optional_ptr<case_insensitive_map_t<Value>> values);
+
 private:
 	//! Lock on using the ClientContext in parallel
 	mutex context_lock;

package/src/duckdb/src/include/duckdb/main/client_context_file_opener.hpp

@@ -21,8 +21,8 @@ public:
 	explicit ClientContextFileOpener(ClientContext &context_p) : context(context_p) {
 	}
 
-	bool TryGetCurrentSetting(const string &key, Value &result, FileOpenerInfo &info) override;
-	bool TryGetCurrentSetting(const string &key, Value &result) override;
+	SettingLookupResult TryGetCurrentSetting(const string &key, Value &result, FileOpenerInfo &info) override;
+	SettingLookupResult TryGetCurrentSetting(const string &key, Value &result) override;
 
 	ClientContext *TryGetClientContext() override {
 		return &context;

package/src/duckdb/src/include/duckdb/main/client_context_state.hpp

@@ -9,10 +9,16 @@
 #pragma once
 
 #include "duckdb/common/common.hpp"
+#include "duckdb/common/enums/prepared_statement_mode.hpp"
 
 namespace duckdb {
 class ClientContext;
+class ErrorData;
 class MetaTransaction;
+class PreparedStatementData;
+class SQLStatement;
+
+enum class RebindQueryInfo { DO_NOT_REBIND, ATTEMPT_TO_REBIND };
 
 //! ClientContextState is virtual base class for ClientContext-local (or Query-Local, using QueryEnd callback) state
 //! e.g. caches that need to live as long as a ClientContext or Query.
@@ -32,6 +38,20 @@ public:
 	}
 	virtual void TransactionRollback(MetaTransaction &transaction, ClientContext &context) {
 	}
+	virtual bool CanRequestRebind() {
+		return false;
+	}
+	virtual RebindQueryInfo OnPlanningError(ClientContext &context, SQLStatement &statement, ErrorData &error) {
+		return RebindQueryInfo::DO_NOT_REBIND;
+	}
+	virtual RebindQueryInfo OnFinalizePrepare(ClientContext &context, PreparedStatementData &prepared_statement,
+	                                          PreparedStatementMode mode) {
+		return RebindQueryInfo::DO_NOT_REBIND;
+	}
+	virtual RebindQueryInfo OnExecutePrepared(ClientContext &context, PreparedStatementData &prepared_statement,
+	                                          RebindQueryInfo current_rebind) {
+		return RebindQueryInfo::DO_NOT_REBIND;
+	}
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/main/client_data.hpp

@@ -13,7 +13,7 @@
 #include "duckdb/common/types/value.hpp"
 #include "duckdb/common/case_insensitive_map.hpp"
 #include "duckdb/common/atomic.hpp"
-#include "duckdb/execution/operator/csv_scanner/state_machine/csv_state_machine_cache.hpp"
+#include "duckdb/execution/operator/csv_scanner/csv_state_machine_cache.hpp"
 
 namespace duckdb {
 class AttachedDatabase;

package/src/duckdb/src/include/duckdb/main/database.hpp

@@ -12,6 +12,7 @@
 #include "duckdb/main/valid_checker.hpp"
 #include "duckdb/common/winapi.hpp"
 #include "duckdb/main/extension.hpp"
+#include "duckdb/main/settings.hpp"
 
 namespace duckdb {
 class BufferManager;
@@ -51,9 +52,9 @@ public:
 	DUCKDB_API static DatabaseInstance &GetDatabase(ClientContext &context);
 
 	DUCKDB_API const unordered_set<std::string> &LoadedExtensions();
-	DUCKDB_API bool ExtensionIsLoaded(const std::string &name);
+	DUCKDB_API bool ExtensionIsLoaded(const string &name);
 
-	DUCKDB_API bool TryGetCurrentSetting(const std::string &key, Value &result);
+	DUCKDB_API SettingLookupResult TryGetCurrentSetting(const string &key, Value &result);
 
 	unique_ptr<AttachedDatabase> CreateAttachedDatabase(ClientContext &context, const AttachInfo &info,
 	                                                    const string &type, AccessMode access_mode);

package/src/duckdb/src/include/duckdb/main/database_manager.hpp

@@ -24,6 +24,7 @@ class Catalog;
 class CatalogSet;
 class ClientContext;
 class DatabaseInstance;
+class TaskScheduler;
 
 //! The DatabaseManager is a class that sits at the root of all attached databases
 class DatabaseManager {
@@ -67,7 +68,7 @@ public:
 	vector<reference<AttachedDatabase>> GetDatabases(ClientContext &context);
 	//! Removes all databases from the catalog set. This is necessary for the database instance's destructor,
 	//! as the database manager has to be alive when destroying the catalog set objects.
-	void ResetDatabases();
+	void ResetDatabases(unique_ptr<TaskScheduler> &scheduler);
 
 	transaction_t GetNewQueryNumber() {
 		return current_query_number++;

package/src/duckdb/src/include/duckdb/main/extension/generated_extension_loader.hpp

@@ -12,7 +12,7 @@
 #include "duckdb/common/vector.hpp"
 #include "duckdb/main/database.hpp"
 
-#if defined(GENERATED_EXTENSION_HEADERS) and !defined(DUCKDB_AMALGAMATION)
+#if defined(GENERATED_EXTENSION_HEADERS) && !defined(DUCKDB_AMALGAMATION)
 #include "duckdb/common/common.hpp"
 #include "generated_extension_headers.hpp"