duckdb 0.8.1-dev65.0 → 0.8.1

package/src/duckdb/src/include/duckdb/function/cast/vector_cast_helpers.hpp

@@ -201,15 +201,15 @@ struct VectorCastHelpers {
 struct VectorStringToList {
 	static idx_t CountPartsList(const string_t &input);
 	static bool SplitStringList(const string_t &input, string_t *child_data, idx_t &child_start, Vector &child);
-	static bool StringToNestedTypeCastLoop(string_t *source_data, ValidityMask &source_mask, Vector &result,
+	static bool StringToNestedTypeCastLoop(const string_t *source_data, ValidityMask &source_mask, Vector &result,
 	                                       ValidityMask &result_mask, idx_t count, CastParameters &parameters,
 	                                       const SelectionVector *sel);
 };
 
 struct VectorStringToStruct {
-	static bool SplitStruct(string_t &input, vector<unique_ptr<Vector>> &varchar_vectors, idx_t &row_idx,
+	static bool SplitStruct(const string_t &input, vector<unique_ptr<Vector>> &varchar_vectors, idx_t &row_idx,
 	                        string_map_t<idx_t> &child_names, vector<ValidityMask *> &child_masks);
-	static bool StringToNestedTypeCastLoop(string_t *source_data, ValidityMask &source_mask, Vector &result,
+	static bool StringToNestedTypeCastLoop(const string_t *source_data, ValidityMask &source_mask, Vector &result,
 	                                       ValidityMask &result_mask, idx_t count, CastParameters &parameters,
 	                                       const SelectionVector *sel);
 };
@@ -218,7 +218,7 @@ struct VectorStringToMap {
 	static idx_t CountPartsMap(const string_t &input);
 	static bool SplitStringMap(const string_t &input, string_t *child_key_data, string_t *child_val_data,
 	                           idx_t &child_start, Vector &varchar_key, Vector &varchar_val);
-	static bool StringToNestedTypeCastLoop(string_t *source_data, ValidityMask &source_mask, Vector &result,
+	static bool StringToNestedTypeCastLoop(const string_t *source_data, ValidityMask &source_mask, Vector &result,
 	                                       ValidityMask &result_mask, idx_t count, CastParameters &parameters,
 	                                       const SelectionVector *sel);
 };

package/src/duckdb/src/include/duckdb/function/compression_function.hpp

@@ -29,16 +29,49 @@ struct SegmentScanState;
 struct AnalyzeState {
 	virtual ~AnalyzeState() {
 	}
+
+	template <class TARGET>
+	TARGET &Cast() {
+		D_ASSERT(dynamic_cast<TARGET *>(this));
+		return reinterpret_cast<TARGET &>(*this);
+	}
+	template <class TARGET>
+	const TARGET &Cast() const {
+		D_ASSERT(dynamic_cast<const TARGET *>(this));
+		return reinterpret_cast<const TARGET &>(*this);
+	}
 };
 
 struct CompressionState {
 	virtual ~CompressionState() {
 	}
+
+	template <class TARGET>
+	TARGET &Cast() {
+		D_ASSERT(dynamic_cast<TARGET *>(this));
+		return reinterpret_cast<TARGET &>(*this);
+	}
+	template <class TARGET>
+	const TARGET &Cast() const {
+		D_ASSERT(dynamic_cast<const TARGET *>(this));
+		return reinterpret_cast<const TARGET &>(*this);
+	}
 };
 
 struct CompressedSegmentState {
 	virtual ~CompressedSegmentState() {
 	}
+
+	template <class TARGET>
+	TARGET &Cast() {
+		D_ASSERT(dynamic_cast<TARGET *>(this));
+		return reinterpret_cast<TARGET &>(*this);
+	}
+	template <class TARGET>
+	const TARGET &Cast() const {
+		D_ASSERT(dynamic_cast<const TARGET *>(this));
+		return reinterpret_cast<const TARGET &>(*this);
+	}
 };
 
 struct CompressionAppendState {
@@ -48,6 +81,17 @@ struct CompressionAppendState {
 	}
 
 	BufferHandle handle;
+
+	template <class TARGET>
+	TARGET &Cast() {
+		D_ASSERT(dynamic_cast<TARGET *>(this));
+		return reinterpret_cast<TARGET &>(*this);
+	}
+	template <class TARGET>
+	const TARGET &Cast() const {
+		D_ASSERT(dynamic_cast<const TARGET *>(this));
+		return reinterpret_cast<const TARGET &>(*this);
+	}
 };
 
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/include/duckdb/function/copy_function.hpp

@@ -27,12 +27,12 @@ struct LocalFunctionData {
 	template <class TARGET>
 	TARGET &Cast() {
 		D_ASSERT(dynamic_cast<TARGET *>(this));
-		return (TARGET &)*this;
+		return reinterpret_cast<TARGET &>(*this);
 	}
 	template <class TARGET>
 	const TARGET &Cast() const {
 		D_ASSERT(dynamic_cast<const TARGET *>(this));
-		return (const TARGET &)*this;
+		return reinterpret_cast<const TARGET &>(*this);
 	}
 };
 
@@ -43,12 +43,12 @@ struct GlobalFunctionData {
 	template <class TARGET>
 	TARGET &Cast() {
 		D_ASSERT(dynamic_cast<TARGET *>(this));
-		return (TARGET &)*this;
+		return reinterpret_cast<TARGET &>(*this);
 	}
 	template <class TARGET>
 	const TARGET &Cast() const {
 		D_ASSERT(dynamic_cast<const TARGET *>(this));
-		return (const TARGET &)*this;
+		return reinterpret_cast<const TARGET &>(*this);
 	}
 };
 
@@ -59,12 +59,12 @@ struct PreparedBatchData {
 	template <class TARGET>
 	TARGET &Cast() {
 		D_ASSERT(dynamic_cast<TARGET *>(this));
-		return (TARGET &)*this;
+		return reinterpret_cast<TARGET &>(*this);
 	}
 	template <class TARGET>
 	const TARGET &Cast() const {
 		D_ASSERT(dynamic_cast<const TARGET *>(this));
-		return (const TARGET &)*this;
+		return reinterpret_cast<const TARGET &>(*this);
 	}
 };
 

package/src/duckdb/src/include/duckdb/function/function.hpp

@@ -49,17 +49,17 @@ struct FunctionData {
 	template <class TARGET>
 	TARGET &Cast() {
 		D_ASSERT(dynamic_cast<TARGET *>(this));
-		return (TARGET &)*this;
+		return reinterpret_cast<TARGET &>(*this);
 	}
 	template <class TARGET>
 	const TARGET &Cast() const {
 		D_ASSERT(dynamic_cast<const TARGET *>(this));
-		return (const TARGET &)*this;
+		return reinterpret_cast<const TARGET &>(*this);
 	}
 	// FIXME: this function should be removed in the future
 	template <class TARGET>
 	TARGET &CastNoConst() const {
-		return (TARGET &)*this;
+		return const_cast<TARGET &>(reinterpret_cast<const TARGET &>(*this));
 	}
 };
 

package/src/duckdb/src/include/duckdb/function/function_serialization.hpp

@@ -25,8 +25,10 @@ public:
 		bool serialize = function.serialize;
 		writer.WriteField(serialize);
 		if (serialize) {
-			D_ASSERT(function.deserialize);
 			function.serialize(writer, bind_info, function);
+			// First check if serialize throws a NotImplementedException, in which case it doesn't require a deserialize
+			// function
+			D_ASSERT(function.deserialize);
 		}
 	}
 
@@ -64,7 +66,7 @@ public:
 				throw SerializationException("Function requires deserialization but no deserialization function for %s",
 				                             function.name);
 			}
-			bind_info = function.deserialize(context, reader, function);
+			bind_info = function.deserialize(state, reader, function);
 		} else {
 			D_ASSERT(!function.serialize);
 			D_ASSERT(!function.deserialize);

package/src/duckdb/src/include/duckdb/function/macro_function.hpp

@@ -57,7 +57,7 @@ public:
 		if (type != TARGET::TYPE) {
 			throw InternalException("Failed to cast macro to type - macro type mismatch");
 		}
-		return (TARGET &)*this;
+		return reinterpret_cast<TARGET &>(*this);
 	}
 
 	template <class TARGET>
@@ -65,7 +65,7 @@ public:
 		if (type != TARGET::TYPE) {
 			throw InternalException("Failed to cast macro to type - macro type mismatch");
 		}
-		return (const TARGET &)*this;
+		return reinterpret_cast<const TARGET &>(*this);
 	}
 };
 

package/src/duckdb/src/include/duckdb/function/scalar/list/contains_or_position.hpp

@@ -27,14 +27,14 @@ static void TemplatedContainsOrPosition(DataChunk &args, Vector &result, bool is
 
 	UnifiedVectorFormat list_data;
 	list.ToUnifiedFormat(count, list_data);
-	auto list_entries = (list_entry_t *)list_data.data;
+	auto list_entries = UnifiedVectorFormat::GetData<list_entry_t>(list_data);
 
 	UnifiedVectorFormat value_data;
 	value_vector.ToUnifiedFormat(count, value_data);
 
 	// not required for a comparison of nested types
-	auto child_value = (CHILD_TYPE *)child_data.data;
-	auto values = (CHILD_TYPE *)value_data.data;
+	auto child_value = UnifiedVectorFormat::GetData<CHILD_TYPE>(child_data);
+	auto values = UnifiedVectorFormat::GetData<CHILD_TYPE>(value_data);
 
 	for (idx_t i = 0; i < count; i++) {
 		auto list_index = list_data.sel->get_index(i);

package/src/duckdb/src/include/duckdb/function/scalar/nested_functions.hpp

@@ -68,7 +68,7 @@ struct VariableReturnBindData : public FunctionData {
 		writer.WriteSerializable(info.stype);
 	}
 
-	static unique_ptr<FunctionData> Deserialize(ClientContext &context, FieldReader &reader,
+	static unique_ptr<FunctionData> Deserialize(PlanDeserializationState &context, FieldReader &reader,
 	                                            ScalarFunction &bound_function) {
 		auto stype = reader.ReadRequiredSerializable<LogicalType, LogicalType>();
 		return make_uniq<VariableReturnBindData>(std::move(stype));

package/src/duckdb/src/include/duckdb/function/scalar_function.hpp

@@ -14,6 +14,7 @@
 #include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/execution/expression_executor_state.hpp"
 #include "duckdb/function/function.hpp"
+#include "duckdb/planner/plan_serialization.hpp"
 #include "duckdb/storage/statistics/base_statistics.hpp"
 #include "duckdb/common/optional_ptr.hpp"
 
@@ -25,12 +26,12 @@ struct FunctionLocalState {
 	template <class TARGET>
 	TARGET &Cast() {
 		D_ASSERT(dynamic_cast<TARGET *>(this));
-		return (TARGET &)*this;
+		return reinterpret_cast<TARGET &>(*this);
 	}
 	template <class TARGET>
 	const TARGET &Cast() const {
 		D_ASSERT(dynamic_cast<const TARGET *>(this));
-		return (const TARGET &)*this;
+		return reinterpret_cast<const TARGET &>(*this);
 	}
 };
 
@@ -65,7 +66,7 @@ typedef void (*dependency_function_t)(BoundFunctionExpression &expr, DependencyL
 
 typedef void (*function_serialize_t)(FieldWriter &writer, const FunctionData *bind_data,
                                      const ScalarFunction &function);
-typedef unique_ptr<FunctionData> (*function_deserialize_t)(ClientContext &context, FieldReader &reader,
+typedef unique_ptr<FunctionData> (*function_deserialize_t)(PlanDeserializationState &state, FieldReader &reader,
                                                            ScalarFunction &function);
 
 class ScalarFunction : public BaseScalarFunction {

package/src/duckdb/src/include/duckdb/function/table_function.hpp

@@ -9,12 +9,12 @@
 #pragma once
 
 #include "duckdb/common/enums/operator_result_type.hpp"
+#include "duckdb/common/optional_ptr.hpp"
 #include "duckdb/execution/execution_context.hpp"
 #include "duckdb/function/function.hpp"
 #include "duckdb/planner/bind_context.hpp"
 #include "duckdb/planner/logical_operator.hpp"
 #include "duckdb/storage/statistics/node_statistics.hpp"
-#include "duckdb/common/optional_ptr.hpp"
 
 #include <functional>
 
@@ -31,12 +31,12 @@ struct TableFunctionInfo {
 	template <class TARGET>
 	TARGET &Cast() {
 		D_ASSERT(dynamic_cast<TARGET *>(this));
-		return (TARGET &)*this;
+		return reinterpret_cast<TARGET &>(*this);
 	}
 	template <class TARGET>
 	const TARGET &Cast() const {
 		D_ASSERT(dynamic_cast<const TARGET *>(this));
-		return (const TARGET &)*this;
+		return reinterpret_cast<const TARGET &>(*this);
 	}
 };
 
@@ -55,12 +55,12 @@ public:
 	template <class TARGET>
 	TARGET &Cast() {
 		D_ASSERT(dynamic_cast<TARGET *>(this));
-		return (TARGET &)*this;
+		return reinterpret_cast<TARGET &>(*this);
 	}
 	template <class TARGET>
 	const TARGET &Cast() const {
 		D_ASSERT(dynamic_cast<const TARGET *>(this));
-		return (const TARGET &)*this;
+		return reinterpret_cast<const TARGET &>(*this);
 	}
 };
 
@@ -70,12 +70,12 @@ struct LocalTableFunctionState {
 	template <class TARGET>
 	TARGET &Cast() {
 		D_ASSERT(dynamic_cast<TARGET *>(this));
-		return (TARGET &)*this;
+		return reinterpret_cast<TARGET &>(*this);
 	}
 	template <class TARGET>
 	const TARGET &Cast() const {
 		D_ASSERT(dynamic_cast<const TARGET *>(this));
-		return (const TARGET &)*this;
+		return reinterpret_cast<const TARGET &>(*this);
 	}
 };
 
@@ -204,7 +204,7 @@ typedef string (*table_function_to_string_t)(const FunctionData *bind_data);
 
 typedef void (*table_function_serialize_t)(FieldWriter &writer, const FunctionData *bind_data,
                                            const TableFunction &function);
-typedef unique_ptr<FunctionData> (*table_function_deserialize_t)(ClientContext &context, FieldReader &reader,
+typedef unique_ptr<FunctionData> (*table_function_deserialize_t)(PlanDeserializationState &context, FieldReader &reader,
                                                                  TableFunction &function);
 
 class TableFunction : public SimpleNamedParameterFunction {
@@ -265,6 +265,7 @@ public:
 
 	table_function_serialize_t serialize;
 	table_function_deserialize_t deserialize;
+	bool verify_serialization = true;
 
 	//! Whether or not the table function supports projection pushdown. If not supported a projection will be added
 	//! that filters out unused columns.

package/src/duckdb/src/include/duckdb/main/attached_database.hpp

@@ -40,7 +40,7 @@ public:
 	//! Create an attached database instance with the specified storage extension
 	AttachedDatabase(DatabaseInstance &db, Catalog &catalog, StorageExtension &ext, string name, AttachInfo &info,
 	                 AccessMode access_mode);
-	~AttachedDatabase();
+	~AttachedDatabase() override;
 
 	void Initialize();
 
@@ -57,6 +57,8 @@ public:
 	bool IsSystem() const;
 	bool IsTemporary() const;
 	bool IsReadOnly() const;
+	bool IsInitialDatabase() const;
+	void SetInitialDatabase();
 
 	static string ExtractDatabaseName(const string &dbpath);
 
@@ -67,6 +69,7 @@ private:
 	unique_ptr<TransactionManager> transaction_manager;
 	AttachedDatabaseType type;
 	optional_ptr<Catalog> parent_catalog;
+	bool is_initial_database = false;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/main/capi/capi_internal.hpp

@@ -13,6 +13,8 @@
 #include "duckdb/common/types.hpp"
 #include "duckdb/common/types/data_chunk.hpp"
 #include "duckdb/main/appender.hpp"
+#include "duckdb/common/arrow/arrow_options.hpp"
+
 #include <cstring>
 #include <cassert>
 
@@ -46,7 +48,7 @@ struct PendingStatementWrapper {
 struct ArrowResultWrapper {
 	unique_ptr<MaterializedQueryResult> result;
 	unique_ptr<DataChunk> current_chunk;
-	string timezone_config;
+	ArrowOptions options;
 };
 
 struct AppenderWrapper {

package/src/duckdb/src/include/duckdb/main/capi/cast/utils.hpp

@@ -82,7 +82,7 @@ struct ToCStringCastWrapper {
 		auto result_size = result_string.GetSize();
 		auto result_data = result_string.GetData();
 
-		char *allocated_data = (char *)duckdb_malloc(result_size + 1);
+		char *allocated_data = char_ptr_cast(duckdb_malloc(result_size + 1));
 		memcpy(allocated_data, result_data, result_size);
 		allocated_data[result_size] = '\0';
 		result.data = allocated_data;

package/src/duckdb/src/include/duckdb/main/config.hpp

@@ -28,6 +28,7 @@
 #include "duckdb/optimizer/optimizer_extension.hpp"
 #include "duckdb/parser/parser_extension.hpp"
 #include "duckdb/planner/operator_extension.hpp"
+#include "duckdb/common/arrow/arrow_options.hpp"
 
 namespace duckdb {
 class BufferPool;
@@ -135,6 +136,8 @@ struct DBConfigOptions {
 	WindowAggregationMode window_mode = WindowAggregationMode::WINDOW;
 	//! Whether or not preserving insertion order should be preserved
 	bool preserve_insertion_order = true;
+	//! Whether Arrow Arrays use Large or Regular buffers
+	ArrowOffsetSize arrow_offset_size = ArrowOffsetSize::REGULAR;
 	//! Database configuration variables as controlled by SET
 	case_insensitive_map_t<Value> set_variables;
 	//! Database configuration variable default values;
@@ -151,6 +154,8 @@ struct DBConfigOptions {
 	DebugInitialize debug_initialize = DebugInitialize::NO_INITIALIZE;
 	//! The set of unrecognized (other) options
 	unordered_map<string, Value> unrecognized_options;
+	//! Whether or not the configuration settings can be altered
+	bool lock_configuration = false;
 	//! Whether to print bindings when printing the plan (debug mode only)
 	static bool debug_print_bindings;
 

package/src/duckdb/src/include/duckdb/main/database_manager.hpp

@@ -44,6 +44,7 @@ public:
 	//! Returns a reference to the system catalog
 	Catalog &GetSystemCatalog();
 	static const string &GetDefaultDatabase(ClientContext &context);
+	void SetDefaultDatabase(ClientContext &context, const string &new_value);
 
 	optional_ptr<AttachedDatabase> GetDatabaseFromPath(ClientContext &context, const string &path);
 	vector<reference<AttachedDatabase>> GetDatabases(ClientContext &context);