duckdb 1.0.1-dev22.0 → 1.0.1-dev27.0

package/src/duckdb/src/include/duckdb/common/arrow/appender/scalar_data.hpp

@@ -3,6 +3,8 @@
 #include "duckdb/common/arrow/appender/append_data.hpp"
 #include "duckdb/function/table/arrow.hpp"
 
+#include "duckdb/common/bswap.hpp"
+
 namespace duckdb {
 
 //===--------------------------------------------------------------------===//
@@ -57,6 +59,25 @@ struct ArrowTimeTzConverter {
 	}
 };
 
+struct ArrowUUIDBlobConverter {
+	template <class TGT, class SRC>
+	static TGT Operation(hugeint_t input) {
+		// Turn into big-end
+		auto upper = BSwap(input.lower);
+		// flip Upper MSD
+		auto lower = BSwap(static_cast<int64_t>(static_cast<uint64_t>(input.upper) ^ (static_cast<uint64_t>(1) << 63)));
+		return {static_cast<int64_t>(upper), static_cast<uint64_t>(lower)};
+	}
+
+	static bool SkipNulls() {
+		return true;
+	}
+
+	template <class TGT>
+	static void SetNull(TGT &value) {
+	}
+};
+
 template <class TGT, class SRC = TGT, class OP = ArrowScalarConverter>
 struct ArrowScalarBaseData {
 	static void Append(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size) {
@@ -70,9 +91,10 @@ struct ArrowScalarBaseData {
 		AppendValidity(append_data, format, from, to);
 
 		// append the main data
-		append_data.main_buffer.resize(append_data.main_buffer.size() + sizeof(TGT) * size);
+		auto &main_buffer = append_data.GetMainBuffer();
+		main_buffer.resize(main_buffer.size() + sizeof(TGT) * size);
 		auto data = UnifiedVectorFormat::GetData<SRC>(format);
-		auto result_data = append_data.main_buffer.GetData<TGT>();
+		auto result_data = main_buffer.GetData<TGT>();
 
 		for (idx_t i = from; i < to; i++) {
 			auto source_idx = format.sel->get_index(i);
@@ -91,12 +113,12 @@ struct ArrowScalarBaseData {
 template <class TGT, class SRC = TGT, class OP = ArrowScalarConverter>
 struct ArrowScalarData : public ArrowScalarBaseData<TGT, SRC, OP> {
 	static void Initialize(ArrowAppendData &result, const LogicalType &type, idx_t capacity) {
-		result.main_buffer.reserve(capacity * sizeof(TGT));
+		result.GetMainBuffer().reserve(capacity * sizeof(TGT));
 	}
 
 	static void Finalize(ArrowAppendData &append_data, const LogicalType &type, ArrowArray *result) {
 		result->n_buffers = 2;
-		result->buffers[1] = append_data.main_buffer.data();
+		result->buffers[1] = append_data.GetMainBuffer().data();
 	}
 };
 

package/src/duckdb/src/include/duckdb/common/arrow/appender/varchar_data.hpp

@@ -2,6 +2,8 @@
 
 #include "duckdb/common/arrow/appender/append_data.hpp"
 #include "duckdb/common/arrow/appender/scalar_data.hpp"
+#include "duckdb/common/types/arrow_string_view_type.hpp"
+#include "duckdb/common/types/uuid.hpp"
 
 namespace duckdb {
 
@@ -35,9 +37,8 @@ struct ArrowUUIDConverter {
 template <class SRC = string_t, class OP = ArrowVarcharConverter, class BUFTYPE = int64_t>
 struct ArrowVarcharData {
 	static void Initialize(ArrowAppendData &result, const LogicalType &type, idx_t capacity) {
-		result.main_buffer.reserve((capacity + 1) * sizeof(BUFTYPE));
-
-		result.aux_buffer.reserve(capacity);
+		result.GetMainBuffer().reserve((capacity + 1) * sizeof(BUFTYPE));
+		result.GetAuxBuffer().reserve(capacity);
 	}
 
 	template <bool LARGE_STRING>
@@ -45,15 +46,18 @@ struct ArrowVarcharData {
 		idx_t size = to - from;
 		UnifiedVectorFormat format;
 		input.ToUnifiedFormat(input_size, format);
+		auto &main_buffer = append_data.GetMainBuffer();
+		auto &validity_buffer = append_data.GetValidityBuffer();
+		auto &aux_buffer = append_data.GetAuxBuffer();
 
 		// resize the validity mask and set up the validity buffer for iteration
-		ResizeValidity(append_data.validity, append_data.row_count + size);
-		auto validity_data = (uint8_t *)append_data.validity.data();
+		ResizeValidity(validity_buffer, append_data.row_count + size);
+		auto validity_data = (uint8_t *)validity_buffer.data();
 
 		// resize the offset buffer - the offset buffer holds the offsets into the child array
-		append_data.main_buffer.resize(append_data.main_buffer.size() + sizeof(BUFTYPE) * (size + 1));
+		main_buffer.resize(main_buffer.size() + sizeof(BUFTYPE) * (size + 1));
 		auto data = UnifiedVectorFormat::GetData<SRC>(format);
-		auto offset_data = append_data.main_buffer.GetData<BUFTYPE>();
+		auto offset_data = main_buffer.GetData<BUFTYPE>();
 		if (append_data.row_count == 0) {
 			// first entry
 			offset_data[0] = 0;
@@ -89,8 +93,8 @@ struct ArrowVarcharData {
 			offset_data[offset_idx] = UnsafeNumericCast<BUFTYPE>(current_offset);
 
 			// resize the string buffer if required, and write the string data
-			append_data.aux_buffer.resize(current_offset);
-			OP::WriteData(append_data.aux_buffer.data() + last_offset, data[source_idx]);
+			aux_buffer.resize(current_offset);
+			OP::WriteData(aux_buffer.data() + last_offset, data[source_idx]);
 
 			last_offset = UnsafeNumericCast<BUFTYPE>(current_offset);
 		}
@@ -108,8 +112,83 @@ struct ArrowVarcharData {
 
 	static void Finalize(ArrowAppendData &append_data, const LogicalType &type, ArrowArray *result) {
 		result->n_buffers = 3;
-		result->buffers[1] = append_data.main_buffer.data();
-		result->buffers[2] = append_data.aux_buffer.data();
+		result->buffers[1] = append_data.GetMainBuffer().data();
+		result->buffers[2] = append_data.GetAuxBuffer().data();
+	}
+};
+
+struct ArrowVarcharToStringViewData {
+	static void Initialize(ArrowAppendData &result, const LogicalType &type, idx_t capacity) {
+		result.GetMainBuffer().reserve((capacity) * sizeof(arrow_string_view_t));
+		result.GetAuxBuffer().reserve(capacity);
+		result.GetBufferSizeBuffer().reserve(sizeof(int64_t));
+	}
+
+	static void Append(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size) {
+		idx_t size = to - from;
+		UnifiedVectorFormat format;
+		input.ToUnifiedFormat(input_size, format);
+		auto &main_buffer = append_data.GetMainBuffer();
+		auto &validity_buffer = append_data.GetValidityBuffer();
+		auto &aux_buffer = append_data.GetAuxBuffer();
+		// resize the validity mask and set up the validity buffer for iteration
+		ResizeValidity(validity_buffer, append_data.row_count + size);
+		auto validity_data = (uint8_t *)validity_buffer.data();
+
+		main_buffer.resize(main_buffer.size() + sizeof(arrow_string_view_t) * (size));
+		// resize the offset buffer - the offset buffer holds the offsets into the child array
+		auto data = UnifiedVectorFormat::GetData<string_t>(format);
+		for (idx_t i = from; i < to; i++) {
+			auto result_idx = append_data.row_count + i - from;
+			auto arrow_data = main_buffer.GetData<arrow_string_view_t>();
+			auto source_idx = format.sel->get_index(i);
+			if (!format.validity.RowIsValid(source_idx)) {
+				// Null value
+				uint8_t current_bit;
+				idx_t current_byte;
+				GetBitPosition(result_idx, current_byte, current_bit);
+				SetNull(append_data, validity_data, current_byte, current_bit);
+				// We have to set these bytes to 0, for some reason
+				arrow_data[result_idx] = arrow_string_view_t(0, "");
+				continue;
+			}
+			// These two are now the same buffer
+			idx_t string_length = ArrowVarcharConverter::GetLength(data[source_idx]);
+			auto string_data = data[source_idx].GetData();
+			if (string_length <= ArrowStringViewConstants::MAX_INLINED_BYTES) {
+				//	This string is inlined
+				//  | Bytes 0-3  | Bytes 4-15                            |
+				//  |------------|---------------------------------------|
+				//  | length     | data (padded with 0)                  |
+				arrow_data[result_idx] = arrow_string_view_t(UnsafeNumericCast<int32_t>(string_length), string_data);
+			} else {
+				// This string is not inlined, we have to check a different buffer and offsets
+				//  | Bytes 0-3  | Bytes 4-7  | Bytes 8-11 | Bytes 12-15 |
+				//  |------------|------------|------------|-------------|
+				//  | length     | prefix     | buf. index | offset      |
+				arrow_data[result_idx] = arrow_string_view_t(UnsafeNumericCast<int32_t>(string_length), string_data, 0,
+				                                             UnsafeNumericCast<int32_t>(append_data.offset));
+				auto current_offset = append_data.offset + string_length;
+				aux_buffer.resize(current_offset);
+				ArrowVarcharConverter::WriteData(aux_buffer.data() + append_data.offset, data[source_idx]);
+				append_data.offset = current_offset;
+			}
+		}
+		append_data.row_count += size;
+	}
+
+	static void Finalize(ArrowAppendData &append_data, const LogicalType &type, ArrowArray *result) {
+		// We output four buffers
+		result->n_buffers = 4;
+		// Buffer 0 is the validity mask
+		// Buffer 1 is our string views (short/long strings)
+		result->buffers[1] = append_data.GetMainBuffer().data();
+		// Buffer 2 is our only data buffer, could theoretically be more [ buffers ]
+		result->buffers[2] = append_data.GetAuxBuffer().data();
+		// Buffer 3 is the data-buffer lengths buffer, and we also populate it in to finalize
+		reinterpret_cast<int64_t *>(append_data.GetBufferSizeBuffer().data())[0] =
+		    UnsafeNumericCast<int64_t>(append_data.offset);
+		result->buffers[3] = append_data.GetBufferSizeBuffer().data();
 	}
 };
 

package/src/duckdb/src/include/duckdb/common/arrow/arrow_appender.hpp

@@ -18,20 +18,20 @@ struct ArrowAppendData;
 //! The ArrowAppender class can be used to incrementally construct an arrow array by appending data chunks into it
 class ArrowAppender {
 public:
-	DUCKDB_API ArrowAppender(vector<LogicalType> types, idx_t initial_capacity, ClientProperties options);
+	DUCKDB_API ArrowAppender(vector<LogicalType> types_p, const idx_t initial_capacity, ClientProperties options);
 	DUCKDB_API ~ArrowAppender();
 
+public:
 	//! Append a data chunk to the underlying arrow array
 	DUCKDB_API void Append(DataChunk &input, idx_t from, idx_t to, idx_t input_size);
 	//! Returns the underlying arrow array
 	DUCKDB_API ArrowArray Finalize();
-
-public:
+	idx_t RowCount() const;
 	static void ReleaseArray(ArrowArray *array);
-	static ArrowArray *FinalizeChild(const LogicalType &type, unique_ptr<ArrowAppendData> append_data);
-	static unique_ptr<ArrowAppendData> InitializeChild(const LogicalType &type, idx_t capacity,
+	static ArrowArray *FinalizeChild(const LogicalType &type, unique_ptr<ArrowAppendData> append_data_p);
+	static unique_ptr<ArrowAppendData> InitializeChild(const LogicalType &type, const idx_t capacity,
 	                                                   ClientProperties &options);
-	static void AddChildren(ArrowAppendData &data, idx_t count);
+	static void AddChildren(ArrowAppendData &data, const idx_t count);
 
 private:
 	//! The types of the chunks that will be appended in

package/src/duckdb/src/include/duckdb/common/arrow/arrow_buffer.hpp

@@ -32,7 +32,7 @@ struct ArrowBuffer {
 	ArrowBuffer(const ArrowBuffer &other) = delete;
 	ArrowBuffer &operator=(const ArrowBuffer &) = delete;
 	//! enable move constructors
-	ArrowBuffer(ArrowBuffer &&other) noexcept {
+	ArrowBuffer(ArrowBuffer &&other) noexcept : count(0), capacity(0) {
 		std::swap(dataptr, other.dataptr);
 		std::swap(count, other.count);
 		std::swap(capacity, other.capacity);
@@ -65,6 +65,13 @@ struct ArrowBuffer {
 		count = bytes;
 	}
 
+	template <class T>
+	void push_back(T value) {
+		reserve(sizeof(T) * (count + 1));
+		reinterpret_cast<T *>(dataptr)[count] = value;
+		count++;
+	}
+
 	idx_t size() { // NOLINT
 		return count;
 	}

package/src/duckdb/src/include/duckdb/common/arrow/arrow_merge_event.hpp

@@ -0,0 +1,62 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/arrow/arrow_merge_event.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/parallel/base_pipeline_event.hpp"
+#include "duckdb/parallel/task.hpp"
+#include "duckdb/common/types/batched_data_collection.hpp"
+#include "duckdb/parallel/pipeline.hpp"
+#include "duckdb/common/vector.hpp"
+#include "duckdb/common/shared_ptr.hpp"
+#include "duckdb/common/arrow/arrow_converter.hpp"
+#include "duckdb/storage/buffer_manager.hpp"
+#include "duckdb/main/chunk_scan_state/batched_data_collection.hpp"
+#include "duckdb/execution/executor.hpp"
+#include "duckdb/main/client_context.hpp"
+#include "duckdb/common/unique_ptr.hpp"
+#include "duckdb/common/helper.hpp"
+#include "duckdb/common/arrow/arrow_query_result.hpp"
+
+namespace duckdb {
+
+// Task to create one RecordBatch by (partially) scanning a BatchedDataCollection
+class ArrowBatchTask : public ExecutorTask {
+public:
+	ArrowBatchTask(ArrowQueryResult &result, vector<idx_t> record_batch_indices, Executor &executor,
+	               shared_ptr<Event> event_p, BatchCollectionChunkScanState scan_state, vector<string> names,
+	               idx_t batch_size);
+	void ProduceRecordBatches();
+	TaskExecutionResult ExecuteTask(TaskExecutionMode mode) override;
+
+private:
+	ArrowQueryResult &result;
+	vector<idx_t> record_batch_indices;
+	shared_ptr<Event> event;
+	idx_t batch_size;
+	vector<string> names;
+	BatchCollectionChunkScanState scan_state;
+};
+
+class ArrowMergeEvent : public BasePipelineEvent {
+public:
+	ArrowMergeEvent(ArrowQueryResult &result, BatchedDataCollection &batches, Pipeline &pipeline_p);
+
+public:
+	void Schedule() override;
+
+public:
+	ArrowQueryResult &result;
+	BatchedDataCollection &batches;
+
+private:
+	//! The max size of a record batch to output
+	idx_t record_batch_size;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/arrow/arrow_query_result.hpp

@@ -0,0 +1,52 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb_python/arrow/arrow_query_result.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/winapi.hpp"
+#include "duckdb/common/arrow/arrow_wrapper.hpp"
+#include "duckdb/main/query_result.hpp"
+#include "duckdb/common/error_data.hpp"
+#include "duckdb/common/unique_ptr.hpp"
+
+namespace duckdb {
+
+class ClientContext;
+
+class ArrowQueryResult : public QueryResult {
+public:
+	static constexpr const QueryResultType TYPE = QueryResultType::ARROW_RESULT;
+
+public:
+	friend class ClientContext;
+	//! Creates a successful query result with the specified names and types
+	DUCKDB_API ArrowQueryResult(StatementType statement_type, StatementProperties properties, vector<string> names_p,
+	                            vector<LogicalType> types_p, ClientProperties client_properties, idx_t batch_size);
+	//! Creates an unsuccessful query result with error condition
+	DUCKDB_API explicit ArrowQueryResult(ErrorData error);
+
+public:
+	//! Fetches a DataChunk from the query result.
+	//! This will consume the result (i.e. the result can only be scanned once with this function)
+	DUCKDB_API unique_ptr<DataChunk> Fetch() override;
+	DUCKDB_API unique_ptr<DataChunk> FetchRaw() override;
+	//! Converts the QueryResult to a string
+	DUCKDB_API string ToString() override;
+
+public:
+	vector<unique_ptr<ArrowArrayWrapper>> ConsumeArrays();
+	vector<unique_ptr<ArrowArrayWrapper>> &Arrays();
+	void SetArrowData(vector<unique_ptr<ArrowArrayWrapper>> arrays);
+	idx_t BatchSize() const;
+
+private:
+	vector<unique_ptr<ArrowArrayWrapper>> arrays;
+	idx_t batch_size;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/arrow/arrow_types_extension.hpp

@@ -0,0 +1,42 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/arrow/arrow_types_extension.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+#include "duckdb/common/arrow/arrow.hpp"
+
+namespace duckdb {
+
+class ClientContext;
+
+//! The OptimizerExtensionInfo holds static information relevant to the optimizer extension
+struct ArrowTypesExtensionInfo {
+	virtual ~ArrowTypesExtensionInfo() {
+	}
+};
+
+:typedef void (*arrow_types_recognized_function_t)(OptimizerExtensionInput &input, unique_ptr<LogicalOperator> &plan);
+typedef void (*arrow_types_format_function_t)(OptimizerExtensionInput &input, unique_ptr<LogicalOperator> &plan);
+typedef void (*arrow_types_write_function_t)(OptimizerExtensionInput &input, unique_ptr<LogicalOperator> &plan);
+typedef void (*arrow_types_read_function_t)(OptimizerExtensionInput &input, unique_ptr<LogicalOperator> &plan);
+
+class ArrowTypesExtension {
+public:
+	//! The parse function of the parser extension.
+	//! Takes a query string as input and returns ParserExtensionParseData (on success) or an error
+	arrow_types_recognized_function_t is_valid_function;
+	arrow_types_format_function_t format_function;
+	arrow_types_read_function_t read_function;
+	arrow_types_write_function_t write_function;
+
+	//! Additional parser info passed to the functions
+	shared_ptr<ArrowTypesExtensionInfo> info;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/arrow/physical_arrow_batch_collector.hpp

@@ -0,0 +1,30 @@
+#pragma once
+
+#include "duckdb/execution/operator/helper/physical_batch_collector.hpp"
+
+namespace duckdb {
+
+class ArrowBatchGlobalState : public BatchCollectorGlobalState {
+public:
+	ArrowBatchGlobalState(ClientContext &context, const PhysicalBatchCollector &op)
+	    : BatchCollectorGlobalState(context, op) {
+	}
+};
+
+class PhysicalArrowBatchCollector : public PhysicalBatchCollector {
+public:
+	PhysicalArrowBatchCollector(PreparedStatementData &data, idx_t batch_size)
+	    : PhysicalBatchCollector(data), record_batch_size(batch_size) {
+	}
+
+public:
+	SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
+	                          OperatorSinkFinalizeInput &input) const override;
+	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
+
+public:
+	//! User provided batch size
+	idx_t record_batch_size;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/arrow/physical_arrow_collector.hpp

@@ -0,0 +1,65 @@
+#pragma once
+
+#include "duckdb/execution/operator/helper/physical_result_collector.hpp"
+#include "duckdb/execution/physical_operator_states.hpp"
+#include "duckdb/common/arrow/arrow.hpp"
+#include "duckdb/common/types/batched_data_collection.hpp"
+#include "duckdb/common/map.hpp"
+#include "duckdb/common/arrow/arrow_appender.hpp"
+
+namespace duckdb {
+
+struct ArrowCollectorGlobalState : public GlobalSinkState {
+public:
+	//! The result returned by GetResult
+	unique_ptr<QueryResult> result;
+	vector<unique_ptr<ArrowArrayWrapper>> chunks;
+	mutex glock;
+	shared_ptr<ClientContext> context;
+	idx_t tuple_count = 0;
+};
+
+struct ArrowCollectorLocalState : public LocalSinkState {
+public:
+	// The appender for the current chunk we're creating
+	unique_ptr<ArrowAppender> appender;
+	vector<unique_ptr<ArrowArrayWrapper>> finished_arrays;
+	idx_t tuple_count = 0;
+
+public:
+	void FinishArray() {
+		auto finished_array = make_uniq<ArrowArrayWrapper>();
+		auto row_count = appender->RowCount();
+		finished_array->arrow_array = appender->Finalize();
+		appender.reset();
+		finished_arrays.push_back(std::move(finished_array));
+		tuple_count += row_count;
+	}
+};
+
+class PhysicalArrowCollector : public PhysicalResultCollector {
+public:
+	PhysicalArrowCollector(PreparedStatementData &data, bool parallel, idx_t batch_size)
+	    : PhysicalResultCollector(data), record_batch_size(batch_size), parallel(parallel) {
+	}
+
+public:
+	static unique_ptr<PhysicalResultCollector> Create(ClientContext &context, PreparedStatementData &data,
+	                                                  idx_t batch_size);
+	SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
+	SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
+	unique_ptr<QueryResult> GetResult(GlobalSinkState &state) override;
+	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
+	unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
+	SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
+	                          OperatorSinkFinalizeInput &input) const override;
+	bool ParallelSink() const override;
+	bool SinkOrderDependent() const override;
+
+public:
+	//! User provided batch size
+	idx_t record_batch_size;
+	bool parallel;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/arrow/schema_metadata.hpp

@@ -0,0 +1,43 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/arrow/schema_metadata.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/main/query_result.hpp"
+#include "duckdb/common/arrow/arrow_wrapper.hpp"
+#include "duckdb/main/chunk_scan_state.hpp"
+
+namespace duckdb {
+class ArrowSchemaMetadata {
+public:
+	//! Constructor used to read a metadata schema, used when importing an arrow object
+	explicit ArrowSchemaMetadata(const char *metadata);
+	//! Constructor used to create a metadata schema, used when exporting an arrow object
+	ArrowSchemaMetadata() {};
+	//! Adds an option to the metadata
+	void AddOption(const string &key, const string &value);
+	//! Gets an option from the metadata, returns an empty string if does not exist.
+	string GetOption(const string &key) const;
+	//! Transforms metadata to a char*, used when creating an arrow object
+	unsafe_unique_array<char> SerializeMetadata() const;
+	//! If the arrow extension is set
+	bool HasExtension();
+	//! Get the extension name if set, otherwise returns empty
+	string GetExtensionName() const;
+	//! Key for encode of the extension type name
+	static constexpr const char *ARROW_EXTENSION_NAME = "ARROW:extension:name";
+	//! Key for encode of the metadata key
+	static constexpr const char *ARROW_METADATA_KEY = "ARROW:extension:metadata";
+	//! Creates the metadata based on an extension name
+	static ArrowSchemaMetadata MetadataFromName(const string &extension_name);
+
+private:
+	//! The unordered map that holds the metadata
+	unordered_map<string, string> metadata_map;
+};
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/bswap.hpp

@@ -25,22 +25,24 @@ namespace duckdb {
 	            (((uint64_t)(x)&0x00000000ff000000ull) << 8) | (((uint64_t)(x)&0x0000000000ff0000ull) << 24) |         \
 	            (((uint64_t)(x)&0x000000000000ff00ull) << 40) | (((uint64_t)(x)&0x00000000000000ffull) << 56)))
 
-template <class T>
-static inline T BSwap(const T &x) {
-	static_assert(sizeof(T) == 1 || sizeof(T) == 2 || sizeof(T) == 4 || sizeof(T) == 8,
-	              "Size of type must be 1, 2, 4, or 8 for BSwap");
-	if (sizeof(T) == 1) {
-		return x;
-	} else if (sizeof(T) == 2) {
-		return BSWAP16(x);
-	} else if (sizeof(T) == 4) {
-		// this check is superfluous as the branch is not taken for small return types but the compiler does not realize
-		// that
-		return UnsafeNumericCast<T>(BSWAP32(x));
-	} else {
-		// see above
-		return UnsafeNumericCast<T>(BSWAP64(x));
-	}
+static inline uint8_t BSwap(const uint8_t &x) {
+	return x;
+}
+
+static inline uint16_t BSwap(const uint16_t &x) {
+	return BSWAP16(x);
+}
+
+static inline uint32_t BSwap(const uint32_t &x) {
+	return BSWAP32(x);
+}
+
+static inline uint64_t BSwap(const uint64_t &x) {
+	return BSWAP64(x);
+}
+
+static inline int64_t BSwap(const int64_t &x) {
+	return static_cast<int64_t>(BSWAP64(x));
 }
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/cgroups.hpp

@@ -0,0 +1,30 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/cgroups.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+#include "duckdb/common/optional_idx.hpp"
+#include "duckdb/common/file_system.hpp"
+
+namespace duckdb {
+
+class CGroups {
+public:
+	static optional_idx GetMemoryLimit(FileSystem &fs);
+	static idx_t GetCPULimit(FileSystem &fs, idx_t physical_cores);
+
+private:
+	static optional_idx GetCGroupV2MemoryLimit(FileSystem &fs);
+	static optional_idx GetCGroupV1MemoryLimit(FileSystem &fs);
+	static string ReadCGroupPath(FileSystem &fs, const char *cgroup_file);
+	static string ReadMemoryCGroupPath(FileSystem &fs, const char *cgroup_file);
+	static optional_idx ReadCGroupValue(FileSystem &fs, const char *file_path);
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/compressed_file_system.hpp

@@ -61,6 +61,8 @@ public:
 	DUCKDB_API CompressedFile(CompressedFileSystem &fs, unique_ptr<FileHandle> child_handle_p, const string &path);
 	DUCKDB_API ~CompressedFile() override;
 
+	DUCKDB_API idx_t GetProgress() override;
+
 	CompressedFileSystem &compressed_fs;
 	unique_ptr<FileHandle> child_handle;
 	//! Whether the file is opened for reading or for writing
@@ -74,6 +76,7 @@ public:
 	DUCKDB_API void Close() override;
 
 private:
+	idx_t current_position = 0;
 	unique_ptr<StreamWrapper> stream_wrapper;
 };
 

package/src/duckdb/src/include/duckdb/common/dl.hpp

@@ -24,7 +24,9 @@ namespace duckdb {
 #ifdef _WIN32
 
 inline void *dlopen(const char *file, int mode) {
-	D_ASSERT(file);
+	if (!file) {
+		return (void *)GetModuleHandle(nullptr);
+	}
 	auto fpath = WindowsUtil::UTF8ToUnicode(file);
 	return (void *)LoadLibraryW(fpath.c_str());
 }
@@ -38,6 +40,11 @@ inline std::string GetDLError(void) {
 	return LocalFileSystem::GetLastErrorAsString();
 }
 
+inline void dlclose(void *handle) {
+	D_ASSERT(handle);
+	FreeLibrary((HINSTANCE)handle);
+}
+
 #else
 
 inline std::string GetDLError(void) {