duckdb 0.8.2-dev2068.0 → 0.8.2-dev2133.0

package/src/duckdb/src/function/table/table_scan.cpp

@@ -15,6 +15,8 @@
 #include "duckdb/catalog/dependency_list.hpp"
 #include "duckdb/function/function_set.hpp"
 #include "duckdb/storage/table/scan_state.hpp"
+#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
 
 namespace duckdb {
 
@@ -447,6 +449,35 @@ static unique_ptr<FunctionData> TableScanDeserialize(PlanDeserializationState &s
 	return std::move(result);
 }
 
+static void TableScanFormatSerialize(FormatSerializer &serializer, const optional_ptr<FunctionData> bind_data_p,
+                                     const TableFunction &function) {
+	auto &bind_data = bind_data_p->Cast<TableScanBindData>();
+	serializer.WriteProperty("catalog", bind_data.table.schema.catalog.GetName());
+	serializer.WriteProperty("schema", bind_data.table.schema.name);
+	serializer.WriteProperty("table", bind_data.table.name);
+	serializer.WriteProperty("is_index_scan", bind_data.is_index_scan);
+	serializer.WriteProperty("is_create_index", bind_data.is_create_index);
+	serializer.WriteProperty("result_ids", bind_data.result_ids);
+	serializer.WriteProperty("result_ids", bind_data.result_ids);
+}
+
+static unique_ptr<FunctionData> TableScanFormatDeserialize(FormatDeserializer &deserializer, TableFunction &function) {
+	auto catalog = deserializer.ReadProperty<string>("catalog");
+	auto schema = deserializer.ReadProperty<string>("schema");
+	auto table = deserializer.ReadProperty<string>("table");
+	auto &catalog_entry =
+	    Catalog::GetEntry<TableCatalogEntry>(deserializer.Get<ClientContext &>(), catalog, schema, table);
+	if (catalog_entry.type != CatalogType::TABLE_ENTRY) {
+		throw SerializationException("Cant find table for %s.%s", schema, table);
+	}
+	auto result = make_uniq<TableScanBindData>(catalog_entry.Cast<DuckTableEntry>());
+	deserializer.ReadProperty("is_index_scan", result->is_index_scan);
+	deserializer.ReadProperty("is_create_index", result->is_create_index);
+	deserializer.ReadProperty("result_ids", result->result_ids);
+	deserializer.ReadProperty("result_ids", result->result_ids);
+	return std::move(result);
+}
+
 TableFunction TableScanFunction::GetIndexScanFunction() {
 	TableFunction scan_function("index_scan", {}, IndexScanFunction);
 	scan_function.init_local = nullptr;
@@ -462,6 +493,8 @@ TableFunction TableScanFunction::GetIndexScanFunction() {
 	scan_function.filter_pushdown = false;
 	scan_function.serialize = TableScanSerialize;
 	scan_function.deserialize = TableScanDeserialize;
+	scan_function.format_serialize = TableScanFormatSerialize;
+	scan_function.format_deserialize = TableScanFormatDeserialize;
 	return scan_function;
 }
 
@@ -482,6 +515,8 @@ TableFunction TableScanFunction::GetFunction() {
 	scan_function.filter_prune = true;
 	scan_function.serialize = TableScanSerialize;
 	scan_function.deserialize = TableScanDeserialize;
+	scan_function.format_serialize = TableScanFormatSerialize;
+	scan_function.format_deserialize = TableScanFormatDeserialize;
 	return scan_function;
 }
 

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.8.2-dev2068"
+#define DUCKDB_VERSION "0.8.2-dev2133"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "83481168e0"
+#define DUCKDB_SOURCE_ID "726b6d1566"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/function/table_function.cpp

@@ -18,8 +18,8 @@ TableFunction::TableFunction(string name, vector<LogicalType> arguments, table_f
       init_global(init_global), init_local(init_local), function(function), in_out_function(nullptr),
       in_out_function_final(nullptr), statistics(nullptr), dependency(nullptr), cardinality(nullptr),
       pushdown_complex_filter(nullptr), to_string(nullptr), table_scan_progress(nullptr), get_batch_index(nullptr),
-      get_batch_info(nullptr), serialize(nullptr), deserialize(nullptr), projection_pushdown(false),
-      filter_pushdown(false), filter_prune(false) {
+      get_batch_info(nullptr), serialize(nullptr), deserialize(nullptr), format_serialize(nullptr),
+      format_deserialize(nullptr), projection_pushdown(false), filter_pushdown(false), filter_prune(false) {
 }
 
 TableFunction::TableFunction(const vector<LogicalType> &arguments, table_function_t function,
@@ -32,7 +32,8 @@ TableFunction::TableFunction()
       init_local(nullptr), function(nullptr), in_out_function(nullptr), statistics(nullptr), dependency(nullptr),
       cardinality(nullptr), pushdown_complex_filter(nullptr), to_string(nullptr), table_scan_progress(nullptr),
       get_batch_index(nullptr), get_batch_info(nullptr), serialize(nullptr), deserialize(nullptr),
-      projection_pushdown(false), filter_pushdown(false), filter_prune(false) {
+      format_serialize(nullptr), format_deserialize(nullptr), projection_pushdown(false), filter_pushdown(false),
+      filter_prune(false) {
 }
 
 bool TableFunction::Equal(const TableFunction &rhs) const {

package/src/duckdb/src/include/duckdb/common/arrow/appender/append_data.hpp

@@ -0,0 +1,109 @@
+#pragma once
+
+#include "duckdb/common/types.hpp"
+#include "duckdb/common/types/vector.hpp"
+#include "duckdb/common/arrow/arrow.hpp"
+#include "duckdb/common/arrow/arrow_buffer.hpp"
+#include "duckdb/common/arrow/arrow_options.hpp"
+#include "duckdb/common/array.hpp"
+
+namespace duckdb {
+
+//===--------------------------------------------------------------------===//
+// Arrow append data
+//===--------------------------------------------------------------------===//
+typedef void (*initialize_t)(ArrowAppendData &result, const LogicalType &type, idx_t capacity);
+// append_data: The arrow array we're appending into
+// input: The data we're appending
+// from: The offset into the input we're scanning
+// to: The last index of the input we're scanning
+// input_size: The total size of the 'input' Vector.
+typedef void (*append_vector_t)(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size);
+typedef void (*finalize_t)(ArrowAppendData &append_data, const LogicalType &type, ArrowArray *result);
+
+// This struct is used to save state for appending a column
+// afterwards the ownership is passed to the arrow array, as 'private_data'
+// FIXME: we should separate the append state variables from the variables required by the ArrowArray into
+// ArrowAppendState
+struct ArrowAppendData {
+	explicit ArrowAppendData(ArrowOptions &options_p) : options(options_p) {
+	}
+	// the buffers of the arrow vector
+	ArrowBuffer validity;
+	ArrowBuffer main_buffer;
+	ArrowBuffer aux_buffer;
+
+	idx_t row_count = 0;
+	idx_t null_count = 0;
+
+	// function pointers for construction
+	initialize_t initialize = nullptr;
+	append_vector_t append_vector = nullptr;
+	finalize_t finalize = nullptr;
+
+	// child data (if any)
+	vector<unique_ptr<ArrowAppendData>> child_data;
+
+	// the arrow array C API data, only set after Finalize
+	unique_ptr<ArrowArray> array;
+	duckdb::array<const void *, 3> buffers = {{nullptr, nullptr, nullptr}};
+	vector<ArrowArray *> child_pointers;
+
+	ArrowOptions options;
+};
+
+//===--------------------------------------------------------------------===//
+// Append Helper Functions
+//===--------------------------------------------------------------------===//
+static void GetBitPosition(idx_t row_idx, idx_t &current_byte, uint8_t &current_bit) {
+	current_byte = row_idx / 8;
+	current_bit = row_idx % 8;
+}
+
+static void UnsetBit(uint8_t *data, idx_t current_byte, uint8_t current_bit) {
+	data[current_byte] &= ~((uint64_t)1 << current_bit);
+}
+
+static void NextBit(idx_t &current_byte, uint8_t &current_bit) {
+	current_bit++;
+	if (current_bit == 8) {
+		current_byte++;
+		current_bit = 0;
+	}
+}
+
+static void ResizeValidity(ArrowBuffer &buffer, idx_t row_count) {
+	auto byte_count = (row_count + 7) / 8;
+	buffer.resize(byte_count, 0xFF);
+}
+
+static void SetNull(ArrowAppendData &append_data, uint8_t *validity_data, idx_t current_byte, uint8_t current_bit) {
+	UnsetBit(validity_data, current_byte, current_bit);
+	append_data.null_count++;
+}
+
+static void AppendValidity(ArrowAppendData &append_data, UnifiedVectorFormat &format, idx_t from, idx_t to) {
+	// resize the buffer, filling the validity buffer with all valid values
+	idx_t size = to - from;
+	ResizeValidity(append_data.validity, append_data.row_count + size);
+	if (format.validity.AllValid()) {
+		// if all values are valid we don't need to do anything else
+		return;
+	}
+
+	// otherwise we iterate through the validity mask
+	auto validity_data = (uint8_t *)append_data.validity.data();
+	uint8_t current_bit;
+	idx_t current_byte;
+	GetBitPosition(append_data.row_count, current_byte, current_bit);
+	for (idx_t i = from; i < to; i++) {
+		auto source_idx = format.sel->get_index(i);
+		// append the validity mask
+		if (!format.validity.RowIsValid(source_idx)) {
+			SetNull(append_data, validity_data, current_byte, current_bit);
+		}
+		NextBit(current_byte, current_bit);
+	}
+}
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/arrow/appender/bool_data.hpp

@@ -0,0 +1,15 @@
+#pragma once
+
+#include "duckdb/common/arrow/appender/append_data.hpp"
+#include "duckdb/common/types/vector.hpp"
+
+namespace duckdb {
+
+struct ArrowBoolData {
+public:
+	static void Initialize(ArrowAppendData &result, const LogicalType &type, idx_t capacity);
+	static void Append(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size);
+	static void Finalize(ArrowAppendData &append_data, const LogicalType &type, ArrowArray *result);
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/arrow/appender/enum_data.hpp

@@ -0,0 +1,69 @@
+#pragma once
+
+#include "duckdb/common/arrow/appender/append_data.hpp"
+#include "duckdb/common/arrow/appender/scalar_data.hpp"
+
+namespace duckdb {
+
+//===--------------------------------------------------------------------===//
+// Enums
+//===--------------------------------------------------------------------===//
+template <class TGT>
+struct ArrowEnumData : public ArrowScalarBaseData<TGT> {
+	static idx_t GetLength(string_t input) {
+		return input.GetSize();
+	}
+	static void WriteData(data_ptr_t target, string_t input) {
+		memcpy(target, input.GetData(), input.GetSize());
+	}
+	static void EnumAppendVector(ArrowAppendData &append_data, const Vector &input, idx_t size) {
+		D_ASSERT(input.GetVectorType() == VectorType::FLAT_VECTOR);
+
+		// resize the validity mask and set up the validity buffer for iteration
+		ResizeValidity(append_data.validity, append_data.row_count + size);
+
+		// resize the offset buffer - the offset buffer holds the offsets into the child array
+		append_data.main_buffer.resize(append_data.main_buffer.size() + sizeof(uint32_t) * (size + 1));
+		auto data = FlatVector::GetData<string_t>(input);
+		auto offset_data = append_data.main_buffer.GetData<uint32_t>();
+		if (append_data.row_count == 0) {
+			// first entry
+			offset_data[0] = 0;
+		}
+		// now append the string data to the auxiliary buffer
+		// the auxiliary buffer's length depends on the string lengths, so we resize as required
+		auto last_offset = offset_data[append_data.row_count];
+		for (idx_t i = 0; i < size; i++) {
+			auto offset_idx = append_data.row_count + i + 1;
+
+			auto string_length = GetLength(data[i]);
+
+			// append the offset data
+			auto current_offset = last_offset + string_length;
+			offset_data[offset_idx] = current_offset;
+
+			// resize the string buffer if required, and write the string data
+			append_data.aux_buffer.resize(current_offset);
+			WriteData(append_data.aux_buffer.data() + last_offset, data[i]);
+
+			last_offset = current_offset;
+		}
+		append_data.row_count += size;
+	}
+	static void Initialize(ArrowAppendData &result, const LogicalType &type, idx_t capacity) {
+		result.main_buffer.reserve(capacity * sizeof(TGT));
+		// construct the enum child data
+		auto enum_data = ArrowAppender::InitializeChild(LogicalType::VARCHAR, EnumType::GetSize(type), result.options);
+		EnumAppendVector(*enum_data, EnumType::GetValuesInsertOrder(type), EnumType::GetSize(type));
+		result.child_data.push_back(std::move(enum_data));
+	}
+
+	static void Finalize(ArrowAppendData &append_data, const LogicalType &type, ArrowArray *result) {
+		result->n_buffers = 2;
+		result->buffers[1] = append_data.main_buffer.data();
+		// finalize the enum child data, and assign it to the dictionary
+		result->dictionary = ArrowAppender::FinalizeChild(LogicalType::VARCHAR, *append_data.child_data[0]);
+	}
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/arrow/appender/list.hpp

@@ -0,0 +1,8 @@
+#include "duckdb/common/arrow/appender/bool_data.hpp"
+#include "duckdb/common/arrow/appender/enum_data.hpp"
+#include "duckdb/common/arrow/appender/list_data.hpp"
+#include "duckdb/common/arrow/appender/map_data.hpp"
+#include "duckdb/common/arrow/appender/scalar_data.hpp"
+#include "duckdb/common/arrow/appender/struct_data.hpp"
+#include "duckdb/common/arrow/appender/union_data.hpp"
+#include "duckdb/common/arrow/appender/varchar_data.hpp"

package/src/duckdb/src/include/duckdb/common/arrow/appender/list_data.hpp

@@ -0,0 +1,18 @@
+#pragma once
+
+#include "duckdb/common/arrow/appender/append_data.hpp"
+
+namespace duckdb {
+
+struct ArrowListData {
+public:
+	static void Initialize(ArrowAppendData &result, const LogicalType &type, idx_t capacity);
+	static void Append(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size);
+	static void Finalize(ArrowAppendData &append_data, const LogicalType &type, ArrowArray *result);
+
+public:
+	static void AppendOffsets(ArrowAppendData &append_data, UnifiedVectorFormat &format, idx_t from, idx_t to,
+	                          vector<sel_t> &child_sel);
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/arrow/appender/map_data.hpp

@@ -0,0 +1,18 @@
+#pragma once
+
+#include "duckdb/common/arrow/arrow_appender.hpp"
+#include "duckdb/common/arrow/appender/append_data.hpp"
+
+namespace duckdb {
+
+//===--------------------------------------------------------------------===//
+// Maps
+//===--------------------------------------------------------------------===//
+struct ArrowMapData {
+public:
+	static void Initialize(ArrowAppendData &result, const LogicalType &type, idx_t capacity);
+	static void Append(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size);
+	static void Finalize(ArrowAppendData &append_data, const LogicalType &type, ArrowArray *result);
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/arrow/appender/scalar_data.hpp

@@ -0,0 +1,88 @@
+#pragma once
+
+#include "duckdb/common/arrow/appender/append_data.hpp"
+#include "duckdb/function/table/arrow.hpp"
+
+namespace duckdb {
+
+//===--------------------------------------------------------------------===//
+// Scalar Types
+//===--------------------------------------------------------------------===//
+struct ArrowScalarConverter {
+	template <class TGT, class SRC>
+	static TGT Operation(SRC input) {
+		return input;
+	}
+
+	static bool SkipNulls() {
+		return false;
+	}
+
+	template <class TGT>
+	static void SetNull(TGT &value) {
+	}
+};
+
+struct ArrowIntervalConverter {
+	template <class TGT, class SRC>
+	static TGT Operation(SRC input) {
+		ArrowInterval result;
+		result.months = input.months;
+		result.days = input.days;
+		result.nanoseconds = input.micros * Interval::NANOS_PER_MICRO;
+		return result;
+	}
+
+	static bool SkipNulls() {
+		return true;
+	}
+
+	template <class TGT>
+	static void SetNull(TGT &value) {
+	}
+};
+
+template <class TGT, class SRC = TGT, class OP = ArrowScalarConverter>
+struct ArrowScalarBaseData {
+	static void Append(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size) {
+		D_ASSERT(to >= from);
+		idx_t size = to - from;
+		D_ASSERT(size <= input_size);
+		UnifiedVectorFormat format;
+		input.ToUnifiedFormat(input_size, format);
+
+		// append the validity mask
+		AppendValidity(append_data, format, from, to);
+
+		// append the main data
+		append_data.main_buffer.resize(append_data.main_buffer.size() + sizeof(TGT) * size);
+		auto data = UnifiedVectorFormat::GetData<SRC>(format);
+		auto result_data = append_data.main_buffer.GetData<TGT>();
+
+		for (idx_t i = from; i < to; i++) {
+			auto source_idx = format.sel->get_index(i);
+			auto result_idx = append_data.row_count + i - from;
+
+			if (OP::SkipNulls() && !format.validity.RowIsValid(source_idx)) {
+				OP::template SetNull<TGT>(result_data[result_idx]);
+				continue;
+			}
+			result_data[result_idx] = OP::template Operation<TGT, SRC>(data[source_idx]);
+		}
+		append_data.row_count += size;
+	}
+};
+
+template <class TGT, class SRC = TGT, class OP = ArrowScalarConverter>
+struct ArrowScalarData : public ArrowScalarBaseData<TGT, SRC, OP> {
+	static void Initialize(ArrowAppendData &result, const LogicalType &type, idx_t capacity) {
+		result.main_buffer.reserve(capacity * sizeof(TGT));
+	}
+
+	static void Finalize(ArrowAppendData &append_data, const LogicalType &type, ArrowArray *result) {
+		result->n_buffers = 2;
+		result->buffers[1] = append_data.main_buffer.data();
+	}
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/arrow/appender/struct_data.hpp

@@ -0,0 +1,18 @@
+#pragma once
+
+#include "duckdb/common/arrow/appender/append_data.hpp"
+#include "duckdb/common/arrow/appender/scalar_data.hpp"
+
+namespace duckdb {
+
+//===--------------------------------------------------------------------===//
+// Structs
+//===--------------------------------------------------------------------===//
+struct ArrowStructData {
+public:
+	static void Initialize(ArrowAppendData &result, const LogicalType &type, idx_t capacity);
+	static void Append(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size);
+	static void Finalize(ArrowAppendData &append_data, const LogicalType &type, ArrowArray *result);
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/arrow/appender/union_data.hpp

@@ -0,0 +1,21 @@
+#pragma once
+
+#include "duckdb/common/arrow/appender/append_data.hpp"
+
+namespace duckdb {
+
+//===--------------------------------------------------------------------===//
+// Unions
+//===--------------------------------------------------------------------===//
+/**
+ * Based on https://arrow.apache.org/docs/format/Columnar.html#union-layout &
+ * https://arrow.apache.org/docs/format/CDataInterface.html
+ */
+struct ArrowUnionData {
+public:
+	static void Initialize(ArrowAppendData &result, const LogicalType &type, idx_t capacity);
+	static void Append(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size);
+	static void Finalize(ArrowAppendData &append_data, const LogicalType &type, ArrowArray *result);
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/arrow/appender/varchar_data.hpp

@@ -0,0 +1,105 @@
+#pragma once
+
+#include "duckdb/common/arrow/appender/append_data.hpp"
+#include "duckdb/common/arrow/appender/scalar_data.hpp"
+
+namespace duckdb {
+
+//===--------------------------------------------------------------------===//
+// Varchar
+//===--------------------------------------------------------------------===//
+struct ArrowVarcharConverter {
+	template <class SRC>
+	static idx_t GetLength(SRC input) {
+		return input.GetSize();
+	}
+
+	template <class SRC>
+	static void WriteData(data_ptr_t target, SRC input) {
+		memcpy(target, input.GetData(), input.GetSize());
+	}
+};
+
+struct ArrowUUIDConverter {
+	template <class SRC>
+	static idx_t GetLength(SRC input) {
+		return UUID::STRING_SIZE;
+	}
+
+	template <class SRC>
+	static void WriteData(data_ptr_t target, SRC input) {
+		UUID::ToString(input, char_ptr_cast(target));
+	}
+};
+
+template <class SRC = string_t, class OP = ArrowVarcharConverter, class BUFTYPE = uint64_t>
+struct ArrowVarcharData {
+	static void Initialize(ArrowAppendData &result, const LogicalType &type, idx_t capacity) {
+		result.main_buffer.reserve((capacity + 1) * sizeof(BUFTYPE));
+
+		result.aux_buffer.reserve(capacity);
+	}
+
+	static void Append(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size) {
+		idx_t size = to - from;
+		UnifiedVectorFormat format;
+		input.ToUnifiedFormat(input_size, format);
+
+		// resize the validity mask and set up the validity buffer for iteration
+		ResizeValidity(append_data.validity, append_data.row_count + size);
+		auto validity_data = (uint8_t *)append_data.validity.data();
+
+		// resize the offset buffer - the offset buffer holds the offsets into the child array
+		append_data.main_buffer.resize(append_data.main_buffer.size() + sizeof(BUFTYPE) * (size + 1));
+		auto data = UnifiedVectorFormat::GetData<SRC>(format);
+		auto offset_data = append_data.main_buffer.GetData<BUFTYPE>();
+		if (append_data.row_count == 0) {
+			// first entry
+			offset_data[0] = 0;
+		}
+		// now append the string data to the auxiliary buffer
+		// the auxiliary buffer's length depends on the string lengths, so we resize as required
+		auto last_offset = offset_data[append_data.row_count];
+		idx_t max_offset = append_data.row_count + to - from;
+		if (max_offset > NumericLimits<uint32_t>::Maximum() &&
+		    append_data.options.offset_size == ArrowOffsetSize::REGULAR) {
+			throw InvalidInputException("Arrow Appender: The maximum total string size for regular string buffers is "
+			                            "%u but the offset of %lu exceeds this.",
+			                            NumericLimits<uint32_t>::Maximum(), max_offset);
+		}
+		for (idx_t i = from; i < to; i++) {
+			auto source_idx = format.sel->get_index(i);
+			auto offset_idx = append_data.row_count + i + 1 - from;
+
+			if (!format.validity.RowIsValid(source_idx)) {
+				uint8_t current_bit;
+				idx_t current_byte;
+				GetBitPosition(append_data.row_count + i - from, current_byte, current_bit);
+				SetNull(append_data, validity_data, current_byte, current_bit);
+				offset_data[offset_idx] = last_offset;
+				continue;
+			}
+
+			auto string_length = OP::GetLength(data[source_idx]);
+
+			// append the offset data
+			auto current_offset = last_offset + string_length;
+			offset_data[offset_idx] = current_offset;
+
+			// resize the string buffer if required, and write the string data
+			append_data.aux_buffer.resize(current_offset);
+			OP::WriteData(append_data.aux_buffer.data() + last_offset, data[source_idx]);
+
+			last_offset = current_offset;
+		}
+		append_data.row_count += size;
+	}
+
+	static void Finalize(ArrowAppendData &append_data, const LogicalType &type, ArrowArray *result) {
+		result->n_buffers = 3;
+		result->buffers[1] = append_data.main_buffer.data();
+		result->buffers[2] = append_data.aux_buffer.data();
+	}
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/arrow/arrow_appender.hpp

@@ -27,6 +27,11 @@ public:
 	//! Returns the underlying arrow array
 	DUCKDB_API ArrowArray Finalize();
 
+public:
+	static void ReleaseArray(ArrowArray *array);
+	static ArrowArray *FinalizeChild(const LogicalType &type, ArrowAppendData &append_data);
+	static unique_ptr<ArrowAppendData> InitializeChild(const LogicalType &type, idx_t capacity, ArrowOptions &options);
+
 private:
 	//! The types of the chunks that will be appended in
 	vector<LogicalType> types;

package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp

@@ -8,7 +8,7 @@
 
 #pragma once
 
-#include "duckdb/common/types.hpp"
+#include "duckdb/common/common.hpp"
 #include "duckdb/common/multi_file_reader_options.hpp"
 #include "duckdb/common/enums/file_glob_options.hpp"
 #include "duckdb/common/union_by_name.hpp"
@@ -32,6 +32,8 @@ struct HivePartitioningIndex {
 
 	DUCKDB_API void Serialize(Serializer &serializer) const;
 	DUCKDB_API static HivePartitioningIndex Deserialize(Deserializer &source);
+	DUCKDB_API void FormatSerialize(FormatSerializer &serializer) const;
+	DUCKDB_API static HivePartitioningIndex FormatDeserialize(FormatDeserializer &deserializer);
 };
 
 //! The bind data for the multi-file reader, obtained through MultiFileReader::BindReader
@@ -43,6 +45,8 @@ struct MultiFileReaderBindData {
 
 	DUCKDB_API void Serialize(Serializer &serializer) const;
 	DUCKDB_API static MultiFileReaderBindData Deserialize(Deserializer &source);
+	DUCKDB_API void FormatSerialize(FormatSerializer &serializer) const;
+	DUCKDB_API static MultiFileReaderBindData FormatDeserialize(FormatDeserializer &deserializer);
 };
 
 struct MultiFileFilterEntry {

package/src/duckdb/src/include/duckdb/common/multi_file_reader_options.hpp

@@ -28,6 +28,8 @@ struct MultiFileReaderOptions {
 
 	DUCKDB_API void Serialize(Serializer &serializer) const;
 	DUCKDB_API static MultiFileReaderOptions Deserialize(Deserializer &source);
+	DUCKDB_API void FormatSerialize(FormatSerializer &serializer) const;
+	DUCKDB_API static MultiFileReaderOptions FormatDeserialize(FormatDeserializer &source);
 	DUCKDB_API void AddBatchInfo(BindInfo &bind_info) const;
 	DUCKDB_API void AutoDetectHivePartitioning(const vector<string> &files, ClientContext &context);
 	DUCKDB_API static bool AutoDetectHivePartitioningInternal(const vector<string> &files);

package/src/duckdb/src/include/duckdb/common/serializer/format_deserializer.hpp

@@ -127,6 +127,16 @@ public:
 		return data.Unset<T>();
 	}
 
+	// Manually begin an object - should be followed by EndObject
+	void BeginObject(const char *tag) {
+		SetTag(tag);
+		OnObjectBegin();
+	}
+
+	void EndObject() {
+		OnObjectEnd();
+	}
+
 private:
 	// Deserialize anything implementing a FormatDeserialize method
 	template <typename T = void>
@@ -208,6 +218,28 @@ private:
 		return map;
 	}
 
+	template <typename T = void>
+	inline typename std::enable_if<is_map<T>::value, T>::type Read() {
+		using KEY_TYPE = typename is_map<T>::KEY_TYPE;
+		using VALUE_TYPE = typename is_map<T>::VALUE_TYPE;
+
+		T map;
+		auto size = OnMapBegin();
+		for (idx_t i = 0; i < size; i++) {
+			OnMapEntryBegin();
+			OnMapKeyBegin();
+			auto key = Read<KEY_TYPE>();
+			OnMapKeyEnd();
+			OnMapValueBegin();
+			auto value = Read<VALUE_TYPE>();
+			OnMapValueEnd();
+			OnMapEntryEnd();
+			map[std::move(key)] = std::move(value);
+		}
+		OnMapEnd();
+		return map;
+	}
+
 	// Deserialize an unordered set
 	template <typename T = void>
 	inline typename std::enable_if<is_unordered_set<T>::value, T>::type Read() {