duckdb 0.8.2-dev2320.0 → 0.8.2-dev2399.0

package/src/duckdb/src/core_functions/aggregate/nested/list.cpp

@@ -5,24 +5,6 @@
 
 namespace duckdb {
 
-static void RecursiveFlatten(Vector &vector, idx_t &count) {
-	if (vector.GetVectorType() != VectorType::FLAT_VECTOR) {
-		vector.Flatten(count);
-	}
-
-	auto internal_type = vector.GetType().InternalType();
-	if (internal_type == PhysicalType::LIST) {
-		auto &child_vector = ListVector::GetEntry(vector);
-		auto child_vector_count = ListVector::GetListSize(vector);
-		RecursiveFlatten(child_vector, child_vector_count);
-	} else if (internal_type == PhysicalType::STRUCT) {
-		auto &children = StructVector::GetEntries(vector);
-		for (auto &child : children) {
-			RecursiveFlatten(*child, count);
-		}
-	}
-}
-
 struct ListBindData : public FunctionData {
 	explicit ListBindData(const LogicalType &stype_p);
 	~ListBindData() override;
@@ -60,12 +42,6 @@ struct ListFunction {
 		state.linked_list.first_segment = nullptr;
 		state.linked_list.last_segment = nullptr;
 	}
-
-	template <class STATE>
-	static void Destroy(STATE &state, AggregateInputData &aggr_input_data) {
-		// nop
-	}
-
 	static bool IgnoreNull() {
 		return false;
 	}
@@ -73,58 +49,54 @@ struct ListFunction {
 
 static void ListUpdateFunction(Vector inputs[], AggregateInputData &aggr_input_data, idx_t input_count,
                                Vector &state_vector, idx_t count) {
-	D_ASSERT(input_count == 1);
 
+	D_ASSERT(input_count == 1);
 	auto &input = inputs[0];
-	UnifiedVectorFormat sdata;
-	state_vector.ToUnifiedFormat(count, sdata);
+	RecursiveUnifiedVectorFormat input_data;
+	Vector::RecursiveToUnifiedFormat(input, count, input_data);
 
-	auto states = UnifiedVectorFormat::GetData<ListAggState *>(sdata);
-	RecursiveFlatten(input, count);
+	UnifiedVectorFormat states_data;
+	state_vector.ToUnifiedFormat(count, states_data);
+	auto states = UnifiedVectorFormat::GetData<ListAggState *>(states_data);
 
 	auto &list_bind_data = aggr_input_data.bind_data->Cast<ListBindData>();
 
 	for (idx_t i = 0; i < count; i++) {
-		auto &state = *states[sdata.sel->get_index(i)];
-		list_bind_data.functions.AppendRow(aggr_input_data.allocator, state.linked_list, input, i, count);
+		auto &state = *states[states_data.sel->get_index(i)];
+		list_bind_data.functions.AppendRow(aggr_input_data.allocator, state.linked_list, input_data, i);
 	}
 }
 
-static void ListCombineFunction(Vector &state, Vector &combined, AggregateInputData &aggr_input_data, idx_t count) {
-	UnifiedVectorFormat sdata;
-	state.ToUnifiedFormat(count, sdata);
-	auto states_ptr = UnifiedVectorFormat::GetData<ListAggState *>(sdata);
+static void ListCombineFunction(Vector &states_vector, Vector &combined, AggregateInputData &, idx_t count) {
 
-	auto &list_bind_data = aggr_input_data.bind_data->Cast<ListBindData>();
+	UnifiedVectorFormat states_data;
+	states_vector.ToUnifiedFormat(count, states_data);
+	auto states_ptr = UnifiedVectorFormat::GetData<ListAggState *>(states_data);
 
 	auto combined_ptr = FlatVector::GetData<ListAggState *>(combined);
 	for (idx_t i = 0; i < count; i++) {
-		auto &state = *states_ptr[sdata.sel->get_index(i)];
-		if (state.linked_list.total_capacity == 0) {
-			// NULL, no need to append.
-			continue;
-		}
 
-		// copy the linked list of the state
-		auto copied_linked_list = LinkedList(state.linked_list.total_capacity, nullptr, nullptr);
-		list_bind_data.functions.CopyLinkedList(state.linked_list, copied_linked_list, aggr_input_data.allocator);
+		auto &state = *states_ptr[states_data.sel->get_index(i)];
+		D_ASSERT(state.linked_list.total_capacity != 0);
 
-		// append the copied linked list to the combined state
-		if (combined_ptr[i]->linked_list.last_segment) {
-			combined_ptr[i]->linked_list.last_segment->next = copied_linked_list.first_segment;
-		} else {
-			combined_ptr[i]->linked_list.first_segment = copied_linked_list.first_segment;
+		if (combined_ptr[i]->linked_list.total_capacity == 0) {
+			combined_ptr[i]->linked_list = state.linked_list;
+			continue;
 		}
-		combined_ptr[i]->linked_list.last_segment = copied_linked_list.last_segment;
-		combined_ptr[i]->linked_list.total_capacity += copied_linked_list.total_capacity;
+
+		// append the linked list
+		combined_ptr[i]->linked_list.last_segment->next = state.linked_list.first_segment;
+		combined_ptr[i]->linked_list.last_segment = state.linked_list.last_segment;
+		combined_ptr[i]->linked_list.total_capacity += state.linked_list.total_capacity;
 	}
 }
 
-static void ListFinalize(Vector &state_vector, AggregateInputData &aggr_input_data, Vector &result, idx_t count,
+static void ListFinalize(Vector &states_vector, AggregateInputData &aggr_input_data, Vector &result, idx_t count,
                          idx_t offset) {
-	UnifiedVectorFormat sdata;
-	state_vector.ToUnifiedFormat(count, sdata);
-	auto states = UnifiedVectorFormat::GetData<ListAggState *>(sdata);
+
+	UnifiedVectorFormat states_data;
+	states_vector.ToUnifiedFormat(count, states_data);
+	auto states = UnifiedVectorFormat::GetData<ListAggState *>(states_data);
 
 	D_ASSERT(result.GetType().id() == LogicalTypeId::LIST);
 
@@ -133,9 +105,11 @@ static void ListFinalize(Vector &state_vector, AggregateInputData &aggr_input_da
 	size_t total_len = ListVector::GetListSize(result);
 
 	auto &list_bind_data = aggr_input_data.bind_data->Cast<ListBindData>();
-	// first iterate over all of the entries and set up the list entries, plus get the newly required total length
+
+	// first iterate over all entries and set up the list entries, and get the newly required total length
 	for (idx_t i = 0; i < count; i++) {
-		auto &state = *states[sdata.sel->get_index(i)];
+
+		auto &state = *states[states_data.sel->get_index(i)];
 		const auto rid = i + offset;
 		result_data[rid].offset = total_len;
 		if (state.linked_list.total_capacity == 0) {
@@ -143,16 +117,19 @@ static void ListFinalize(Vector &state_vector, AggregateInputData &aggr_input_da
 			result_data[rid].length = 0;
 			continue;
 		}
+
 		// set the length and offset of this list in the result vector
 		auto total_capacity = state.linked_list.total_capacity;
 		result_data[rid].length = total_capacity;
 		total_len += total_capacity;
 	}
-	// reserve capacity, then iterate over all of the entries again and copy over the data tot he child vector
+
+	// reserve capacity, then iterate over all entries again and copy over the data to the child vector
 	ListVector::Reserve(result, total_len);
 	auto &result_child = ListVector::GetEntry(result);
 	for (idx_t i = 0; i < count; i++) {
-		auto &state = *states[sdata.sel->get_index(i)];
+
+		auto &state = *states[states_data.sel->get_index(i)];
 		const auto rid = i + offset;
 		if (state.linked_list.total_capacity == 0) {
 			continue;
@@ -161,6 +138,48 @@ static void ListFinalize(Vector &state_vector, AggregateInputData &aggr_input_da
 		idx_t current_offset = result_data[rid].offset;
 		list_bind_data.functions.BuildListVector(state.linked_list, result_child, current_offset);
 	}
+
+	ListVector::SetListSize(result, total_len);
+}
+
+static void ListWindow(Vector inputs[], const ValidityMask &filter_mask, AggregateInputData &aggr_input_data,
+                       idx_t input_count, data_ptr_t state, const FrameBounds &frame, const FrameBounds &prev,
+                       Vector &result, idx_t rid, idx_t bias) {
+
+	auto &list_bind_data = aggr_input_data.bind_data->Cast<ListBindData>();
+	LinkedList linked_list;
+
+	// UPDATE step
+
+	D_ASSERT(input_count == 1);
+	auto &input = inputs[0];
+
+	// FIXME: we unify more values than necessary (count is frame.end)
+	RecursiveUnifiedVectorFormat input_data;
+	Vector::RecursiveToUnifiedFormat(input, frame.end, input_data);
+
+	for (idx_t i = frame.start; i < frame.end; i++) {
+		list_bind_data.functions.AppendRow(aggr_input_data.allocator, linked_list, input_data, i);
+	}
+
+	// FINALIZE step
+
+	D_ASSERT(result.GetType().id() == LogicalTypeId::LIST);
+	auto result_data = FlatVector::GetData<list_entry_t>(result);
+	size_t total_len = ListVector::GetListSize(result);
+
+	// set the length and offset of this list in the result vector
+	result_data[rid].offset = total_len;
+	result_data[rid].length = linked_list.total_capacity;
+	D_ASSERT(linked_list.total_capacity != 0);
+	total_len += linked_list.total_capacity;
+
+	// reserve capacity, then copy over the data to the child vector
+	ListVector::Reserve(result, total_len);
+	auto &result_child = ListVector::GetEntry(result);
+	idx_t offset = result_data[rid].offset;
+	list_bind_data.functions.BuildListVector(linked_list, result_child, offset);
+
 	ListVector::SetListSize(result, total_len);
 }
 
@@ -182,8 +201,8 @@ unique_ptr<FunctionData> ListBindFunction(ClientContext &context, AggregateFunct
 AggregateFunction ListFun::GetFunction() {
 	return AggregateFunction({LogicalType::ANY}, LogicalTypeId::LIST, AggregateFunction::StateSize<ListAggState>,
 	                         AggregateFunction::StateInitialize<ListAggState, ListFunction>, ListUpdateFunction,
-	                         ListCombineFunction, ListFinalize, nullptr, ListBindFunction,
-	                         AggregateFunction::StateDestroy<ListAggState, ListFunction>, nullptr, nullptr);
+	                         ListCombineFunction, ListFinalize, nullptr, ListBindFunction, nullptr, nullptr,
+	                         ListWindow);
 }
 
 } // namespace duckdb

package/src/duckdb/src/execution/aggregate_hashtable.cpp

@@ -584,6 +584,12 @@ void GroupedAggregateHashTable::Combine(GroupedAggregateHashTable &other) {
 	}
 
 	Verify();
+
+	// if we combine states, then we also need to combine the arena allocators
+	for (auto &stored_allocator : other.stored_allocators) {
+		stored_allocators.push_back(stored_allocator);
+	}
+	stored_allocators.push_back(other.aggregate_allocator);
 }
 
 void GroupedAggregateHashTable::Append(GroupedAggregateHashTable &other) {

package/src/duckdb/src/execution/perfect_aggregate_hashtable.cpp

@@ -12,7 +12,8 @@ PerfectAggregateHashTable::PerfectAggregateHashTable(ClientContext &context, All
                                                      vector<Value> group_minima_p, vector<idx_t> required_bits_p)
     : BaseAggregateHashTable(context, allocator, aggregate_objects_p, std::move(payload_types_p)),
       addresses(LogicalType::POINTER), required_bits(std::move(required_bits_p)), total_required_bits(0),
-      group_minima(std::move(group_minima_p)), sel(STANDARD_VECTOR_SIZE), aggregate_allocator(allocator) {
+      group_minima(std::move(group_minima_p)), sel(STANDARD_VECTOR_SIZE),
+      aggregate_allocator(make_uniq<ArenaAllocator>(allocator)) {
 	for (auto &group_bits : required_bits) {
 		total_required_bits += group_bits;
 	}
@@ -136,7 +137,7 @@ void PerfectAggregateHashTable::AddChunk(DataChunk &groups, DataChunk &payload)
 	// after finding the group location we update the aggregates
 	idx_t payload_idx = 0;
 	auto &aggregates = layout.GetAggregates();
-	RowOperationsState row_state(aggregate_allocator);
+	RowOperationsState row_state(*aggregate_allocator);
 	for (idx_t aggr_idx = 0; aggr_idx < aggregates.size(); aggr_idx++) {
 		auto &aggregate = aggregates[aggr_idx];
 		auto input_count = (idx_t)aggregate.child_count;
@@ -165,7 +166,7 @@ void PerfectAggregateHashTable::Combine(PerfectAggregateHashTable &other) {
 	data_ptr_t source_ptr = other.data;
 	data_ptr_t target_ptr = data;
 	idx_t combine_count = 0;
-	RowOperationsState row_state(aggregate_allocator);
+	RowOperationsState row_state(*aggregate_allocator);
 	for (idx_t i = 0; i < total_groups; i++) {
 		auto has_entry_source = other.group_is_set[i];
 		// we only have any work to do if the source has an entry for this group
@@ -183,6 +184,11 @@ void PerfectAggregateHashTable::Combine(PerfectAggregateHashTable &other) {
 		target_ptr += tuple_size;
 	}
 	RowOperations::CombineStates(row_state, layout, source_addresses, target_addresses, combine_count);
+
+	// FIXME: after moving the arena allocator, we currently have to ensure that the pointer is not nullptr, because the
+	// FIXME: Destroy()-function of the hash table expects an allocator in some cases (e.g., for sorted aggregates)
+	stored_allocators.push_back(std::move(other.aggregate_allocator));
+	other.aggregate_allocator = make_uniq<ArenaAllocator>(allocator);
 }
 
 template <class T>
@@ -268,7 +274,7 @@ void PerfectAggregateHashTable::Scan(idx_t &scan_position, DataChunk &result) {
 	}
 	// then construct the payloads
 	result.SetCardinality(entry_count);
-	RowOperationsState row_state(aggregate_allocator);
+	RowOperationsState row_state(*aggregate_allocator);
 	RowOperations::FinalizeStates(row_state, layout, addresses, result, grouping_columns);
 }
 
@@ -289,7 +295,7 @@ void PerfectAggregateHashTable::Destroy() {
 	idx_t count = 0;
 
 	// iterate over all initialised slots of the hash table
-	RowOperationsState row_state(aggregate_allocator);
+	RowOperationsState row_state(*aggregate_allocator);
 	data_ptr_t payload_ptr = data;
 	for (idx_t i = 0; i < total_groups; i++) {
 		if (group_is_set[i]) {

package/src/duckdb/src/execution/window_executor.cpp

@@ -204,19 +204,19 @@ static idx_t FindTypedRangeBound(const WindowInputColumn &over, const idx_t orde
 	WindowColumnIterator<T> begin(over, order_begin);
 	WindowColumnIterator<T> end(over, order_end);
 
-	if (order_begin < prev.first && prev.first < order_end) {
-		const auto first = over.GetCell<T>(prev.first);
+	if (order_begin < prev.start && prev.start < order_end) {
+		const auto first = over.GetCell<T>(prev.start);
 		if (!comp(val, first)) {
 			//	prev.first <= val, so we can start further forward
-			begin += (prev.first - order_begin);
+			begin += (prev.start - order_begin);
 		}
 	}
-	if (order_begin <= prev.second && prev.second < order_end) {
-		const auto second = over.GetCell<T>(prev.second);
+	if (order_begin <= prev.end && prev.end < order_end) {
+		const auto second = over.GetCell<T>(prev.end);
 		if (!comp(second, val)) {
 			//	val <= prev.second, so we can end further back
 			// (prev.second is the largest peer)
-			end -= (order_end - prev.second - 1);
+			end -= (order_end - prev.end - 1);
 		}
 	}
 
@@ -278,8 +278,6 @@ static idx_t FindOrderedRangeBound(const WindowInputColumn &over, const OrderTyp
 }
 
 struct WindowBoundariesState {
-	using FrameBounds = std::pair<idx_t, idx_t>;
-
 	static inline bool IsScalar(const unique_ptr<Expression> &expr) {
 		return expr ? expr->IsScalar() : true;
 	}
@@ -375,8 +373,8 @@ void WindowBoundariesState::Update(const idx_t row_idx, const WindowInputColumn
 				}
 
 				//	Reset range hints
-				prev.first = valid_start;
-				prev.second = valid_end;
+				prev.start = valid_start;
+				prev.end = valid_end;
 			}
 		} else if (!is_peer) {
 			peer_start = row_idx;
@@ -427,9 +425,9 @@ void WindowBoundariesState::Update(const idx_t row_idx, const WindowInputColumn
 		if (boundary_start.CellIsNull(chunk_idx)) {
 			window_start = peer_start;
 		} else {
-			prev.first = FindOrderedRangeBound<true>(range_collection, range_sense, valid_start, row_idx,
+			prev.start = FindOrderedRangeBound<true>(range_collection, range_sense, valid_start, row_idx,
 			                                         boundary_start, chunk_idx, prev);
-			window_start = prev.first;
+			window_start = prev.start;
 		}
 		break;
 	}
@@ -437,9 +435,9 @@ void WindowBoundariesState::Update(const idx_t row_idx, const WindowInputColumn
 		if (boundary_start.CellIsNull(chunk_idx)) {
 			window_start = peer_start;
 		} else {
-			prev.first = FindOrderedRangeBound<true>(range_collection, range_sense, row_idx, valid_end, boundary_start,
+			prev.start = FindOrderedRangeBound<true>(range_collection, range_sense, row_idx, valid_end, boundary_start,
 			                                         chunk_idx, prev);
-			window_start = prev.first;
+			window_start = prev.start;
 		}
 		break;
 	}
@@ -472,9 +470,9 @@ void WindowBoundariesState::Update(const idx_t row_idx, const WindowInputColumn
 		if (boundary_end.CellIsNull(chunk_idx)) {
 			window_end = peer_end;
 		} else {
-			prev.second = FindOrderedRangeBound<false>(range_collection, range_sense, valid_start, row_idx,
-			                                           boundary_end, chunk_idx, prev);
-			window_end = prev.second;
+			prev.end = FindOrderedRangeBound<false>(range_collection, range_sense, valid_start, row_idx, boundary_end,
+			                                        chunk_idx, prev);
+			window_end = prev.end;
 		}
 		break;
 	}
@@ -482,9 +480,9 @@ void WindowBoundariesState::Update(const idx_t row_idx, const WindowInputColumn
 		if (boundary_end.CellIsNull(chunk_idx)) {
 			window_end = peer_end;
 		} else {
-			prev.second = FindOrderedRangeBound<false>(range_collection, range_sense, row_idx, valid_end, boundary_end,
-			                                           chunk_idx, prev);
-			window_end = prev.second;
+			prev.end = FindOrderedRangeBound<false>(range_collection, range_sense, row_idx, valid_end, boundary_end,
+			                                        chunk_idx, prev);
+			window_end = prev.end;
 		}
 		break;
 	}

package/src/duckdb/src/function/aggregate/distributive/count.cpp

@@ -39,8 +39,8 @@ struct CountStarFunction : public BaseCountFunction {
 	                   Vector &result, idx_t rid, idx_t bias) {
 		D_ASSERT(input_count == 0);
 		auto data = FlatVector::GetData<RESULT_TYPE>(result);
-		const auto begin = frame.first;
-		const auto end = frame.second;
+		const auto begin = frame.start;
+		const auto end = frame.end;
 		// Slice to any filtered rows
 		if (!filter_mask.AllValid()) {
 			RESULT_TYPE filtered = 0;

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.8.2-dev2320"
+#define DUCKDB_VERSION "0.8.2-dev2399"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "02412e1015"
+#define DUCKDB_SOURCE_ID "20ad35b3fa"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/common/arrow/appender/append_data.hpp

@@ -4,7 +4,7 @@
 #include "duckdb/common/types/vector.hpp"
 #include "duckdb/common/arrow/arrow.hpp"
 #include "duckdb/common/arrow/arrow_buffer.hpp"
-#include "duckdb/common/arrow/arrow_options.hpp"
+#include "duckdb/main/client_properties.hpp"
 #include "duckdb/common/array.hpp"
 
 namespace duckdb {
@@ -26,7 +26,7 @@ typedef void (*finalize_t)(ArrowAppendData &append_data, const LogicalType &type
 // FIXME: we should separate the append state variables from the variables required by the ArrowArray into
 // ArrowAppendState
 struct ArrowAppendData {
-	explicit ArrowAppendData(ArrowOptions &options_p) : options(options_p) {
+	explicit ArrowAppendData(ClientProperties &options_p) : options(options_p) {
 	}
 	// the buffers of the arrow vector
 	ArrowBuffer validity;
@@ -49,7 +49,7 @@ struct ArrowAppendData {
 	duckdb::array<const void *, 3> buffers = {{nullptr, nullptr, nullptr}};
 	vector<ArrowArray *> child_pointers;
 
-	ArrowOptions options;
+	ClientProperties options;
 };
 
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/include/duckdb/common/arrow/appender/varchar_data.hpp

@@ -62,7 +62,7 @@ struct ArrowVarcharData {
 		auto last_offset = offset_data[append_data.row_count];
 		idx_t max_offset = append_data.row_count + to - from;
 		if (max_offset > NumericLimits<uint32_t>::Maximum() &&
-		    append_data.options.offset_size == ArrowOffsetSize::REGULAR) {
+		    append_data.options.arrow_offset_size == ArrowOffsetSize::REGULAR) {
 			throw InvalidInputException("Arrow Appender: The maximum total string size for regular string buffers is "
 			                            "%u but the offset of %lu exceeds this.",
 			                            NumericLimits<uint32_t>::Maximum(), max_offset);

package/src/duckdb/src/include/duckdb/common/arrow/arrow_appender.hpp

@@ -19,7 +19,7 @@ struct ArrowAppendData;
 //! The ArrowAppender class can be used to incrementally construct an arrow array by appending data chunks into it
 class ArrowAppender {
 public:
-	DUCKDB_API ArrowAppender(vector<LogicalType> types, idx_t initial_capacity, ArrowOptions options);
+	DUCKDB_API ArrowAppender(vector<LogicalType> types, idx_t initial_capacity, ClientProperties options);
 	DUCKDB_API ~ArrowAppender();
 
 	//! Append a data chunk to the underlying arrow array
@@ -30,7 +30,8 @@ public:
 public:
 	static void ReleaseArray(ArrowArray *array);
 	static ArrowArray *FinalizeChild(const LogicalType &type, ArrowAppendData &append_data);
-	static unique_ptr<ArrowAppendData> InitializeChild(const LogicalType &type, idx_t capacity, ArrowOptions &options);
+	static unique_ptr<ArrowAppendData> InitializeChild(const LogicalType &type, idx_t capacity,
+	                                                   ClientProperties &options);
 
 private:
 	//! The types of the chunks that will be appended in
@@ -40,7 +41,7 @@ private:
 	//! The total row count that has been appended
 	idx_t row_count = 0;
 
-	ArrowOptions options;
+	ClientProperties options;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/arrow/arrow_converter.hpp

@@ -10,7 +10,7 @@
 
 #include "duckdb/common/types/data_chunk.hpp"
 #include "duckdb/common/arrow/arrow.hpp"
-#include "duckdb/common/arrow/arrow_options.hpp"
+#include "duckdb/main/client_properties.hpp"
 
 struct ArrowSchema;
 
@@ -18,8 +18,8 @@ namespace duckdb {
 
 struct ArrowConverter {
 	DUCKDB_API static void ToArrowSchema(ArrowSchema *out_schema, const vector<LogicalType> &types,
-	                                     const vector<string> &names, const ArrowOptions &options);
-	DUCKDB_API static void ToArrowArray(DataChunk &input, ArrowArray *out_array, ArrowOptions options);
+	                                     const vector<string> &names, const ClientProperties &options);
+	DUCKDB_API static void ToArrowArray(DataChunk &input, ArrowArray *out_array, ClientProperties options);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/arrow/arrow_wrapper.hpp

@@ -11,7 +11,7 @@
 #include "duckdb/common/helper.hpp"
 #include "duckdb/common/preserved_error.hpp"
 #include "duckdb/main/chunk_scan_state.hpp"
-#include "duckdb/common/arrow/arrow_options.hpp"
+#include "duckdb/main/client_properties.hpp"
 
 //! Here we have the internal duckdb classes that interact with Arrow's Internal Header (i.e., duckdb/commons/arrow.hpp)
 namespace duckdb {
@@ -58,9 +58,9 @@ public:
 
 class ArrowUtil {
 public:
-	static bool TryFetchChunk(ChunkScanState &scan_state, ArrowOptions options, idx_t chunk_size, ArrowArray *out,
+	static bool TryFetchChunk(ChunkScanState &scan_state, ClientProperties options, idx_t chunk_size, ArrowArray *out,
 	                          idx_t &result_count, PreservedError &error);
-	static idx_t FetchChunk(ChunkScanState &scan_state, ArrowOptions options, idx_t chunk_size, ArrowArray *out);
+	static idx_t FetchChunk(ChunkScanState &scan_state, ClientProperties options, idx_t chunk_size, ArrowArray *out);
 
 private:
 	static bool TryFetchNext(QueryResult &result, unique_ptr<DataChunk> &out, PreservedError &error);

package/src/duckdb/src/include/duckdb/common/types/list_segment.hpp

@@ -22,14 +22,14 @@ struct ListSegment {
 	ListSegment *next;
 };
 struct LinkedList {
-	LinkedList() {};
+	LinkedList() : total_capacity(0), first_segment(nullptr), last_segment(nullptr) {};
 	LinkedList(idx_t total_capacity_p, ListSegment *first_segment_p, ListSegment *last_segment_p)
 	    : total_capacity(total_capacity_p), first_segment(first_segment_p), last_segment(last_segment_p) {
 	}
 
-	idx_t total_capacity = 0;
-	ListSegment *first_segment = nullptr;
-	ListSegment *last_segment = nullptr;
+	idx_t total_capacity;
+	ListSegment *first_segment;
+	ListSegment *last_segment;
 };
 
 // forward declarations
@@ -37,23 +37,21 @@ struct ListSegmentFunctions;
 typedef ListSegment *(*create_segment_t)(const ListSegmentFunctions &functions, ArenaAllocator &allocator,
                                          uint16_t capacity);
 typedef void (*write_data_to_segment_t)(const ListSegmentFunctions &functions, ArenaAllocator &allocator,
-                                        ListSegment *segment, Vector &input, idx_t &entry_idx, idx_t &count);
+                                        ListSegment *segment, RecursiveUnifiedVectorFormat &input_data,
+                                        idx_t &entry_idx);
 typedef void (*read_data_from_segment_t)(const ListSegmentFunctions &functions, const ListSegment *segment,
                                          Vector &result, idx_t &total_count);
-typedef ListSegment *(*copy_data_from_segment_t)(const ListSegmentFunctions &functions, const ListSegment *source,
-                                                 ArenaAllocator &allocator);
 
 struct ListSegmentFunctions {
 	create_segment_t create_segment;
 	write_data_to_segment_t write_data;
 	read_data_from_segment_t read_data;
-	copy_data_from_segment_t copy_data;
+
 	vector<ListSegmentFunctions> child_functions;
 
-	void AppendRow(ArenaAllocator &allocator, LinkedList &linked_list, Vector &input, idx_t &entry_idx,
-	               idx_t &count) const;
+	void AppendRow(ArenaAllocator &allocator, LinkedList &linked_list, RecursiveUnifiedVectorFormat &input_data,
+	               idx_t &entry_idx) const;
 	void BuildListVector(const LinkedList &linked_list, Vector &result, idx_t &initial_total_count) const;
-	void CopyLinkedList(const LinkedList &source_list, LinkedList &target_list, ArenaAllocator &allocator) const;
 };
 
 void GetSegmentDataFunctions(ListSegmentFunctions &functions, const LogicalType &type);

package/src/duckdb/src/include/duckdb/common/types/vector.hpp

@@ -35,6 +35,11 @@ struct UnifiedVectorFormat {
 	}
 };
 
+struct RecursiveUnifiedVectorFormat {
+	UnifiedVectorFormat unified;
+	vector<RecursiveUnifiedVectorFormat> children;
+};
+
 class VectorCache;
 class VectorStructBuffer;
 class VectorListBuffer;
@@ -140,6 +145,8 @@ public:
 	//! The most common vector types (flat, constant & dictionary) can be converted to the canonical format "for free"
 	//! ToUnifiedFormat was originally called Orrify, as a tribute to Orri Erling who came up with it
 	DUCKDB_API void ToUnifiedFormat(idx_t count, UnifiedVectorFormat &data);
+	//! Recursively calls UnifiedVectorFormat on a vector and its child vectors (for nested types)
+	static void RecursiveToUnifiedFormat(Vector &input, idx_t count, RecursiveUnifiedVectorFormat &data);
 
 	//! Turn the vector into a sequence vector
 	DUCKDB_API void Sequence(int64_t start, int64_t increment, idx_t count);

package/src/duckdb/src/include/duckdb/common/vector_operations/aggregate_executor.hpp

@@ -15,9 +15,14 @@
 
 namespace duckdb {
 
+// structs
 struct AggregateInputData;
-
-typedef std::pair<idx_t, idx_t> FrameBounds;
+struct FrameBounds {
+	FrameBounds() : start(0), end(0) {};
+	FrameBounds(idx_t start, idx_t end) : start(start), end(end) {};
+	idx_t start = 0;
+	idx_t end = 0;
+};
 
 class AggregateExecutor {
 private:

package/src/duckdb/src/include/duckdb/execution/perfect_aggregate_hashtable.hpp

@@ -56,8 +56,10 @@ protected:
 	//! Reused selection vector
 	SelectionVector sel;
 
-	//! The arena allocator used by the aggregates for their internal state
-	ArenaAllocator aggregate_allocator;
+	//! The active arena allocator used by the aggregates for their internal state
+	unique_ptr<ArenaAllocator> aggregate_allocator;
+	//! Owning arena allocators that this HT has data from
+	vector<unique_ptr<ArenaAllocator>> stored_allocators;
 
 private:
 	//! Destroy the perfect aggregate HT (called automatically by the destructor)

package/src/duckdb/src/include/duckdb/execution/window_segment_tree.hpp

@@ -117,8 +117,6 @@ public:
 
 class WindowSegmentTree : public WindowAggregator {
 public:
-	using FrameBounds = std::pair<idx_t, idx_t>;
-
 	WindowSegmentTree(AggregateObject aggr, const LogicalType &result_type, idx_t count, WindowAggregationMode mode_p);
 	~WindowSegmentTree() override;
 

package/src/duckdb/src/include/duckdb/function/aggregate_function.hpp

@@ -41,7 +41,6 @@ typedef void (*aggregate_simple_update_t)(Vector inputs[], AggregateInputData &a
                                           data_ptr_t state, idx_t count);
 
 //! The type used for updating complex windowed aggregate functions (optional)
-typedef std::pair<idx_t, idx_t> FrameBounds;
 typedef void (*aggregate_window_t)(Vector inputs[], const ValidityMask &filter_mask,
                                    AggregateInputData &aggr_input_data, idx_t input_count, data_ptr_t state,
                                    const FrameBounds &frame, const FrameBounds &prev, Vector &result, idx_t rid,

package/src/duckdb/src/include/duckdb/main/capi/capi_internal.hpp

@@ -13,7 +13,6 @@
 #include "duckdb/common/types.hpp"
 #include "duckdb/common/types/data_chunk.hpp"
 #include "duckdb/main/appender.hpp"
-#include "duckdb/common/arrow/arrow_options.hpp"
 
 #include <cstring>
 #include <cassert>
@@ -48,7 +47,7 @@ struct PendingStatementWrapper {
 struct ArrowResultWrapper {
 	unique_ptr<MaterializedQueryResult> result;
 	unique_ptr<DataChunk> current_chunk;
-	ArrowOptions options;
+	ClientProperties options;
 };
 
 struct AppenderWrapper {