duckdb 0.9.3-dev2.0 → 0.10.0

package/src/duckdb/src/common/row_operations/row_heap_scatter.cpp

@@ -1,11 +1,52 @@
 #include "duckdb/common/helper.hpp"
 #include "duckdb/common/row_operations/row_operations.hpp"
 #include "duckdb/common/types/vector.hpp"
+#include "duckdb/common/uhugeint.hpp"
 
 namespace duckdb {
 
 using ValidityBytes = TemplatedValidityMask<uint8_t>;
 
+NestedValidity::NestedValidity(data_ptr_t validitymask_location)
+    : list_validity_location(validitymask_location), struct_validity_locations(nullptr), entry_idx(0), idx_in_entry(0) {
+}
+
+NestedValidity::NestedValidity(data_ptr_t *validitymask_locations, idx_t child_vector_index)
+    : list_validity_location(nullptr), struct_validity_locations(validitymask_locations), entry_idx(0),
+      idx_in_entry(0) {
+	ValidityBytes::GetEntryIndex(child_vector_index, entry_idx, idx_in_entry);
+}
+
+void NestedValidity::SetInvalid(idx_t idx) {
+	if (list_validity_location) {
+		// Is List
+		idx_t list_entry_idx;
+		idx_t list_idx_in_entry;
+		ValidityBytes::GetEntryIndex(idx, list_entry_idx, list_idx_in_entry);
+		const auto bit = ~(1UL << list_idx_in_entry);
+		list_validity_location[list_entry_idx] &= bit;
+	} else {
+		// Is Struct
+		const auto bit = ~(1UL << idx_in_entry);
+		*(struct_validity_locations[idx] + entry_idx) &= bit;
+	}
+}
+
+bool NestedValidity::IsValid(idx_t idx) {
+	if (list_validity_location) {
+		// Is List
+		idx_t list_entry_idx;
+		idx_t list_idx_in_entry;
+		ValidityBytes::GetEntryIndex(idx, list_entry_idx, list_idx_in_entry);
+		const auto bit = (1UL << list_idx_in_entry);
+		return list_validity_location[list_entry_idx] & bit;
+	} else {
+		// Is Struct
+		const auto bit = (1UL << idx_in_entry);
+		return *(struct_validity_locations[idx] + entry_idx) & bit;
+	}
+}
+
 static void ComputeStringEntrySizes(UnifiedVectorFormat &vdata, idx_t entry_sizes[], const idx_t ser_count,
                                     const SelectionVector &sel, const idx_t offset) {
 	auto strings = UnifiedVectorFormat::GetData<string_t>(vdata);
@@ -78,6 +119,49 @@ static void ComputeListEntrySizes(Vector &v, UnifiedVectorFormat &vdata, idx_t e
 	}
 }
 
+static void ComputeArrayEntrySizes(Vector &v, UnifiedVectorFormat &vdata, idx_t entry_sizes[], idx_t ser_count,
+                                   const SelectionVector &sel, idx_t offset) {
+
+	auto array_size = ArrayType::GetSize(v.GetType());
+	auto child_vector = ArrayVector::GetEntry(v);
+
+	idx_t array_entry_sizes[STANDARD_VECTOR_SIZE];
+	const idx_t array_validitymask_size = (array_size + 7) / 8;
+
+	for (idx_t i = 0; i < ser_count; i++) {
+
+		// Validity for the array elements
+		entry_sizes[i] += array_validitymask_size;
+
+		// serialize size of each entry (if non-constant size)
+		if (!TypeIsConstantSize(ArrayType::GetChildType(v.GetType()).InternalType())) {
+			entry_sizes[i] += array_size * sizeof(idx_t);
+		}
+
+		auto elem_idx = sel.get_index(i);
+		auto source_idx = vdata.sel->get_index(elem_idx + offset);
+
+		auto array_start = source_idx * array_size;
+		auto elem_remaining = array_size;
+
+		// the array could span multiple vectors, so we divide it into chunks
+		while (elem_remaining > 0) {
+			auto chunk_size = MinValue(static_cast<idx_t>(STANDARD_VECTOR_SIZE), elem_remaining);
+
+			// compute and add to the total
+			std::fill_n(array_entry_sizes, chunk_size, 0);
+			RowOperations::ComputeEntrySizes(child_vector, array_entry_sizes, chunk_size, chunk_size,
+			                                 *FlatVector::IncrementalSelectionVector(), array_start);
+			for (idx_t arr_elem_idx = 0; arr_elem_idx < chunk_size; arr_elem_idx++) {
+				entry_sizes[i] += array_entry_sizes[arr_elem_idx];
+			}
+			// update for next iteration
+			elem_remaining -= chunk_size;
+			array_start += chunk_size;
+		}
+	}
+}
+
 void RowOperations::ComputeEntrySizes(Vector &v, UnifiedVectorFormat &vdata, idx_t entry_sizes[], idx_t vcount,
                                       idx_t ser_count, const SelectionVector &sel, idx_t offset) {
 	const auto physical_type = v.GetType().InternalType();
@@ -97,6 +181,9 @@ void RowOperations::ComputeEntrySizes(Vector &v, UnifiedVectorFormat &vdata, idx
 		case PhysicalType::LIST:
 			ComputeListEntrySizes(v, vdata, entry_sizes, ser_count, sel, offset);
 			break;
+		case PhysicalType::ARRAY:
+			ComputeArrayEntrySizes(v, vdata, entry_sizes, ser_count, sel, offset);
+			break;
 		default:
 			// LCOV_EXCL_START
 			throw NotImplementedException("Column with variable size type %s cannot be serialized to row-format",
@@ -114,10 +201,11 @@ void RowOperations::ComputeEntrySizes(Vector &v, idx_t entry_sizes[], idx_t vcou
 }
 
 template <class T>
-static void TemplatedHeapScatter(UnifiedVectorFormat &vdata, const SelectionVector &sel, idx_t count, idx_t col_idx,
-                                 data_ptr_t *key_locations, data_ptr_t *validitymask_locations, idx_t offset) {
+static void TemplatedHeapScatter(UnifiedVectorFormat &vdata, const SelectionVector &sel, idx_t count,
+                                 data_ptr_t *key_locations, optional_ptr<NestedValidity> parent_validity,
+                                 idx_t offset) {
 	auto source = UnifiedVectorFormat::GetData<T>(vdata);
-	if (!validitymask_locations) {
+	if (!parent_validity) {
 		for (idx_t i = 0; i < count; i++) {
 			auto idx = sel.get_index(i);
 			auto source_idx = vdata.sel->get_index(idx + offset);
@@ -127,10 +215,6 @@ static void TemplatedHeapScatter(UnifiedVectorFormat &vdata, const SelectionVect
 			key_locations[i] += sizeof(T);
 		}
 	} else {
-		idx_t entry_idx;
-		idx_t idx_in_entry;
-		ValidityBytes::GetEntryIndex(col_idx, entry_idx, idx_in_entry);
-		const auto bit = ~(1UL << idx_in_entry);
 		for (idx_t i = 0; i < count; i++) {
 			auto idx = sel.get_index(i);
 			auto source_idx = vdata.sel->get_index(idx + offset);
@@ -141,19 +225,20 @@ static void TemplatedHeapScatter(UnifiedVectorFormat &vdata, const SelectionVect
 
 			// set the validitymask
 			if (!vdata.validity.RowIsValid(source_idx)) {
-				*(validitymask_locations[i] + entry_idx) &= bit;
+				parent_validity->SetInvalid(i);
 			}
 		}
 	}
 }
 
-static void HeapScatterStringVector(Vector &v, idx_t vcount, const SelectionVector &sel, idx_t ser_count, idx_t col_idx,
-                                    data_ptr_t *key_locations, data_ptr_t *validitymask_locations, idx_t offset) {
+static void HeapScatterStringVector(Vector &v, idx_t vcount, const SelectionVector &sel, idx_t ser_count,
+                                    data_ptr_t *key_locations, optional_ptr<NestedValidity> parent_validity,
+                                    idx_t offset) {
 	UnifiedVectorFormat vdata;
 	v.ToUnifiedFormat(vcount, vdata);
 
 	auto strings = UnifiedVectorFormat::GetData<string_t>(vdata);
-	if (!validitymask_locations) {
+	if (!parent_validity) {
 		for (idx_t i = 0; i < ser_count; i++) {
 			auto idx = sel.get_index(i);
 			auto source_idx = vdata.sel->get_index(idx + offset);
@@ -168,10 +253,6 @@ static void HeapScatterStringVector(Vector &v, idx_t vcount, const SelectionVect
 			}
 		}
 	} else {
-		idx_t entry_idx;
-		idx_t idx_in_entry;
-		ValidityBytes::GetEntryIndex(col_idx, entry_idx, idx_in_entry);
-		const auto bit = ~(1UL << idx_in_entry);
 		for (idx_t i = 0; i < ser_count; i++) {
 			auto idx = sel.get_index(i);
 			auto source_idx = vdata.sel->get_index(idx + offset);
@@ -185,26 +266,21 @@ static void HeapScatterStringVector(Vector &v, idx_t vcount, const SelectionVect
 				key_locations[i] += string_entry.GetSize();
 			} else {
 				// set the validitymask
-				*(validitymask_locations[i] + entry_idx) &= bit;
+				parent_validity->SetInvalid(i);
 			}
 		}
 	}
 }
 
-static void HeapScatterStructVector(Vector &v, idx_t vcount, const SelectionVector &sel, idx_t ser_count, idx_t col_idx,
-                                    data_ptr_t *key_locations, data_ptr_t *validitymask_locations, idx_t offset) {
+static void HeapScatterStructVector(Vector &v, idx_t vcount, const SelectionVector &sel, idx_t ser_count,
+                                    data_ptr_t *key_locations, optional_ptr<NestedValidity> parent_validity,
+                                    idx_t offset) {
 	UnifiedVectorFormat vdata;
 	v.ToUnifiedFormat(vcount, vdata);
 
 	auto &children = StructVector::GetEntries(v);
 	idx_t num_children = children.size();
 
-	// the whole struct itself can be NULL
-	idx_t entry_idx;
-	idx_t idx_in_entry;
-	ValidityBytes::GetEntryIndex(col_idx, entry_idx, idx_in_entry);
-	const auto bit = ~(1UL << idx_in_entry);
-
 	// struct must have a validitymask for its fields
 	const idx_t struct_validitymask_size = (num_children + 7) / 8;
 	data_ptr_t struct_validitymask_locations[STANDARD_VECTOR_SIZE];
@@ -217,30 +293,26 @@ static void HeapScatterStructVector(Vector &v, idx_t vcount, const SelectionVect
 		// set whether the whole struct is null
 		auto idx = sel.get_index(i);
 		auto source_idx = vdata.sel->get_index(idx) + offset;
-		if (validitymask_locations && !vdata.validity.RowIsValid(source_idx)) {
-			*(validitymask_locations[i] + entry_idx) &= bit;
+		if (parent_validity && !vdata.validity.RowIsValid(source_idx)) {
+			parent_validity->SetInvalid(i);
 		}
 	}
 
 	// now serialize the struct vectors
 	for (idx_t i = 0; i < children.size(); i++) {
 		auto &struct_vector = *children[i];
-		RowOperations::HeapScatter(struct_vector, vcount, sel, ser_count, i, key_locations,
-		                           struct_validitymask_locations, offset);
+		NestedValidity struct_validity(struct_validitymask_locations, i);
+		RowOperations::HeapScatter(struct_vector, vcount, sel, ser_count, key_locations, &struct_validity, offset);
 	}
 }
 
-static void HeapScatterListVector(Vector &v, idx_t vcount, const SelectionVector &sel, idx_t ser_count, idx_t col_no,
-                                  data_ptr_t *key_locations, data_ptr_t *validitymask_locations, idx_t offset) {
+static void HeapScatterListVector(Vector &v, idx_t vcount, const SelectionVector &sel, idx_t ser_count,
+                                  data_ptr_t *key_locations, optional_ptr<NestedValidity> parent_validity,
+                                  idx_t offset) {
 	UnifiedVectorFormat vdata;
 	v.ToUnifiedFormat(vcount, vdata);
 
-	idx_t entry_idx;
-	idx_t idx_in_entry;
-	ValidityBytes::GetEntryIndex(col_no, entry_idx, idx_in_entry);
-
 	auto list_data = ListVector::GetData(v);
-
 	auto &child_vector = ListVector::GetEntry(v);
 
 	UnifiedVectorFormat list_vdata;
@@ -254,10 +326,9 @@ static void HeapScatterListVector(Vector &v, idx_t vcount, const SelectionVector
 		auto idx = sel.get_index(i);
 		auto source_idx = vdata.sel->get_index(idx + offset);
 		if (!vdata.validity.RowIsValid(source_idx)) {
-			if (validitymask_locations) {
+			if (parent_validity) {
 				// set the row validitymask for this column to invalid
-				ValidityBytes row_mask(validitymask_locations[i]);
-				row_mask.SetInvalidUnsafe(entry_idx, idx_in_entry);
+				parent_validity->SetInvalid(i);
 			}
 			continue;
 		}
@@ -321,8 +392,8 @@ static void HeapScatterListVector(Vector &v, idx_t vcount, const SelectionVector
 
 			// now serialize to the locations
 			RowOperations::HeapScatter(child_vector, ListVector::GetListSize(v),
-			                           *FlatVector::IncrementalSelectionVector(), next, 0, list_entry_locations,
-			                           nullptr, entry_offset);
+			                           *FlatVector::IncrementalSelectionVector(), next, list_entry_locations, nullptr,
+			                           entry_offset);
 
 			// update for next iteration
 			entry_remaining -= next;
@@ -331,23 +402,110 @@ static void HeapScatterListVector(Vector &v, idx_t vcount, const SelectionVector
 	}
 }
 
-void RowOperations::HeapScatter(Vector &v, idx_t vcount, const SelectionVector &sel, idx_t ser_count, idx_t col_idx,
-                                data_ptr_t *key_locations, data_ptr_t *validitymask_locations, idx_t offset) {
+static void HeapScatterArrayVector(Vector &v, idx_t vcount, const SelectionVector &sel, idx_t ser_count,
+                                   data_ptr_t *key_locations, optional_ptr<NestedValidity> parent_validity,
+                                   idx_t offset) {
+
+	auto &child_vector = ArrayVector::GetEntry(v);
+	auto array_size = ArrayType::GetSize(v.GetType());
+	auto child_type = ArrayType::GetChildType(v.GetType());
+	auto child_type_size = GetTypeIdSize(child_type.InternalType());
+	auto child_type_is_var_size = !TypeIsConstantSize(child_type.InternalType());
+
+	UnifiedVectorFormat vdata;
+	v.ToUnifiedFormat(vcount, vdata);
+
+	UnifiedVectorFormat child_vdata;
+	child_vector.ToUnifiedFormat(ArrayVector::GetTotalSize(v), child_vdata);
+
+	data_ptr_t array_entry_locations[STANDARD_VECTOR_SIZE];
+	idx_t array_entry_sizes[STANDARD_VECTOR_SIZE];
+
+	// array must have a validitymask for its elements
+	auto array_validitymask_size = (array_size + 7) / 8;
+
+	for (idx_t i = 0; i < ser_count; i++) {
+		// Set if the whole array itself is null in the parent entry
+		auto source_idx = vdata.sel->get_index(sel.get_index(i) + offset);
+		if (parent_validity && !vdata.validity.RowIsValid(source_idx)) {
+			parent_validity->SetInvalid(i);
+		}
+
+		// Now we can serialize the array itself
+		// Every array starts with a validity mask for the children
+		data_ptr_t array_validitymask_location = key_locations[i];
+		memset(array_validitymask_location, -1, array_validitymask_size);
+		key_locations[i] += array_validitymask_size;
+
+		// If the array contains variable size entries, we reserve spaces for them here
+		data_ptr_t var_entry_size_ptr = nullptr;
+		if (child_type_is_var_size) {
+			var_entry_size_ptr = key_locations[i];
+			key_locations[i] += array_size * sizeof(idx_t);
+		}
+
+		// Then comes the elements
+		auto array_start = source_idx * array_size;
+		auto elem_remaining = array_size;
+
+		while (elem_remaining > 0) {
+			// the array elements can span multiple vectors, so we divide it into chunks
+			auto chunk_size = MinValue(static_cast<idx_t>(STANDARD_VECTOR_SIZE), elem_remaining);
+
+			// Setup the locations for the elements
+			if (child_type_is_var_size) {
+				// The elements are variable sized
+				std::fill_n(array_entry_sizes, chunk_size, 0);
+				RowOperations::ComputeEntrySizes(child_vector, array_entry_sizes, chunk_size, chunk_size,
+				                                 *FlatVector::IncrementalSelectionVector(), array_start);
+				for (idx_t elem_idx = 0; elem_idx < chunk_size; elem_idx++) {
+					array_entry_locations[elem_idx] = key_locations[i];
+					key_locations[i] += array_entry_sizes[elem_idx];
+
+					// Now store the size of the entry
+					Store<idx_t>(array_entry_sizes[elem_idx], var_entry_size_ptr);
+					var_entry_size_ptr += sizeof(idx_t);
+				}
+			} else {
+				// The elements are constant sized
+				for (idx_t elem_idx = 0; elem_idx < chunk_size; elem_idx++) {
+					array_entry_locations[elem_idx] = key_locations[i];
+					key_locations[i] += child_type_size;
+				}
+			}
+
+			NestedValidity array_parent_validity(array_validitymask_location);
+			RowOperations::HeapScatter(child_vector, ArrayVector::GetTotalSize(v),
+			                           *FlatVector::IncrementalSelectionVector(), chunk_size, array_entry_locations,
+			                           &array_parent_validity, array_start);
+
+			// update for next iteration
+			elem_remaining -= chunk_size;
+			array_start += chunk_size;
+		}
+	}
+}
+
+void RowOperations::HeapScatter(Vector &v, idx_t vcount, const SelectionVector &sel, idx_t ser_count,
+                                data_ptr_t *key_locations, optional_ptr<NestedValidity> parent_validity, idx_t offset) {
 	if (TypeIsConstantSize(v.GetType().InternalType())) {
 		UnifiedVectorFormat vdata;
 		v.ToUnifiedFormat(vcount, vdata);
-		RowOperations::HeapScatterVData(vdata, v.GetType().InternalType(), sel, ser_count, col_idx, key_locations,
-		                                validitymask_locations, offset);
+		RowOperations::HeapScatterVData(vdata, v.GetType().InternalType(), sel, ser_count, key_locations,
+		                                parent_validity, offset);
 	} else {
 		switch (v.GetType().InternalType()) {
 		case PhysicalType::VARCHAR:
-			HeapScatterStringVector(v, vcount, sel, ser_count, col_idx, key_locations, validitymask_locations, offset);
+			HeapScatterStringVector(v, vcount, sel, ser_count, key_locations, parent_validity, offset);
 			break;
 		case PhysicalType::STRUCT:
-			HeapScatterStructVector(v, vcount, sel, ser_count, col_idx, key_locations, validitymask_locations, offset);
+			HeapScatterStructVector(v, vcount, sel, ser_count, key_locations, parent_validity, offset);
 			break;
 		case PhysicalType::LIST:
-			HeapScatterListVector(v, vcount, sel, ser_count, col_idx, key_locations, validitymask_locations, offset);
+			HeapScatterListVector(v, vcount, sel, ser_count, key_locations, parent_validity, offset);
+			break;
+		case PhysicalType::ARRAY:
+			HeapScatterArrayVector(v, vcount, sel, ser_count, key_locations, parent_validity, offset);
 			break;
 		default:
 			// LCOV_EXCL_START
@@ -359,45 +517,48 @@ void RowOperations::HeapScatter(Vector &v, idx_t vcount, const SelectionVector &
 }
 
 void RowOperations::HeapScatterVData(UnifiedVectorFormat &vdata, PhysicalType type, const SelectionVector &sel,
-                                     idx_t ser_count, idx_t col_idx, data_ptr_t *key_locations,
-                                     data_ptr_t *validitymask_locations, idx_t offset) {
+                                     idx_t ser_count, data_ptr_t *key_locations,
+                                     optional_ptr<NestedValidity> parent_validity, idx_t offset) {
 	switch (type) {
 	case PhysicalType::BOOL:
 	case PhysicalType::INT8:
-		TemplatedHeapScatter<int8_t>(vdata, sel, ser_count, col_idx, key_locations, validitymask_locations, offset);
+		TemplatedHeapScatter<int8_t>(vdata, sel, ser_count, key_locations, parent_validity, offset);
 		break;
 	case PhysicalType::INT16:
-		TemplatedHeapScatter<int16_t>(vdata, sel, ser_count, col_idx, key_locations, validitymask_locations, offset);
+		TemplatedHeapScatter<int16_t>(vdata, sel, ser_count, key_locations, parent_validity, offset);
 		break;
 	case PhysicalType::INT32:
-		TemplatedHeapScatter<int32_t>(vdata, sel, ser_count, col_idx, key_locations, validitymask_locations, offset);
+		TemplatedHeapScatter<int32_t>(vdata, sel, ser_count, key_locations, parent_validity, offset);
 		break;
 	case PhysicalType::INT64:
-		TemplatedHeapScatter<int64_t>(vdata, sel, ser_count, col_idx, key_locations, validitymask_locations, offset);
+		TemplatedHeapScatter<int64_t>(vdata, sel, ser_count, key_locations, parent_validity, offset);
 		break;
 	case PhysicalType::UINT8:
-		TemplatedHeapScatter<uint8_t>(vdata, sel, ser_count, col_idx, key_locations, validitymask_locations, offset);
+		TemplatedHeapScatter<uint8_t>(vdata, sel, ser_count, key_locations, parent_validity, offset);
 		break;
 	case PhysicalType::UINT16:
-		TemplatedHeapScatter<uint16_t>(vdata, sel, ser_count, col_idx, key_locations, validitymask_locations, offset);
+		TemplatedHeapScatter<uint16_t>(vdata, sel, ser_count, key_locations, parent_validity, offset);
 		break;
 	case PhysicalType::UINT32:
-		TemplatedHeapScatter<uint32_t>(vdata, sel, ser_count, col_idx, key_locations, validitymask_locations, offset);
+		TemplatedHeapScatter<uint32_t>(vdata, sel, ser_count, key_locations, parent_validity, offset);
 		break;
 	case PhysicalType::UINT64:
-		TemplatedHeapScatter<uint64_t>(vdata, sel, ser_count, col_idx, key_locations, validitymask_locations, offset);
+		TemplatedHeapScatter<uint64_t>(vdata, sel, ser_count, key_locations, parent_validity, offset);
 		break;
 	case PhysicalType::INT128:
-		TemplatedHeapScatter<hugeint_t>(vdata, sel, ser_count, col_idx, key_locations, validitymask_locations, offset);
+		TemplatedHeapScatter<hugeint_t>(vdata, sel, ser_count, key_locations, parent_validity, offset);
+		break;
+	case PhysicalType::UINT128:
+		TemplatedHeapScatter<uhugeint_t>(vdata, sel, ser_count, key_locations, parent_validity, offset);
 		break;
 	case PhysicalType::FLOAT:
-		TemplatedHeapScatter<float>(vdata, sel, ser_count, col_idx, key_locations, validitymask_locations, offset);
+		TemplatedHeapScatter<float>(vdata, sel, ser_count, key_locations, parent_validity, offset);
 		break;
 	case PhysicalType::DOUBLE:
-		TemplatedHeapScatter<double>(vdata, sel, ser_count, col_idx, key_locations, validitymask_locations, offset);
+		TemplatedHeapScatter<double>(vdata, sel, ser_count, key_locations, parent_validity, offset);
 		break;
 	case PhysicalType::INTERVAL:
-		TemplatedHeapScatter<interval_t>(vdata, sel, ser_count, col_idx, key_locations, validitymask_locations, offset);
+		TemplatedHeapScatter<interval_t>(vdata, sel, ser_count, key_locations, parent_validity, offset);
 		break;
 	default:
 		throw NotImplementedException("FIXME: Serialize to of constant type column to row-format");

package/src/duckdb/src/common/row_operations/row_matcher.cpp

@@ -176,7 +176,7 @@ static idx_t GenericNestedMatch(Vector &lhs_vector, const TupleDataVectorFormat
 	Vector key(type);
 	const auto gather_function = TupleDataCollection::GetGatherFunction(type);
 	gather_function.function(rhs_layout, rhs_row_locations, col_idx, sel, count, key,
-	                         *FlatVector::IncrementalSelectionVector(), key, gather_function.child_functions);
+	                         *FlatVector::IncrementalSelectionVector(), nullptr, gather_function.child_functions);
 
 	// Densify the input column
 	Vector sliced(lhs_vector, sel, count);
@@ -238,6 +238,8 @@ MatchFunction RowMatcher::GetMatchFunction(const LogicalType &type, const Expres
 		return GetMatchFunction<NO_MATCH_SEL, uint32_t>(predicate);
 	case PhysicalType::UINT64:
 		return GetMatchFunction<NO_MATCH_SEL, uint64_t>(predicate);
+	case PhysicalType::UINT128:
+		return GetMatchFunction<NO_MATCH_SEL, uhugeint_t>(predicate);
 	case PhysicalType::FLOAT:
 		return GetMatchFunction<NO_MATCH_SEL, float>(predicate);
 	case PhysicalType::DOUBLE:
@@ -250,6 +252,9 @@ MatchFunction RowMatcher::GetMatchFunction(const LogicalType &type, const Expres
 		return GetStructMatchFunction<NO_MATCH_SEL>(type, predicate);
 	case PhysicalType::LIST:
 		return GetListMatchFunction<NO_MATCH_SEL>(predicate);
+	case PhysicalType::ARRAY:
+		// Same logic as for lists
+		return GetListMatchFunction<NO_MATCH_SEL>(predicate);
 	default:
 		throw InternalException("Unsupported PhysicalType for RowMatcher::GetMatchFunction: %s",
 		                        EnumUtil::ToString(type.InternalType()));

package/src/duckdb/src/common/row_operations/row_radix_scatter.cpp

@@ -2,6 +2,7 @@
 #include "duckdb/common/radix.hpp"
 #include "duckdb/common/row_operations/row_operations.hpp"
 #include "duckdb/common/types/vector.hpp"
+#include "duckdb/common/uhugeint.hpp"
 
 namespace duckdb {
 
@@ -173,6 +174,49 @@ void RadixScatterListVector(Vector &v, UnifiedVectorFormat &vdata, const Selecti
 	}
 }
 
+void RadixScatterArrayVector(Vector &v, UnifiedVectorFormat &vdata, idx_t vcount, const SelectionVector &sel,
+                             idx_t add_count, data_ptr_t *key_locations, const bool desc, const bool has_null,
+                             const bool nulls_first, const idx_t prefix_len, idx_t width, const idx_t offset) {
+	// serialize null values
+	if (has_null) {
+		auto &validity = vdata.validity;
+		const data_t valid = nulls_first ? 1 : 0;
+		const data_t invalid = 1 - valid;
+
+		for (idx_t i = 0; i < add_count; i++) {
+			auto idx = sel.get_index(i);
+			auto source_idx = vdata.sel->get_index(idx) + offset;
+			// write validity and according value
+			if (validity.RowIsValid(source_idx)) {
+				key_locations[i][0] = valid;
+			} else {
+				key_locations[i][0] = invalid;
+			}
+			key_locations[i]++;
+		}
+		width--;
+	}
+
+	// serialize the inner child
+	auto &child_vector = ArrayVector::GetEntry(v);
+	auto array_size = ArrayType::GetSize(v.GetType());
+	for (idx_t i = 0; i < add_count; i++) {
+		auto idx = sel.get_index(i);
+		auto source_idx = vdata.sel->get_index(idx) + offset;
+		auto array_offset = source_idx * array_size;
+		data_ptr_t key_location = key_locations[i];
+
+		RowOperations::RadixScatter(child_vector, array_size, *FlatVector::IncrementalSelectionVector(), 1,
+		                            key_locations + i, false, true, false, prefix_len, width - 1, array_offset);
+		// invert bits if desc
+		if (desc) {
+			for (idx_t s = 0; s < width; s++) {
+				*(key_location + s) = ~*(key_location + s);
+			}
+		}
+	}
+}
+
 void RadixScatterStructVector(Vector &v, UnifiedVectorFormat &vdata, idx_t vcount, const SelectionVector &sel,
                               idx_t add_count, data_ptr_t *key_locations, const bool desc, const bool has_null,
                               const bool nulls_first, const idx_t prefix_len, idx_t width, const idx_t offset) {
@@ -243,6 +287,9 @@ void RowOperations::RadixScatter(Vector &v, idx_t vcount, const SelectionVector
 	case PhysicalType::INT128:
 		TemplatedRadixScatter<hugeint_t>(vdata, sel, ser_count, key_locations, desc, has_null, nulls_first, offset);
 		break;
+	case PhysicalType::UINT128:
+		TemplatedRadixScatter<uhugeint_t>(vdata, sel, ser_count, key_locations, desc, has_null, nulls_first, offset);
+		break;
 	case PhysicalType::FLOAT:
 		TemplatedRadixScatter<float>(vdata, sel, ser_count, key_locations, desc, has_null, nulls_first, offset);
 		break;
@@ -263,6 +310,10 @@ void RowOperations::RadixScatter(Vector &v, idx_t vcount, const SelectionVector
 		RadixScatterStructVector(v, vdata, vcount, sel, ser_count, key_locations, desc, has_null, nulls_first,
 		                         prefix_len, width, offset);
 		break;
+	case PhysicalType::ARRAY:
+		RadixScatterArrayVector(v, vdata, vcount, sel, ser_count, key_locations, desc, has_null, nulls_first,
+		                        prefix_len, width, offset);
+		break;
 	default:
 		throw NotImplementedException("Cannot ORDER BY column with type %s", v.GetType().ToString());
 	}

package/src/duckdb/src/common/row_operations/row_scatter.cpp

@@ -12,6 +12,7 @@
 #include "duckdb/common/types/row/row_layout.hpp"
 #include "duckdb/common/types/selection_vector.hpp"
 #include "duckdb/common/types/vector.hpp"
+#include "duckdb/common/uhugeint.hpp"
 
 namespace duckdb {
 
@@ -106,7 +107,8 @@ static void ScatterNestedVector(Vector &vec, UnifiedVectorFormat &col, Vector &r
 	}
 
 	// Serialise the data
-	RowOperations::HeapScatter(vec, vcount, sel, count, col_no, data_locations, validitymask_locations);
+	NestedValidity parent_validity(validitymask_locations, col_no);
+	RowOperations::HeapScatter(vec, vcount, sel, count, data_locations, &parent_validity);
 }
 
 void RowOperations::Scatter(DataChunk &columns, UnifiedVectorFormat col_data[], const RowLayout &layout, Vector &rows,
@@ -146,6 +148,7 @@ void RowOperations::Scatter(DataChunk &columns, UnifiedVectorFormat col_data[],
 				break;
 			case PhysicalType::LIST:
 			case PhysicalType::STRUCT:
+			case PhysicalType::ARRAY:
 				RowOperations::ComputeEntrySizes(vec, col, entry_sizes, vcount, count, sel);
 				break;
 			default:
@@ -203,6 +206,9 @@ void RowOperations::Scatter(DataChunk &columns, UnifiedVectorFormat col_data[],
 		case PhysicalType::INT128:
 			TemplatedScatter<hugeint_t>(col, rows, sel, count, col_offset, col_no);
 			break;
+		case PhysicalType::UINT128:
+			TemplatedScatter<uhugeint_t>(col, rows, sel, count, col_offset, col_no);
+			break;
 		case PhysicalType::FLOAT:
 			TemplatedScatter<float>(col, rows, sel, count, col_offset, col_no);
 			break;
@@ -217,6 +223,7 @@ void RowOperations::Scatter(DataChunk &columns, UnifiedVectorFormat col_data[],
 			break;
 		case PhysicalType::LIST:
 		case PhysicalType::STRUCT:
+		case PhysicalType::ARRAY:
 			ScatterNestedVector(vec, col, rows, data_locations, sel, count, col_offset, col_no, vcount);
 			break;
 		default:

package/src/duckdb/src/common/serializer/binary_deserializer.cpp

@@ -124,6 +124,12 @@ hugeint_t BinaryDeserializer::ReadHugeInt() {
 	return hugeint_t(upper, lower);
 }
 
+uhugeint_t BinaryDeserializer::ReadUhugeInt() {
+	auto upper = VarIntDecode<uint64_t>();
+	auto lower = VarIntDecode<uint64_t>();
+	return uhugeint_t(upper, lower);
+}
+
 void BinaryDeserializer::ReadDataPtr(data_ptr_t &ptr_p, idx_t count) {
 	auto len = VarIntDecode<uint64_t>();
 	if (len != count) {

package/src/duckdb/src/common/serializer/binary_serializer.cpp

@@ -136,6 +136,11 @@ void BinarySerializer::WriteValue(hugeint_t value) {
 	VarIntEncode(value.lower);
 }
 
+void BinarySerializer::WriteValue(uhugeint_t value) {
+	VarIntEncode(value.upper);
+	VarIntEncode(value.lower);
+}
+
 void BinarySerializer::WriteValue(float value) {
 	Write(value);
 }

package/src/duckdb/src/common/serializer/serializer.cpp

@@ -1,4 +1,5 @@
 #include "duckdb/common/serializer/serializer.hpp"
+#include "duckdb/common/types/value.hpp"
 
 namespace duckdb {
 
@@ -12,4 +13,22 @@ void Serializer::WriteValue(const vector<bool> &vec) {
 	OnListEnd();
 }
 
+template <>
+void Serializer::WritePropertyWithDefault<Value>(const field_id_t field_id, const char *tag, const Value &value,
+                                                 const Value &&default_value) {
+	// If current value is default, don't write it
+	if (!serialize_default_values && ValueOperations::NotDistinctFrom(value, default_value)) {
+		OnOptionalPropertyBegin(field_id, tag, false);
+		OnOptionalPropertyEnd(false);
+		return;
+	}
+	OnOptionalPropertyBegin(field_id, tag, true);
+	WriteValue(value);
+	OnOptionalPropertyEnd(true);
+}
+
+void Serializer::List::WriteElement(data_ptr_t ptr, idx_t size) {
+	serializer.WriteDataPtr(ptr, size);
+}
+
 } // namespace duckdb