duckdb 0.9.3-dev2.0 → 0.10.0

package/src/duckdb/src/function/table/arrow_conversion.cpp

@@ -4,9 +4,26 @@
 #include "duckdb/common/types/hugeint.hpp"
 #include "duckdb/common/types/arrow_aux_data.hpp"
 #include "duckdb/function/scalar/nested_functions.hpp"
+#include "duckdb/common/exception/conversion_exception.hpp"
 
 namespace duckdb {
 
+namespace {
+
+enum class ArrowArrayPhysicalType : uint8_t { DICTIONARY_ENCODED, RUN_END_ENCODED, DEFAULT };
+
+ArrowArrayPhysicalType GetArrowArrayPhysicalType(const ArrowType &type) {
+	if (type.HasDictionary()) {
+		return ArrowArrayPhysicalType::DICTIONARY_ENCODED;
+	}
+	if (type.RunEndEncoded()) {
+		return ArrowArrayPhysicalType::RUN_END_ENCODED;
+	}
+	return ArrowArrayPhysicalType::DEFAULT;
+}
+
+} // namespace
+
 static void ShiftRight(unsigned char *ar, int size, int shift) {
 	int carry = 0;
 	while (shift--) {
@@ -18,13 +35,26 @@ static void ShiftRight(unsigned char *ar, int size, int shift) {
 	}
 }
 
+idx_t GetEffectiveOffset(ArrowArray &array, int64_t parent_offset, const ArrowScanLocalState &state,
+                         int64_t nested_offset = -1) {
+	if (nested_offset != -1) {
+		// The parent of this array is a list
+		// We just ignore the parent offset, it's already applied to the list
+		return array.offset + nested_offset;
+	}
+	// Parent offset is set in the case of a struct, it applies to all child arrays
+	// 'chunk_offset' is how much of the chunk we've already scanned, in case the chunk size exceeds
+	// STANDARD_VECTOR_SIZE
+	return array.offset + parent_offset + state.chunk_offset;
+}
+
 template <class T>
 T *ArrowBufferData(ArrowArray &array, idx_t buffer_idx) {
 	return (T *)array.buffers[buffer_idx]; // NOLINT
 }
 
-static void GetValidityMask(ValidityMask &mask, ArrowArray &array, ArrowScanLocalState &scan_state, idx_t size,
-                            int64_t nested_offset = -1, bool add_null = false) {
+static void GetValidityMask(ValidityMask &mask, ArrowArray &array, const ArrowScanLocalState &scan_state, idx_t size,
+                            int64_t parent_offset, int64_t nested_offset = -1, bool add_null = false) {
 	// In certains we don't need to or cannot copy arrow's validity mask to duckdb.
 	//
 	// The conditions where we do want to copy arrow's mask to duckdb are:
@@ -32,10 +62,7 @@ static void GetValidityMask(ValidityMask &mask, ArrowArray &array, ArrowScanLoca
 	// 2. n_buffers > 0, meaning the array's arrow type is not `null`
 	// 3. the validity buffer (the first buffer) is not a nullptr
 	if (array.null_count != 0 && array.n_buffers > 0 && array.buffers[0]) {
-		auto bit_offset = scan_state.chunk_offset + array.offset;
-		if (nested_offset != -1) {
-			bit_offset = nested_offset;
-		}
+		auto bit_offset = GetEffectiveOffset(array, parent_offset, scan_state, nested_offset);
 		mask.EnsureWritable();
 #if STANDARD_VECTOR_SIZE > 64
 		auto n_bitmask_bytes = (size + 8 - 1) / 8;
@@ -73,13 +100,17 @@ static void GetValidityMask(ValidityMask &mask, ArrowArray &array, ArrowScanLoca
 	}
 }
 
-static void SetValidityMask(Vector &vector, ArrowArray &array, ArrowScanLocalState &scan_state, idx_t size,
-                            int64_t nested_offset, bool add_null = false) {
+static void SetValidityMask(Vector &vector, ArrowArray &array, const ArrowScanLocalState &scan_state, idx_t size,
+                            int64_t parent_offset, int64_t nested_offset, bool add_null = false) {
 	D_ASSERT(vector.GetVectorType() == VectorType::FLAT_VECTOR);
 	auto &mask = FlatVector::Validity(vector);
-	GetValidityMask(mask, array, scan_state, size, nested_offset, add_null);
+	GetValidityMask(mask, array, scan_state, size, parent_offset, nested_offset, add_null);
 }
 
+static void ColumnArrowToDuckDBRunEndEncoded(Vector &vector, ArrowArray &array, ArrowArrayScanState &array_state,
+                                             idx_t size, const ArrowType &arrow_type, int64_t nested_offset = -1,
+                                             ValidityMask *parent_mask = nullptr, uint64_t parent_offset = 0);
+
 static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArrayScanState &array_state, idx_t size,
                                 const ArrowType &arrow_type, int64_t nested_offset = -1,
                                 ValidityMask *parent_mask = nullptr, uint64_t parent_offset = 0);
@@ -89,21 +120,19 @@ static void ColumnArrowToDuckDBDictionary(Vector &vector, ArrowArray &array, Arr
                                           ValidityMask *parent_mask = nullptr, uint64_t parent_offset = 0);
 
 static void ArrowToDuckDBList(Vector &vector, ArrowArray &array, ArrowArrayScanState &array_state, idx_t size,
-                              const ArrowType &arrow_type, int64_t nested_offset, ValidityMask *parent_mask) {
+                              const ArrowType &arrow_type, int64_t nested_offset, ValidityMask *parent_mask,
+                              int64_t parent_offset) {
 	auto size_type = arrow_type.GetSizeType();
 	idx_t list_size = 0;
 	auto &scan_state = array_state.state;
 
-	SetValidityMask(vector, array, scan_state, size, nested_offset);
+	SetValidityMask(vector, array, scan_state, size, parent_offset, nested_offset);
 	idx_t start_offset = 0;
 	idx_t cur_offset = 0;
 	if (size_type == ArrowVariableSizeType::FIXED_SIZE) {
 		auto fixed_size = arrow_type.FixedSize();
 		//! Have to check validity mask before setting this up
-		idx_t offset = (scan_state.chunk_offset + array.offset) * fixed_size;
-		if (nested_offset != -1) {
-			offset = fixed_size * nested_offset;
-		}
+		idx_t offset = GetEffectiveOffset(array, parent_offset, scan_state, nested_offset) * fixed_size;
 		start_offset = offset;
 		auto list_data = FlatVector::GetData<list_entry_t>(vector);
 		for (idx_t i = 0; i < size; i++) {
@@ -114,10 +143,8 @@ static void ArrowToDuckDBList(Vector &vector, ArrowArray &array, ArrowArrayScanS
 		}
 		list_size = start_offset + cur_offset;
 	} else if (size_type == ArrowVariableSizeType::NORMAL) {
-		auto offsets = ArrowBufferData<uint32_t>(array, 1) + array.offset + scan_state.chunk_offset;
-		if (nested_offset != -1) {
-			offsets = ArrowBufferData<uint32_t>(array, 1) + nested_offset;
-		}
+		auto offsets =
+		    ArrowBufferData<uint32_t>(array, 1) + GetEffectiveOffset(array, parent_offset, scan_state, nested_offset);
 		start_offset = offsets[0];
 		auto list_data = FlatVector::GetData<list_entry_t>(vector);
 		for (idx_t i = 0; i < size; i++) {
@@ -128,10 +155,8 @@ static void ArrowToDuckDBList(Vector &vector, ArrowArray &array, ArrowArrayScanS
 		}
 		list_size = offsets[size];
 	} else {
-		auto offsets = ArrowBufferData<uint64_t>(array, 1) + array.offset + scan_state.chunk_offset;
-		if (nested_offset != -1) {
-			offsets = ArrowBufferData<uint64_t>(array, 1) + nested_offset;
-		}
+		auto offsets =
+		    ArrowBufferData<uint64_t>(array, 1) + GetEffectiveOffset(array, parent_offset, scan_state, nested_offset);
 		start_offset = offsets[0];
 		auto list_data = FlatVector::GetData<list_entry_t>(vector);
 		for (idx_t i = 0; i < size; i++) {
@@ -146,7 +171,7 @@ static void ArrowToDuckDBList(Vector &vector, ArrowArray &array, ArrowArrayScanS
 	ListVector::Reserve(vector, list_size);
 	ListVector::SetListSize(vector, list_size);
 	auto &child_vector = ListVector::GetEntry(vector);
-	SetValidityMask(child_vector, *array.children[0], scan_state, list_size, start_offset);
+	SetValidityMask(child_vector, *array.children[0], scan_state, list_size, array.offset, start_offset);
 	auto &list_mask = FlatVector::Validity(vector);
 	if (parent_mask) {
 		//! Since this List is owned by a struct we must guarantee their validity map matches on Null
@@ -164,27 +189,34 @@ static void ArrowToDuckDBList(Vector &vector, ArrowArray &array, ArrowArrayScanS
 	if (list_size == 0 && start_offset == 0) {
 		D_ASSERT(!child_array.dictionary);
 		ColumnArrowToDuckDB(child_vector, child_array, child_state, list_size, child_type, -1);
-	} else {
-		if (child_array.dictionary) {
-			// TODO: add support for offsets
-			ColumnArrowToDuckDBDictionary(child_vector, child_array, child_state, list_size, child_type, start_offset);
-		} else {
-			ColumnArrowToDuckDB(child_vector, child_array, child_state, list_size, child_type, start_offset);
-		}
+		return;
+	}
+
+	auto array_physical_type = GetArrowArrayPhysicalType(child_type);
+	switch (array_physical_type) {
+	case ArrowArrayPhysicalType::DICTIONARY_ENCODED:
+		// TODO: add support for offsets
+		ColumnArrowToDuckDBDictionary(child_vector, child_array, child_state, list_size, child_type, start_offset);
+		break;
+	case ArrowArrayPhysicalType::RUN_END_ENCODED:
+		ColumnArrowToDuckDBRunEndEncoded(child_vector, child_array, child_state, list_size, child_type, start_offset);
+		break;
+	case ArrowArrayPhysicalType::DEFAULT:
+		ColumnArrowToDuckDB(child_vector, child_array, child_state, list_size, child_type, start_offset);
+		break;
+	default:
+		throw NotImplementedException("ArrowArrayPhysicalType not recognized");
 	}
 }
 
-static void ArrowToDuckDBBlob(Vector &vector, ArrowArray &array, ArrowScanLocalState &scan_state, idx_t size,
-                              const ArrowType &arrow_type, int64_t nested_offset) {
+static void ArrowToDuckDBBlob(Vector &vector, ArrowArray &array, const ArrowScanLocalState &scan_state, idx_t size,
+                              const ArrowType &arrow_type, int64_t nested_offset, int64_t parent_offset) {
 	auto size_type = arrow_type.GetSizeType();
-	SetValidityMask(vector, array, scan_state, size, nested_offset);
+	SetValidityMask(vector, array, scan_state, size, parent_offset, nested_offset);
 	if (size_type == ArrowVariableSizeType::FIXED_SIZE) {
 		auto fixed_size = arrow_type.FixedSize();
 		//! Have to check validity mask before setting this up
-		idx_t offset = (scan_state.chunk_offset + array.offset) * fixed_size;
-		if (nested_offset != -1) {
-			offset = fixed_size * nested_offset;
-		}
+		idx_t offset = GetEffectiveOffset(array, parent_offset, scan_state, nested_offset) * fixed_size;
 		auto cdata = ArrowBufferData<char>(array, 1);
 		for (idx_t row_idx = 0; row_idx < size; row_idx++) {
 			if (FlatVector::IsNull(vector, row_idx)) {
@@ -196,10 +228,8 @@ static void ArrowToDuckDBBlob(Vector &vector, ArrowArray &array, ArrowScanLocalS
 			offset += blob_len;
 		}
 	} else if (size_type == ArrowVariableSizeType::NORMAL) {
-		auto offsets = ArrowBufferData<uint32_t>(array, 1) + array.offset + scan_state.chunk_offset;
-		if (nested_offset != -1) {
-			offsets = ArrowBufferData<uint32_t>(array, 1) + array.offset + nested_offset;
-		}
+		auto offsets =
+		    ArrowBufferData<uint32_t>(array, 1) + GetEffectiveOffset(array, parent_offset, scan_state, nested_offset);
 		auto cdata = ArrowBufferData<char>(array, 2);
 		for (idx_t row_idx = 0; row_idx < size; row_idx++) {
 			if (FlatVector::IsNull(vector, row_idx)) {
@@ -214,10 +244,8 @@ static void ArrowToDuckDBBlob(Vector &vector, ArrowArray &array, ArrowScanLocalS
 		if (ArrowBufferData<uint64_t>(array, 1)[array.length] > NumericLimits<uint32_t>::Maximum()) { // LCOV_EXCL_START
 			throw ConversionException("DuckDB does not support Blobs over 4GB");
 		} // LCOV_EXCL_STOP
-		auto offsets = ArrowBufferData<uint64_t>(array, 1) + array.offset + scan_state.chunk_offset;
-		if (nested_offset != -1) {
-			offsets = ArrowBufferData<uint64_t>(array, 1) + array.offset + nested_offset;
-		}
+		auto offsets =
+		    ArrowBufferData<uint64_t>(array, 1) + GetEffectiveOffset(array, parent_offset, scan_state, nested_offset);
 		auto cdata = ArrowBufferData<char>(array, 2);
 		for (idx_t row_idx = 0; row_idx < size; row_idx++) {
 			if (FlatVector::IsNull(vector, row_idx)) {
@@ -266,26 +294,20 @@ static void SetVectorString(Vector &vector, idx_t size, char *cdata, T *offsets)
 	}
 }
 
-static void DirectConversion(Vector &vector, ArrowArray &array, ArrowScanLocalState &scan_state, int64_t nested_offset,
-                             uint64_t parent_offset) {
+static void DirectConversion(Vector &vector, ArrowArray &array, const ArrowScanLocalState &scan_state,
+                             int64_t nested_offset, uint64_t parent_offset) {
 	auto internal_type = GetTypeIdSize(vector.GetType().InternalType());
-	auto data_ptr =
-	    ArrowBufferData<data_t>(array, 1) + internal_type * (scan_state.chunk_offset + array.offset + parent_offset);
-	if (nested_offset != -1) {
-		data_ptr = ArrowBufferData<data_t>(array, 1) + internal_type * (array.offset + nested_offset + parent_offset);
-	}
+	auto data_ptr = ArrowBufferData<data_t>(array, 1) +
+	                internal_type * GetEffectiveOffset(array, parent_offset, scan_state, nested_offset);
 	FlatVector::SetData(vector, data_ptr);
 }
 
 template <class T>
-static void TimeConversion(Vector &vector, ArrowArray &array, ArrowScanLocalState &scan_state, int64_t nested_offset,
-                           idx_t size, int64_t conversion) {
+static void TimeConversion(Vector &vector, ArrowArray &array, const ArrowScanLocalState &scan_state,
+                           int64_t nested_offset, int64_t parent_offset, idx_t size, int64_t conversion) {
 	auto tgt_ptr = FlatVector::GetData<dtime_t>(vector);
 	auto &validity_mask = FlatVector::Validity(vector);
-	auto src_ptr = (T *)array.buffers[1] + scan_state.chunk_offset + array.offset;
-	if (nested_offset != -1) {
-		src_ptr = (T *)array.buffers[1] + nested_offset + array.offset;
-	}
+	auto src_ptr = (T *)array.buffers[1] + GetEffectiveOffset(array, parent_offset, scan_state, nested_offset);
 	for (idx_t row = 0; row < size; row++) {
 		if (!validity_mask.RowIsValid(row)) {
 			continue;
@@ -296,14 +318,12 @@ static void TimeConversion(Vector &vector, ArrowArray &array, ArrowScanLocalStat
 	}
 }
 
-static void TimestampTZConversion(Vector &vector, ArrowArray &array, ArrowScanLocalState &scan_state,
-                                  int64_t nested_offset, idx_t size, int64_t conversion) {
+static void TimestampTZConversion(Vector &vector, ArrowArray &array, const ArrowScanLocalState &scan_state,
+                                  int64_t nested_offset, int64_t parent_offset, idx_t size, int64_t conversion) {
 	auto tgt_ptr = FlatVector::GetData<timestamp_t>(vector);
 	auto &validity_mask = FlatVector::Validity(vector);
-	auto src_ptr = ArrowBufferData<int64_t>(array, 1) + scan_state.chunk_offset + array.offset;
-	if (nested_offset != -1) {
-		src_ptr = ArrowBufferData<int64_t>(array, 1) + nested_offset + array.offset;
-	}
+	auto src_ptr =
+	    ArrowBufferData<int64_t>(array, 1) + GetEffectiveOffset(array, parent_offset, scan_state, nested_offset);
 	for (idx_t row = 0; row < size; row++) {
 		if (!validity_mask.RowIsValid(row)) {
 			continue;
@@ -314,13 +334,11 @@ static void TimestampTZConversion(Vector &vector, ArrowArray &array, ArrowScanLo
 	}
 }
 
-static void IntervalConversionUs(Vector &vector, ArrowArray &array, ArrowScanLocalState &scan_state,
-                                 int64_t nested_offset, idx_t size, int64_t conversion) {
+static void IntervalConversionUs(Vector &vector, ArrowArray &array, const ArrowScanLocalState &scan_state,
+                                 int64_t nested_offset, int64_t parent_offset, idx_t size, int64_t conversion) {
 	auto tgt_ptr = FlatVector::GetData<interval_t>(vector);
-	auto src_ptr = ArrowBufferData<int64_t>(array, 1) + scan_state.chunk_offset + array.offset;
-	if (nested_offset != -1) {
-		src_ptr = ArrowBufferData<int64_t>(array, 1) + nested_offset + array.offset;
-	}
+	auto src_ptr =
+	    ArrowBufferData<int64_t>(array, 1) + GetEffectiveOffset(array, parent_offset, scan_state, nested_offset);
 	for (idx_t row = 0; row < size; row++) {
 		tgt_ptr[row].days = 0;
 		tgt_ptr[row].months = 0;
@@ -330,13 +348,11 @@ static void IntervalConversionUs(Vector &vector, ArrowArray &array, ArrowScanLoc
 	}
 }
 
-static void IntervalConversionMonths(Vector &vector, ArrowArray &array, ArrowScanLocalState &scan_state,
-                                     int64_t nested_offset, idx_t size) {
+static void IntervalConversionMonths(Vector &vector, ArrowArray &array, const ArrowScanLocalState &scan_state,
+                                     int64_t nested_offset, int64_t parent_offset, idx_t size) {
 	auto tgt_ptr = FlatVector::GetData<interval_t>(vector);
-	auto src_ptr = ArrowBufferData<int32_t>(array, 1) + scan_state.chunk_offset + array.offset;
-	if (nested_offset != -1) {
-		src_ptr = ArrowBufferData<int32_t>(array, 1) + nested_offset + array.offset;
-	}
+	auto src_ptr =
+	    ArrowBufferData<int32_t>(array, 1) + GetEffectiveOffset(array, parent_offset, scan_state, nested_offset);
 	for (idx_t row = 0; row < size; row++) {
 		tgt_ptr[row].days = 0;
 		tgt_ptr[row].micros = 0;
@@ -344,13 +360,11 @@ static void IntervalConversionMonths(Vector &vector, ArrowArray &array, ArrowSca
 	}
 }
 
-static void IntervalConversionMonthDayNanos(Vector &vector, ArrowArray &array, ArrowScanLocalState &scan_state,
-                                            int64_t nested_offset, idx_t size) {
+static void IntervalConversionMonthDayNanos(Vector &vector, ArrowArray &array, const ArrowScanLocalState &scan_state,
+                                            int64_t nested_offset, int64_t parent_offset, idx_t size) {
 	auto tgt_ptr = FlatVector::GetData<interval_t>(vector);
-	auto src_ptr = ArrowBufferData<ArrowInterval>(array, 1) + scan_state.chunk_offset + array.offset;
-	if (nested_offset != -1) {
-		src_ptr = ArrowBufferData<ArrowInterval>(array, 1) + nested_offset + array.offset;
-	}
+	auto src_ptr =
+	    ArrowBufferData<ArrowInterval>(array, 1) + GetEffectiveOffset(array, parent_offset, scan_state, nested_offset);
 	for (idx_t row = 0; row < size; row++) {
 		tgt_ptr[row].days = src_ptr[row].days;
 		tgt_ptr[row].micros = src_ptr[row].nanoseconds / Interval::NANOS_PER_MICRO;
@@ -358,11 +372,228 @@ static void IntervalConversionMonthDayNanos(Vector &vector, ArrowArray &array, A
 	}
 }
 
+// Find the index of the first run-end that is strictly greater than the offset.
+// count is returned if no such run-end is found.
+template <class RUN_END_TYPE>
+static idx_t FindRunIndex(const RUN_END_TYPE *run_ends, idx_t count, idx_t offset) {
+	// Binary-search within the [0, count) range. For example:
+	// [0, 0, 0, 1, 1, 2] encoded as
+	// run_ends: [3, 5, 6]:
+	// 0, 1, 2 -> 0
+	//    3, 4 -> 1
+	//       5 -> 2
+	// 6, 7 .. -> 3 (3 == count [not found])
+	idx_t begin = 0;
+	idx_t end = count;
+	while (begin < end) {
+		idx_t middle = (begin + end) / 2;
+		// begin < end implies middle < end
+		if (offset >= static_cast<idx_t>(run_ends[middle])) {
+			// keep searching in [middle + 1, end)
+			begin = middle + 1;
+		} else {
+			// offset < run_ends[middle], so keep searching in [begin, middle)
+			end = middle;
+		}
+	}
+	return begin;
+}
+
+template <class RUN_END_TYPE, class VALUE_TYPE>
+static void FlattenRunEnds(Vector &result, ArrowRunEndEncodingState &run_end_encoding, idx_t compressed_size,
+                           idx_t scan_offset, idx_t count) {
+	auto &runs = *run_end_encoding.run_ends;
+	auto &values = *run_end_encoding.values;
+
+	UnifiedVectorFormat run_end_format;
+	UnifiedVectorFormat value_format;
+	runs.ToUnifiedFormat(compressed_size, run_end_format);
+	values.ToUnifiedFormat(compressed_size, value_format);
+	auto run_ends_data = run_end_format.GetData<RUN_END_TYPE>(run_end_format);
+	auto values_data = value_format.GetData<VALUE_TYPE>(value_format);
+	auto result_data = FlatVector::GetData<VALUE_TYPE>(result);
+	auto &validity = FlatVector::Validity(result);
+
+	// According to the arrow spec, the 'run_ends' array is always valid
+	// so we will assume this is true and not check the validity map
+
+	// Now construct the result vector from the run_ends and the values
+
+	auto run = FindRunIndex(run_ends_data, compressed_size, scan_offset);
+	idx_t logical_index = scan_offset;
+	idx_t index = 0;
+	if (value_format.validity.AllValid()) {
+		// None of the compressed values are NULL
+		for (; run < compressed_size; run++) {
+			auto run_end_index = run_end_format.sel->get_index(run);
+			auto value_index = value_format.sel->get_index(run);
+			auto &value = values_data[value_index];
+			auto run_end = static_cast<idx_t>(run_ends_data[run_end_index]);
+
+			D_ASSERT(run_end > (logical_index + index));
+			auto to_scan = run_end - (logical_index + index);
+			// Cap the amount to scan so we don't go over size
+			to_scan = MinValue<idx_t>(to_scan, (count - index));
+
+			for (idx_t i = 0; i < to_scan; i++) {
+				result_data[index + i] = value;
+			}
+			index += to_scan;
+			if (index >= count) {
+				if (logical_index + index >= run_end) {
+					// The last run was completed, forward the run index
+					run++;
+				}
+				break;
+			}
+		}
+	} else {
+		for (; run < compressed_size; run++) {
+			auto run_end_index = run_end_format.sel->get_index(run);
+			auto value_index = value_format.sel->get_index(run);
+			auto run_end = static_cast<idx_t>(run_ends_data[run_end_index]);
+
+			D_ASSERT(run_end > (logical_index + index));
+			auto to_scan = run_end - (logical_index + index);
+			// Cap the amount to scan so we don't go over size
+			to_scan = MinValue<idx_t>(to_scan, (count - index));
+
+			if (value_format.validity.RowIsValidUnsafe(value_index)) {
+				auto &value = values_data[value_index];
+				for (idx_t i = 0; i < to_scan; i++) {
+					result_data[index + i] = value;
+					validity.SetValid(index + i);
+				}
+			} else {
+				for (idx_t i = 0; i < to_scan; i++) {
+					validity.SetInvalid(index + i);
+				}
+			}
+			index += to_scan;
+			if (index >= count) {
+				if (logical_index + index >= run_end) {
+					// The last run was completed, forward the run index
+					run++;
+				}
+				break;
+			}
+		}
+	}
+}
+
+template <class RUN_END_TYPE>
+static void FlattenRunEndsSwitch(Vector &result, ArrowRunEndEncodingState &run_end_encoding, idx_t compressed_size,
+                                 idx_t scan_offset, idx_t size) {
+	auto &values = *run_end_encoding.values;
+	auto physical_type = values.GetType().InternalType();
+
+	switch (physical_type) {
+	case PhysicalType::INT8:
+		FlattenRunEnds<RUN_END_TYPE, int8_t>(result, run_end_encoding, compressed_size, scan_offset, size);
+		break;
+	case PhysicalType::INT16:
+		FlattenRunEnds<RUN_END_TYPE, int16_t>(result, run_end_encoding, compressed_size, scan_offset, size);
+		break;
+	case PhysicalType::INT32:
+		FlattenRunEnds<RUN_END_TYPE, int32_t>(result, run_end_encoding, compressed_size, scan_offset, size);
+		break;
+	case PhysicalType::INT64:
+		FlattenRunEnds<RUN_END_TYPE, int64_t>(result, run_end_encoding, compressed_size, scan_offset, size);
+		break;
+	case PhysicalType::INT128:
+		FlattenRunEnds<RUN_END_TYPE, hugeint_t>(result, run_end_encoding, compressed_size, scan_offset, size);
+		break;
+	case PhysicalType::UINT8:
+		FlattenRunEnds<RUN_END_TYPE, uint8_t>(result, run_end_encoding, compressed_size, scan_offset, size);
+		break;
+	case PhysicalType::UINT16:
+		FlattenRunEnds<RUN_END_TYPE, uint16_t>(result, run_end_encoding, compressed_size, scan_offset, size);
+		break;
+	case PhysicalType::UINT32:
+		FlattenRunEnds<RUN_END_TYPE, uint32_t>(result, run_end_encoding, compressed_size, scan_offset, size);
+		break;
+	case PhysicalType::UINT64:
+		FlattenRunEnds<RUN_END_TYPE, uint64_t>(result, run_end_encoding, compressed_size, scan_offset, size);
+		break;
+	case PhysicalType::BOOL:
+		FlattenRunEnds<RUN_END_TYPE, bool>(result, run_end_encoding, compressed_size, scan_offset, size);
+		break;
+	case PhysicalType::FLOAT:
+		FlattenRunEnds<RUN_END_TYPE, float>(result, run_end_encoding, compressed_size, scan_offset, size);
+		break;
+	case PhysicalType::DOUBLE:
+		FlattenRunEnds<RUN_END_TYPE, double>(result, run_end_encoding, compressed_size, scan_offset, size);
+		break;
+	case PhysicalType::INTERVAL:
+		FlattenRunEnds<RUN_END_TYPE, interval_t>(result, run_end_encoding, compressed_size, scan_offset, size);
+		break;
+	case PhysicalType::VARCHAR: {
+		// Share the string heap, we don't need to allocate new strings, we just reference the existing ones
+		result.SetAuxiliary(values.GetAuxiliary());
+		FlattenRunEnds<RUN_END_TYPE, string_t>(result, run_end_encoding, compressed_size, scan_offset, size);
+		break;
+	}
+	default:
+		throw NotImplementedException("RunEndEncoded value type '%s' not supported yet", TypeIdToString(physical_type));
+	}
+}
+
+static void ColumnArrowToDuckDBRunEndEncoded(Vector &vector, ArrowArray &array, ArrowArrayScanState &array_state,
+                                             idx_t size, const ArrowType &arrow_type, int64_t nested_offset,
+                                             ValidityMask *parent_mask, uint64_t parent_offset) {
+	// Scan the 'run_ends' array
+	D_ASSERT(array.n_children == 2);
+	auto &run_ends_array = *array.children[0];
+	auto &values_array = *array.children[1];
+
+	auto &run_ends_type = arrow_type[0];
+	auto &values_type = arrow_type[1];
+	D_ASSERT(vector.GetType() == values_type.GetDuckType());
+
+	auto &scan_state = array_state.state;
+
+	D_ASSERT(run_ends_array.length == values_array.length);
+	auto compressed_size = run_ends_array.length;
+	// Create a vector for the run ends and the values
+	auto &run_end_encoding = array_state.RunEndEncoding();
+	if (!run_end_encoding.run_ends) {
+		// The run ends and values have not been scanned yet for this array
+		D_ASSERT(!run_end_encoding.values);
+		run_end_encoding.run_ends = make_uniq<Vector>(run_ends_type.GetDuckType(), compressed_size);
+		run_end_encoding.values = make_uniq<Vector>(values_type.GetDuckType(), compressed_size);
+
+		ColumnArrowToDuckDB(*run_end_encoding.run_ends, run_ends_array, array_state, compressed_size, run_ends_type);
+		auto &values = *run_end_encoding.values;
+		SetValidityMask(values, values_array, scan_state, compressed_size, parent_offset, nested_offset);
+		ColumnArrowToDuckDB(values, values_array, array_state, compressed_size, values_type);
+	}
+
+	idx_t scan_offset = GetEffectiveOffset(array, parent_offset, scan_state, nested_offset);
+	auto physical_type = run_ends_type.GetDuckType().InternalType();
+	switch (physical_type) {
+	case PhysicalType::INT16:
+		FlattenRunEndsSwitch<int16_t>(vector, run_end_encoding, compressed_size, scan_offset, size);
+		break;
+	case PhysicalType::INT32:
+		FlattenRunEndsSwitch<int32_t>(vector, run_end_encoding, compressed_size, scan_offset, size);
+		break;
+	case PhysicalType::INT64:
+		FlattenRunEndsSwitch<int32_t>(vector, run_end_encoding, compressed_size, scan_offset, size);
+		break;
+	default:
+		throw NotImplementedException("Type '%s' not implemented for RunEndEncoding", TypeIdToString(physical_type));
+	}
+}
+
 static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArrayScanState &array_state, idx_t size,
                                 const ArrowType &arrow_type, int64_t nested_offset, ValidityMask *parent_mask,
                                 uint64_t parent_offset) {
+	if (parent_offset != 0) {
+		(void)array_state;
+	}
 	auto &scan_state = array_state.state;
 	D_ASSERT(!array.dictionary);
+
 	switch (vector.GetType().id()) {
 	case LogicalTypeId::SQLNULL:
 		vector.Reference(Value());
@@ -370,11 +601,8 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArraySca
 	case LogicalTypeId::BOOLEAN: {
 		//! Arrow bit-packs boolean values
 		//! Lets first figure out where we are in the source array
-		auto src_ptr = ArrowBufferData<uint8_t>(array, 1) + (scan_state.chunk_offset + array.offset) / 8;
-
-		if (nested_offset != -1) {
-			src_ptr = ArrowBufferData<uint8_t>(array, 1) + (nested_offset + array.offset) / 8;
-		}
+		auto src_ptr = ArrowBufferData<uint8_t>(array, 1) +
+		               GetEffectiveOffset(array, parent_offset, scan_state, nested_offset) / 8;
 		auto tgt_ptr = (uint8_t *)FlatVector::GetData(vector);
 		int src_pos = 0;
 		idx_t cur_bit = scan_state.chunk_offset % 8;
@@ -406,6 +634,7 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArraySca
 	case LogicalTypeId::UBIGINT:
 	case LogicalTypeId::BIGINT:
 	case LogicalTypeId::HUGEINT:
+	case LogicalTypeId::UHUGEINT:
 	case LogicalTypeId::TIMESTAMP:
 	case LogicalTypeId::TIMESTAMP_SEC:
 	case LogicalTypeId::TIMESTAMP_MS:
@@ -417,16 +646,12 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArraySca
 		auto size_type = arrow_type.GetSizeType();
 		auto cdata = ArrowBufferData<char>(array, 2);
 		if (size_type == ArrowVariableSizeType::SUPER_SIZE) {
-			auto offsets = ArrowBufferData<uint64_t>(array, 1) + array.offset + scan_state.chunk_offset;
-			if (nested_offset != -1) {
-				offsets = ArrowBufferData<uint64_t>(array, 1) + array.offset + nested_offset;
-			}
+			auto offsets = ArrowBufferData<uint64_t>(array, 1) +
+			               GetEffectiveOffset(array, parent_offset, scan_state, nested_offset);
 			SetVectorString(vector, size, cdata, offsets);
 		} else {
-			auto offsets = ArrowBufferData<uint32_t>(array, 1) + array.offset + scan_state.chunk_offset;
-			if (nested_offset != -1) {
-				offsets = ArrowBufferData<uint32_t>(array, 1) + array.offset + nested_offset;
-			}
+			auto offsets = ArrowBufferData<uint32_t>(array, 1) +
+			               GetEffectiveOffset(array, parent_offset, scan_state, nested_offset);
 			SetVectorString(vector, size, cdata, offsets);
 		}
 		break;
@@ -441,10 +666,8 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArraySca
 		}
 		case ArrowDateTimeType::MILLISECONDS: {
 			//! convert date from nanoseconds to days
-			auto src_ptr = ArrowBufferData<uint64_t>(array, 1) + scan_state.chunk_offset + array.offset;
-			if (nested_offset != -1) {
-				src_ptr = ArrowBufferData<uint64_t>(array, 1) + nested_offset + array.offset;
-			}
+			auto src_ptr = ArrowBufferData<uint64_t>(array, 1) +
+			               GetEffectiveOffset(array, parent_offset, scan_state, nested_offset);
 			auto tgt_ptr = FlatVector::GetData<date_t>(vector);
 			for (idx_t row = 0; row < size; row++) {
 				tgt_ptr[row] = date_t(int64_t(src_ptr[row]) / static_cast<int64_t>(1000 * 60 * 60 * 24));
@@ -460,23 +683,21 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArraySca
 		auto precision = arrow_type.GetDateTimeType();
 		switch (precision) {
 		case ArrowDateTimeType::SECONDS: {
-			TimeConversion<int32_t>(vector, array, scan_state, nested_offset, size, 1000000);
+			TimeConversion<int32_t>(vector, array, scan_state, nested_offset, parent_offset, size, 1000000);
 			break;
 		}
 		case ArrowDateTimeType::MILLISECONDS: {
-			TimeConversion<int32_t>(vector, array, scan_state, nested_offset, size, 1000);
+			TimeConversion<int32_t>(vector, array, scan_state, nested_offset, parent_offset, size, 1000);
 			break;
 		}
 		case ArrowDateTimeType::MICROSECONDS: {
-			TimeConversion<int64_t>(vector, array, scan_state, nested_offset, size, 1);
+			TimeConversion<int64_t>(vector, array, scan_state, nested_offset, parent_offset, size, 1);
 			break;
 		}
 		case ArrowDateTimeType::NANOSECONDS: {
 			auto tgt_ptr = FlatVector::GetData<dtime_t>(vector);
-			auto src_ptr = ArrowBufferData<int64_t>(array, 1) + scan_state.chunk_offset + array.offset;
-			if (nested_offset != -1) {
-				src_ptr = ArrowBufferData<int64_t>(array, 1) + nested_offset + array.offset;
-			}
+			auto src_ptr = ArrowBufferData<int64_t>(array, 1) +
+			               GetEffectiveOffset(array, parent_offset, scan_state, nested_offset);
 			for (idx_t row = 0; row < size; row++) {
 				tgt_ptr[row].micros = src_ptr[row] / 1000;
 			}
@@ -491,11 +712,11 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArraySca
 		auto precision = arrow_type.GetDateTimeType();
 		switch (precision) {
 		case ArrowDateTimeType::SECONDS: {
-			TimestampTZConversion(vector, array, scan_state, nested_offset, size, 1000000);
+			TimestampTZConversion(vector, array, scan_state, nested_offset, parent_offset, size, 1000000);
 			break;
 		}
 		case ArrowDateTimeType::MILLISECONDS: {
-			TimestampTZConversion(vector, array, scan_state, nested_offset, size, 1000);
+			TimestampTZConversion(vector, array, scan_state, nested_offset, parent_offset, size, 1000);
 			break;
 		}
 		case ArrowDateTimeType::MICROSECONDS: {
@@ -504,10 +725,8 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArraySca
 		}
 		case ArrowDateTimeType::NANOSECONDS: {
 			auto tgt_ptr = FlatVector::GetData<timestamp_t>(vector);
-			auto src_ptr = ArrowBufferData<int64_t>(array, 1) + scan_state.chunk_offset + array.offset;
-			if (nested_offset != -1) {
-				src_ptr = ArrowBufferData<int64_t>(array, 1) + nested_offset + array.offset;
-			}
+			auto src_ptr = ArrowBufferData<int64_t>(array, 1) +
+			               GetEffectiveOffset(array, parent_offset, scan_state, nested_offset);
 			for (idx_t row = 0; row < size; row++) {
 				tgt_ptr[row].value = src_ptr[row] / 1000;
 			}
@@ -522,24 +741,22 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArraySca
 		auto precision = arrow_type.GetDateTimeType();
 		switch (precision) {
 		case ArrowDateTimeType::SECONDS: {
-			IntervalConversionUs(vector, array, scan_state, nested_offset, size, 1000000);
+			IntervalConversionUs(vector, array, scan_state, nested_offset, parent_offset, size, 1000000);
 			break;
 		}
 		case ArrowDateTimeType::DAYS:
 		case ArrowDateTimeType::MILLISECONDS: {
-			IntervalConversionUs(vector, array, scan_state, nested_offset, size, 1000);
+			IntervalConversionUs(vector, array, scan_state, nested_offset, parent_offset, size, 1000);
 			break;
 		}
 		case ArrowDateTimeType::MICROSECONDS: {
-			IntervalConversionUs(vector, array, scan_state, nested_offset, size, 1);
+			IntervalConversionUs(vector, array, scan_state, nested_offset, parent_offset, size, 1);
 			break;
 		}
 		case ArrowDateTimeType::NANOSECONDS: {
 			auto tgt_ptr = FlatVector::GetData<interval_t>(vector);
-			auto src_ptr = ArrowBufferData<int64_t>(array, 1) + scan_state.chunk_offset + array.offset;
-			if (nested_offset != -1) {
-				src_ptr = ArrowBufferData<int64_t>(array, 1) + nested_offset + array.offset;
-			}
+			auto src_ptr = ArrowBufferData<int64_t>(array, 1) +
+			               GetEffectiveOffset(array, parent_offset, scan_state, nested_offset);
 			for (idx_t row = 0; row < size; row++) {
 				tgt_ptr[row].micros = src_ptr[row] / 1000;
 				tgt_ptr[row].days = 0;
@@ -548,11 +765,11 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArraySca
 			break;
 		}
 		case ArrowDateTimeType::MONTHS: {
-			IntervalConversionMonths(vector, array, scan_state, nested_offset, size);
+			IntervalConversionMonths(vector, array, scan_state, nested_offset, parent_offset, size);
 			break;
 		}
 		case ArrowDateTimeType::MONTH_DAY_NANO: {
-			IntervalConversionMonthDayNanos(vector, array, scan_state, nested_offset, size);
+			IntervalConversionMonthDayNanos(vector, array, scan_state, nested_offset, parent_offset, size);
 			break;
 		}
 		default:
@@ -563,10 +780,8 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArraySca
 	case LogicalTypeId::DECIMAL: {
 		auto val_mask = FlatVector::Validity(vector);
 		//! We have to convert from INT128
-		auto src_ptr = ArrowBufferData<hugeint_t>(array, 1) + scan_state.chunk_offset + array.offset;
-		if (nested_offset != -1) {
-			src_ptr = ArrowBufferData<hugeint_t>(array, 1) + nested_offset + array.offset;
-		}
+		auto src_ptr =
+		    ArrowBufferData<hugeint_t>(array, 1) + GetEffectiveOffset(array, parent_offset, scan_state, nested_offset);
 		switch (vector.GetType().InternalType()) {
 		case PhysicalType::INT16: {
 			auto tgt_ptr = FlatVector::GetData<int16_t>(vector);
@@ -602,9 +817,9 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArraySca
 			break;
 		}
 		case PhysicalType::INT128: {
-			FlatVector::SetData(vector,
-			                    ArrowBufferData<data_t>(array, 1) + GetTypeIdSize(vector.GetType().InternalType()) *
-			                                                            (scan_state.chunk_offset + array.offset));
+			FlatVector::SetData(vector, ArrowBufferData<data_t>(array, 1) +
+			                                GetTypeIdSize(vector.GetType().InternalType()) *
+			                                    GetEffectiveOffset(array, parent_offset, scan_state, nested_offset));
 			break;
 		}
 		default:
@@ -614,15 +829,15 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArraySca
 		break;
 	}
 	case LogicalTypeId::BLOB: {
-		ArrowToDuckDBBlob(vector, array, scan_state, size, arrow_type, nested_offset);
+		ArrowToDuckDBBlob(vector, array, scan_state, size, arrow_type, nested_offset, parent_offset);
 		break;
 	}
 	case LogicalTypeId::LIST: {
-		ArrowToDuckDBList(vector, array, array_state, size, arrow_type, nested_offset, parent_mask);
+		ArrowToDuckDBList(vector, array, array_state, size, arrow_type, nested_offset, parent_mask, parent_offset);
 		break;
 	}
 	case LogicalTypeId::MAP: {
-		ArrowToDuckDBList(vector, array, array_state, size, arrow_type, nested_offset, parent_mask);
+		ArrowToDuckDBList(vector, array, array_state, size, arrow_type, nested_offset, parent_mask, parent_offset);
 		ArrowToDuckDBMapVerify(vector, size);
 		break;
 	}
@@ -636,7 +851,7 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArraySca
 			auto &child_type = arrow_type[child_idx];
 			auto &child_state = array_state.GetChild(child_idx);
 
-			SetValidityMask(child_entry, child_array, scan_state, size, nested_offset);
+			SetValidityMask(child_entry, child_array, scan_state, size, array.offset, nested_offset);
 			if (!struct_validity_mask.AllValid()) {
 				auto &child_validity_mark = FlatVector::Validity(child_entry);
 				for (idx_t i = 0; i < size; i++) {
@@ -645,13 +860,23 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArraySca
 					}
 				}
 			}
-			if (child_array.dictionary) {
-				// TODO: add support for offsets
+
+			auto array_physical_type = GetArrowArrayPhysicalType(child_type);
+			switch (array_physical_type) {
+			case ArrowArrayPhysicalType::DICTIONARY_ENCODED:
 				ColumnArrowToDuckDBDictionary(child_entry, child_array, child_state, size, child_type, nested_offset,
 				                              &struct_validity_mask, array.offset);
-			} else {
+				break;
+			case ArrowArrayPhysicalType::RUN_END_ENCODED:
+				ColumnArrowToDuckDBRunEndEncoded(child_entry, child_array, child_state, size, child_type, nested_offset,
+				                                 &struct_validity_mask, array.offset);
+				break;
+			case ArrowArrayPhysicalType::DEFAULT:
 				ColumnArrowToDuckDB(child_entry, child_array, child_state, size, child_type, nested_offset,
 				                    &struct_validity_mask, array.offset);
+				break;
+			default:
+				throw NotImplementedException("ArrowArrayPhysicalType not recognized");
 			}
 		}
 		break;
@@ -670,12 +895,21 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArraySca
 			auto &child_state = array_state.GetChild(child_idx);
 			auto &child_type = arrow_type[child_idx];
 
-			SetValidityMask(child, child_array, scan_state, size, nested_offset);
+			SetValidityMask(child, child_array, scan_state, size, parent_offset, nested_offset);
+			auto array_physical_type = GetArrowArrayPhysicalType(child_type);
 
-			if (child_array.dictionary) {
+			switch (array_physical_type) {
+			case ArrowArrayPhysicalType::DICTIONARY_ENCODED:
 				ColumnArrowToDuckDBDictionary(child, child_array, child_state, size, child_type);
-			} else {
+				break;
+			case ArrowArrayPhysicalType::RUN_END_ENCODED:
+				ColumnArrowToDuckDBRunEndEncoded(child, child_array, child_state, size, child_type);
+				break;
+			case ArrowArrayPhysicalType::DEFAULT:
 				ColumnArrowToDuckDB(child, child_array, child_state, size, child_type, nested_offset, &validity_mask);
+				break;
+			default:
+				throw NotImplementedException("ArrowArrayPhysicalType not recognized");
 			}
 
 			children.push_back(std::move(child));
@@ -823,26 +1057,64 @@ static void SetSelectionVector(SelectionVector &sel, data_ptr_t indices_p, Logic
 	}
 }
 
+static bool CanContainNull(ArrowArray &array, ValidityMask *parent_mask) {
+	if (array.null_count > 0) {
+		return true;
+	}
+	if (!parent_mask) {
+		return false;
+	}
+	return !parent_mask->AllValid();
+}
+
 static void ColumnArrowToDuckDBDictionary(Vector &vector, ArrowArray &array, ArrowArrayScanState &array_state,
                                           idx_t size, const ArrowType &arrow_type, int64_t nested_offset,
                                           ValidityMask *parent_mask, uint64_t parent_offset) {
-	SelectionVector sel;
 	auto &scan_state = array_state.state;
+
+	const bool has_nulls = CanContainNull(array, parent_mask);
 	if (!array_state.HasDictionary()) {
 		//! We need to set the dictionary data for this column
 		auto base_vector = make_uniq<Vector>(vector.GetType(), array.dictionary->length);
-		SetValidityMask(*base_vector, *array.dictionary, scan_state, array.dictionary->length, 0, array.null_count > 0);
-		ColumnArrowToDuckDB(*base_vector, *array.dictionary, array_state, array.dictionary->length,
-		                    arrow_type.GetDictionary());
+		SetValidityMask(*base_vector, *array.dictionary, scan_state, array.dictionary->length, 0, 0, has_nulls);
+		auto &dictionary_type = arrow_type.GetDictionary();
+		auto arrow_physical_type = GetArrowArrayPhysicalType(dictionary_type);
+		switch (arrow_physical_type) {
+		case ArrowArrayPhysicalType::DICTIONARY_ENCODED:
+			ColumnArrowToDuckDBDictionary(*base_vector, *array.dictionary, array_state, array.dictionary->length,
+			                              dictionary_type);
+			break;
+		case ArrowArrayPhysicalType::RUN_END_ENCODED:
+			ColumnArrowToDuckDBRunEndEncoded(*base_vector, *array.dictionary, array_state, array.dictionary->length,
+			                                 dictionary_type);
+			break;
+		case ArrowArrayPhysicalType::DEFAULT:
+			ColumnArrowToDuckDB(*base_vector, *array.dictionary, array_state, array.dictionary->length,
+			                    dictionary_type);
+			break;
+		default:
+			throw NotImplementedException("ArrowArrayPhysicalType not recognized");
+		};
 		array_state.AddDictionary(std::move(base_vector));
 	}
 	auto offset_type = arrow_type.GetDuckType();
 	//! Get Pointer to Indices of Dictionary
-	auto indices = ArrowBufferData<data_t>(array, 1) +
-	               GetTypeIdSize(offset_type.InternalType()) * (scan_state.chunk_offset + array.offset);
-	if (array.null_count > 0) {
+	auto indices =
+	    ArrowBufferData<data_t>(array, 1) +
+	    GetTypeIdSize(offset_type.InternalType()) * GetEffectiveOffset(array, parent_offset, scan_state, nested_offset);
+
+	SelectionVector sel;
+	if (has_nulls) {
 		ValidityMask indices_validity;
-		GetValidityMask(indices_validity, array, scan_state, size);
+		GetValidityMask(indices_validity, array, scan_state, size, parent_offset);
+		if (parent_mask && !parent_mask->AllValid()) {
+			auto &struct_validity_mask = *parent_mask;
+			for (idx_t i = 0; i < size; i++) {
+				if (!struct_validity_mask.RowIsValid(i)) {
+					indices_validity.SetInvalid(i);
+				}
+			}
+		}
 		SetSelectionVector(sel, indices, offset_type, size, &indices_validity, array.dictionary->length);
 	} else {
 		SetSelectionVector(sel, indices, offset_type, size);
@@ -864,6 +1136,7 @@ void ArrowTableFunction::ArrowToDuckDB(ArrowScanLocalState &scan_state, const ar
 			continue;
 		}
 
+		auto &parent_array = scan_state.chunk->arrow_array;
 		auto &array = *scan_state.chunk->arrow_array.children[arrow_array_idx];
 		if (!array.release) {
 			throw InvalidInputException("arrow_scan: released array passed");
@@ -885,11 +1158,21 @@ void ArrowTableFunction::ArrowToDuckDB(ArrowScanLocalState &scan_state, const ar
 		auto &arrow_type = *arrow_convert_data.at(col_idx);
 		auto &array_state = scan_state.GetState(col_idx);
 
-		if (array.dictionary) {
+		auto array_physical_type = GetArrowArrayPhysicalType(arrow_type);
+
+		switch (array_physical_type) {
+		case ArrowArrayPhysicalType::DICTIONARY_ENCODED:
 			ColumnArrowToDuckDBDictionary(output.data[idx], array, array_state, output.size(), arrow_type);
-		} else {
-			SetValidityMask(output.data[idx], array, scan_state, output.size(), -1);
+			break;
+		case ArrowArrayPhysicalType::RUN_END_ENCODED:
+			ColumnArrowToDuckDBRunEndEncoded(output.data[idx], array, array_state, output.size(), arrow_type);
+			break;
+		case ArrowArrayPhysicalType::DEFAULT:
+			SetValidityMask(output.data[idx], array, scan_state, output.size(), parent_array.offset, -1);
 			ColumnArrowToDuckDB(output.data[idx], array, array_state, output.size(), arrow_type);
+			break;
+		default:
+			throw NotImplementedException("ArrowArrayPhysicalType not recognized");
 		}
 	}
 }