duckdb 1.1.3 → 1.1.4-dev9.0

package/src/duckdb/extension/parquet/include/parquet_dbp_decoder.hpp

@@ -1,126 +1,137 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// parquet_dbp_deccoder.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
 #pragma once
+
 #include "decode_utils.hpp"
 
 namespace duckdb {
+
 class DbpDecoder {
 public:
-	DbpDecoder(data_ptr_t buffer, uint32_t buffer_len) : buffer_(buffer, buffer_len) {
-		//<block size in values> <number of miniblocks in a block> <total value count> <first value>
-		// overall header
-		block_value_count = ParquetDecodeUtils::VarintDecode<uint64_t>(buffer_);
-		miniblocks_per_block = ParquetDecodeUtils::VarintDecode<uint64_t>(buffer_);
-		total_value_count = ParquetDecodeUtils::VarintDecode<uint64_t>(buffer_);
-		start_value = ParquetDecodeUtils::ZigzagToInt<int64_t>(ParquetDecodeUtils::VarintDecode<uint64_t>(buffer_));
-
-		// some derivatives
-		D_ASSERT(miniblocks_per_block > 0);
-		values_per_miniblock = block_value_count / miniblocks_per_block;
-		miniblock_bit_widths = unique_ptr<uint8_t[]>(new data_t[miniblocks_per_block]);
-
-		// init state to something sane
-		values_left_in_block = 0;
-		values_left_in_miniblock = 0;
-		miniblock_offset = 0;
-		min_delta = 0;
-		bitpack_pos = 0;
-		is_first_value = true;
+	DbpDecoder(const data_ptr_t buffer, const uint32_t buffer_len)
+	    : buffer_(buffer, buffer_len),
+	      //<block size in values> <number of miniblocks in a block> <total value count> <first value>
+	      block_size_in_values(ParquetDecodeUtils::VarintDecode<uint64_t>(buffer_)),
+	      number_of_miniblocks_per_block(ParquetDecodeUtils::VarintDecode<uint64_t>(buffer_)),
+	      number_of_values_in_a_miniblock(block_size_in_values / number_of_miniblocks_per_block),
+	      total_value_count(ParquetDecodeUtils::VarintDecode<uint64_t>(buffer_)),
+	      previous_value(ParquetDecodeUtils::ZigzagToInt(ParquetDecodeUtils::VarintDecode<uint64_t>(buffer_))),
+	      // init state to something sane
+	      is_first_value(true), read_values(0), min_delta(NumericLimits<int64_t>::Maximum()),
+	      miniblock_index(number_of_miniblocks_per_block - 1), list_of_bitwidths_of_miniblocks(nullptr),
+	      miniblock_offset(number_of_values_in_a_miniblock),
+	      unpacked_data_offset(BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE) {
+		if (!(block_size_in_values % number_of_miniblocks_per_block == 0 &&
+		      number_of_values_in_a_miniblock % BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE == 0)) {
+			throw InvalidInputException("Parquet file has invalid block sizes for DELTA_BINARY_PACKED");
+		}
 	};
 
-	ByteBuffer BufferPtr() {
-		if (bitpack_pos != 0) {
-			buffer_.inc(1);
-			bitpack_pos = 0;
-		}
+	ByteBuffer BufferPtr() const {
 		return buffer_;
 	}
 
+	uint64_t TotalValues() const {
+		return total_value_count;
+	}
+
 	template <typename T>
-	void GetBatch(data_ptr_t values_target_ptr, uint32_t batch_size) {
-		auto values = reinterpret_cast<T *>(values_target_ptr);
+	void GetBatch(const data_ptr_t target_values_ptr, const idx_t batch_size) {
+		if (read_values + batch_size > total_value_count) {
+			throw std::runtime_error("DBP decode did not find enough values");
+		}
+		read_values += batch_size;
+		GetBatchInternal<T>(target_values_ptr, batch_size);
+	}
+
+	void Finalize() {
+		if (miniblock_offset == number_of_values_in_a_miniblock) {
+			return;
+		}
+		auto data = make_unsafe_uniq_array<int64_t>(number_of_values_in_a_miniblock);
+		GetBatchInternal<int64_t>(data_ptr_cast(data.get()), number_of_values_in_a_miniblock - miniblock_offset);
+	}
 
+private:
+	template <typename T>
+	void GetBatchInternal(const data_ptr_t target_values_ptr, const idx_t batch_size) {
 		if (batch_size == 0) {
 			return;
 		}
-		idx_t value_offset = 0;
 
+		auto target_values = reinterpret_cast<T *>(target_values_ptr);
+		idx_t target_values_offset = 0;
 		if (is_first_value) {
-			values[0] = start_value;
-			value_offset++;
+			target_values[0] = static_cast<T>(previous_value);
+			target_values_offset++;
 			is_first_value = false;
 		}
 
-		if (total_value_count == 1) { // I guess it's a special case
-			if (batch_size > 1) {
-				throw std::runtime_error("DBP decode did not find enough values (have 1)");
-			}
-			return;
-		}
-
-		while (value_offset < batch_size) {
-			if (values_left_in_block == 0) { // need to open new block
-				if (bitpack_pos > 0) {       // have to eat the leftovers if any
-					buffer_.inc(1);
-				}
-				min_delta =
-				    ParquetDecodeUtils::ZigzagToInt<int64_t>(ParquetDecodeUtils::VarintDecode<uint64_t>(buffer_));
-				for (idx_t miniblock_idx = 0; miniblock_idx < miniblocks_per_block; miniblock_idx++) {
-					miniblock_bit_widths[miniblock_idx] = buffer_.read<uint8_t>();
-					// TODO what happens if width is 0?
+		while (target_values_offset < batch_size) {
+			// Copy over any remaining data
+			const idx_t next = MinValue(batch_size - target_values_offset,
+			                            BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE - unpacked_data_offset);
+			if (next != 0) {
+				for (idx_t i = 0; i < next; i++) {
+					auto &target = target_values[target_values_offset + i];
+					const auto &unpacked_value = unpacked_data[unpacked_data_offset + i];
+					target = static_cast<T>(static_cast<uint64_t>(previous_value) + static_cast<uint64_t>(min_delta) +
+					                        unpacked_value);
+					previous_value = static_cast<int64_t>(target);
 				}
-				values_left_in_block = block_value_count;
-				miniblock_offset = 0;
-				bitpack_pos = 0;
-				values_left_in_miniblock = values_per_miniblock;
-			}
-			if (values_left_in_miniblock == 0) {
-				miniblock_offset++;
-				values_left_in_miniblock = values_per_miniblock;
+				target_values_offset += next;
+				unpacked_data_offset += next;
+				continue;
 			}
 
-			auto read_now = MinValue(values_left_in_miniblock, (idx_t)batch_size - value_offset);
-			ParquetDecodeUtils::BitUnpack<T>(buffer_, bitpack_pos, &values[value_offset], read_now,
-			                                 miniblock_bit_widths[miniblock_offset]);
-			for (idx_t i = value_offset; i < value_offset + read_now; i++) {
-				values[i] = T(uint64_t((i == 0) ? start_value : values[i - 1]) + min_delta + uint64_t(values[i]));
+			// Move to next miniblock / block
+			D_ASSERT(unpacked_data_offset == BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE);
+			D_ASSERT(miniblock_index < number_of_miniblocks_per_block);
+			D_ASSERT(miniblock_offset <= number_of_values_in_a_miniblock);
+			if (miniblock_offset == number_of_values_in_a_miniblock) {
+				miniblock_offset = 0;
+				if (++miniblock_index == number_of_miniblocks_per_block) {
+					// <min delta> <list of bitwidths of miniblocks> <miniblocks>
+					min_delta = ParquetDecodeUtils::ZigzagToInt(ParquetDecodeUtils::VarintDecode<uint64_t>(buffer_));
+					buffer_.available(number_of_miniblocks_per_block);
+					list_of_bitwidths_of_miniblocks = buffer_.ptr;
+					buffer_.unsafe_inc(number_of_miniblocks_per_block);
+					miniblock_index = 0;
+				}
 			}
-			value_offset += read_now;
-			values_left_in_miniblock -= read_now;
-			values_left_in_block -= read_now;
-		}
 
-		if (value_offset != batch_size) {
-			throw std::runtime_error("DBP decode did not find enough values");
-		}
-		start_value = values[batch_size - 1];
-	}
-	void Finalize() {
-		if (values_left_in_miniblock == 0) {
-			return;
+			// Unpack from current miniblock
+			ParquetDecodeUtils::BitUnpackAligned(buffer_, unpacked_data,
+			                                     BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE,
+			                                     list_of_bitwidths_of_miniblocks[miniblock_index]);
+			unpacked_data_offset = 0;
+			miniblock_offset += BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE;
 		}
-		auto data = unique_ptr<uint32_t[]>(new uint32_t[values_left_in_miniblock]);
-		GetBatch<uint32_t>(data_ptr_cast(data.get()), values_left_in_miniblock);
-	}
-
-	uint64_t TotalValues() {
-		return total_value_count;
 	}
 
 private:
 	ByteBuffer buffer_;
-	idx_t block_value_count;
-	idx_t miniblocks_per_block;
-	idx_t total_value_count;
-	int64_t start_value;
-	idx_t values_per_miniblock;
-
-	unique_ptr<uint8_t[]> miniblock_bit_widths;
-	idx_t values_left_in_block;
-	idx_t values_left_in_miniblock;
-	idx_t miniblock_offset;
-	int64_t min_delta;
+	const idx_t block_size_in_values;
+	const idx_t number_of_miniblocks_per_block;
+	const idx_t number_of_values_in_a_miniblock;
+	const idx_t total_value_count;
+	int64_t previous_value;
 
 	bool is_first_value;
+	idx_t read_values;
 
-	uint8_t bitpack_pos;
+	//! Block stuff
+	int64_t min_delta;
+	idx_t miniblock_index;
+	bitpacking_width_t *list_of_bitwidths_of_miniblocks;
+	idx_t miniblock_offset;
+	uint64_t unpacked_data[BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE];
+	idx_t unpacked_data_offset;
 };
 } // namespace duckdb

package/src/duckdb/extension/parquet/include/parquet_dbp_encoder.hpp

@@ -0,0 +1,179 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// parquet_dbp_encoder.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "decode_utils.hpp"
+
+namespace duckdb {
+
+class DbpEncoder {
+private:
+	static constexpr uint64_t BLOCK_SIZE_IN_VALUES = 2048;
+	static constexpr uint64_t NUMBER_OF_MINIBLOCKS_IN_A_BLOCK = 8;
+	static constexpr uint64_t NUMBER_OF_VALUES_IN_A_MINIBLOCK = BLOCK_SIZE_IN_VALUES / NUMBER_OF_MINIBLOCKS_IN_A_BLOCK;
+
+public:
+	explicit DbpEncoder(const idx_t total_value_count_p) : total_value_count(total_value_count_p), count(0) {
+	}
+
+public:
+	void BeginWrite(WriteStream &writer, const int64_t &first_value) {
+		// <block size in values> <number of miniblocks in a block> <total value count> <first value>
+
+		// the block size is a multiple of 128; it is stored as a ULEB128 int
+		ParquetDecodeUtils::VarintEncode(BLOCK_SIZE_IN_VALUES, writer);
+		// the miniblock count per block is a divisor of the block size such that their quotient,
+		// the number of values in a miniblock, is a multiple of 32
+		static_assert(BLOCK_SIZE_IN_VALUES % NUMBER_OF_MINIBLOCKS_IN_A_BLOCK == 0 &&
+		                  NUMBER_OF_VALUES_IN_A_MINIBLOCK % BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE == 0,
+		              "invalid block sizes for DELTA_BINARY_PACKED");
+		// it is stored as a ULEB128 int
+		ParquetDecodeUtils::VarintEncode(NUMBER_OF_MINIBLOCKS_IN_A_BLOCK, writer);
+		// the total value count is stored as a ULEB128 int
+		ParquetDecodeUtils::VarintEncode(total_value_count, writer);
+		// the first value is stored as a zigzag ULEB128 int
+		ParquetDecodeUtils::VarintEncode(ParquetDecodeUtils::IntToZigzag(first_value), writer);
+
+		// initialize
+		if (total_value_count != 0) {
+			count++;
+		}
+		previous_value = first_value;
+
+		min_delta = NumericLimits<int64_t>::Maximum();
+		block_count = 0;
+	}
+
+	void WriteValue(WriteStream &writer, const int64_t &value) {
+		// 1. Compute the differences between consecutive elements. For the first element in the block,
+		// use the last element in the previous block or, in the case of the first block,
+		// use the first value of the whole sequence, stored in the header.
+
+		// Subtractions in steps 1) and 2) may incur signed arithmetic overflow,
+		// and so will the corresponding additions when decoding.
+		// Overflow should be allowed and handled as wrapping around in 2’s complement notation
+		// so that the original values are correctly restituted.
+		// This may require explicit care in some programming languages
+		// (for example by doing all arithmetic in the unsigned domain).
+		const auto delta = static_cast<int64_t>(static_cast<uint64_t>(value) - static_cast<uint64_t>(previous_value));
+		previous_value = value;
+		// Compute the frame of reference (the minimum of the deltas in the block).
+		min_delta = MinValue(min_delta, delta);
+		// append. if block is full, write it out
+		data[block_count++] = delta;
+		if (block_count == BLOCK_SIZE_IN_VALUES) {
+			WriteBlock(writer);
+		}
+	}
+
+	void FinishWrite(WriteStream &writer) {
+		if (count + block_count != total_value_count) {
+			throw InternalException("value count mismatch when writing DELTA_BINARY_PACKED");
+		}
+		if (block_count != 0) {
+			WriteBlock(writer);
+		}
+	}
+
+private:
+	void WriteBlock(WriteStream &writer) {
+		D_ASSERT(count + block_count == total_value_count || block_count == BLOCK_SIZE_IN_VALUES);
+		const auto number_of_miniblocks =
+		    (block_count + NUMBER_OF_VALUES_IN_A_MINIBLOCK - 1) / NUMBER_OF_VALUES_IN_A_MINIBLOCK;
+		for (idx_t miniblock_idx = 0; miniblock_idx < number_of_miniblocks; miniblock_idx++) {
+			for (idx_t i = 0; i < NUMBER_OF_VALUES_IN_A_MINIBLOCK; i++) {
+				const idx_t index = miniblock_idx * NUMBER_OF_VALUES_IN_A_MINIBLOCK + i;
+				auto &value = data[index];
+				if (index < block_count) {
+					// 2. Compute the frame of reference (the minimum of the deltas in the block).
+					// Subtract this min delta from all deltas in the block.
+					// This guarantees that all values are non-negative.
+					D_ASSERT(min_delta <= value);
+					value = static_cast<int64_t>(static_cast<uint64_t>(value) - static_cast<uint64_t>(min_delta));
+				} else {
+					// If there are not enough values to fill the last miniblock, we pad the miniblock
+					// so that its length is always the number of values in a full miniblock multiplied by the bit
+					// width. The values of the padding bits should be zero, but readers must accept paddings consisting
+					// of arbitrary bits as well.
+					value = 0;
+				}
+			}
+		}
+
+		for (idx_t miniblock_idx = 0; miniblock_idx < NUMBER_OF_MINIBLOCKS_IN_A_BLOCK; miniblock_idx++) {
+			auto &width = list_of_bitwidths_of_miniblocks[miniblock_idx];
+			if (miniblock_idx < number_of_miniblocks) {
+				const auto src = &data[miniblock_idx * NUMBER_OF_VALUES_IN_A_MINIBLOCK];
+				width = BitpackingPrimitives::MinimumBitWidth(reinterpret_cast<uint64_t *>(src),
+				                                              NUMBER_OF_VALUES_IN_A_MINIBLOCK);
+				D_ASSERT(width <= sizeof(int64_t) * 8);
+			} else {
+				// If, in the last block, less than <number of miniblocks in a block> miniblocks are needed to store the
+				// values, the bytes storing the bit widths of the unneeded miniblocks are still present, their value
+				// should be zero, but readers must accept arbitrary values as well. There are no additional padding
+				// bytes for the miniblock bodies though, as if their bit widths were 0 (regardless of the actual byte
+				// values). The reader knows when to stop reading by keeping track of the number of values read.
+				width = 0;
+			}
+		}
+
+		// 3. Encode the frame of reference (min delta) as a zigzag ULEB128 int
+		// followed by the bit widths of the miniblocks
+		// and the delta values (minus the min delta) bit-packed per miniblock.
+		// <min delta> <list of bitwidths of miniblocks> <miniblocks>
+
+		// the min delta is a zigzag ULEB128 int (we compute a minimum as we need positive integers for bit packing)
+		ParquetDecodeUtils::VarintEncode(ParquetDecodeUtils::IntToZigzag(min_delta), writer);
+		// the bitwidth of each block is stored as a byte
+		writer.WriteData(list_of_bitwidths_of_miniblocks, NUMBER_OF_MINIBLOCKS_IN_A_BLOCK);
+		// each miniblock is a list of bit packed ints according to the bit width stored at the beginning of the block
+		for (idx_t miniblock_idx = 0; miniblock_idx < number_of_miniblocks; miniblock_idx++) {
+			const auto src = &data[miniblock_idx * NUMBER_OF_VALUES_IN_A_MINIBLOCK];
+			const auto &width = list_of_bitwidths_of_miniblocks[miniblock_idx];
+			memset(data_packed, 0, sizeof(data_packed));
+			ParquetDecodeUtils::BitPackAligned(reinterpret_cast<uint64_t *>(src), data_packed,
+			                                   NUMBER_OF_VALUES_IN_A_MINIBLOCK, width);
+			const auto write_size = NUMBER_OF_VALUES_IN_A_MINIBLOCK * width / 8;
+#ifdef DEBUG
+			// immediately verify that unpacking yields the input data
+			int64_t verification_data[NUMBER_OF_VALUES_IN_A_MINIBLOCK];
+			ByteBuffer byte_buffer(data_ptr_cast(data_packed), write_size);
+			bitpacking_width_t bitpack_pos = 0;
+			ParquetDecodeUtils::BitUnpack(byte_buffer, bitpack_pos, verification_data, NUMBER_OF_VALUES_IN_A_MINIBLOCK,
+			                              width);
+			for (idx_t i = 0; i < NUMBER_OF_VALUES_IN_A_MINIBLOCK; i++) {
+				D_ASSERT(src[i] == verification_data[i]);
+			}
+#endif
+			writer.WriteData(data_packed, write_size);
+		}
+
+		count += block_count;
+
+		min_delta = NumericLimits<int64_t>::Maximum();
+		block_count = 0;
+	}
+
+private:
+	//! Overall fields
+	const idx_t total_value_count;
+	idx_t count;
+	int64_t previous_value;
+
+	//! Block-specific fields
+	int64_t min_delta;
+	int64_t data[BLOCK_SIZE_IN_VALUES];
+	idx_t block_count;
+
+	//! Bitpacking fields
+	bitpacking_width_t list_of_bitwidths_of_miniblocks[NUMBER_OF_MINIBLOCKS_IN_A_BLOCK];
+	data_t data_packed[NUMBER_OF_VALUES_IN_A_MINIBLOCK * sizeof(int64_t)];
+};
+
+} // namespace duckdb

package/src/duckdb/extension/parquet/include/parquet_decimal_utils.hpp

@@ -16,8 +16,7 @@ namespace duckdb {
 class ParquetDecimalUtils {
 public:
 	template <class PHYSICAL_TYPE>
-	static PHYSICAL_TYPE ReadDecimalValue(const_data_ptr_t pointer, idx_t size,
-	                                      const duckdb_parquet::format::SchemaElement &) {
+	static PHYSICAL_TYPE ReadDecimalValue(const_data_ptr_t pointer, idx_t size, const duckdb_parquet::SchemaElement &) {
 		PHYSICAL_TYPE res = 0;
 
 		auto res_ptr = (uint8_t *)&res;
@@ -54,6 +53,6 @@ public:
 
 template <>
 double ParquetDecimalUtils::ReadDecimalValue(const_data_ptr_t pointer, idx_t size,
-                                             const duckdb_parquet::format::SchemaElement &schema_ele);
+                                             const duckdb_parquet::SchemaElement &schema_ele);
 
 } // namespace duckdb

package/src/duckdb/extension/parquet/include/parquet_dlba_encoder.hpp

@@ -0,0 +1,48 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// parquet_dlba_encoder.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "parquet_dbp_encoder.hpp"
+#include "duckdb/common/serializer/memory_stream.hpp"
+
+namespace duckdb {
+
+class DlbaEncoder {
+public:
+	DlbaEncoder(const idx_t total_value_count_p, const idx_t total_string_size_p)
+	    : dbp_encoder(total_value_count_p), total_string_size(total_string_size_p),
+	      buffer(Allocator::DefaultAllocator().Allocate(total_string_size + 1)),
+	      stream(make_unsafe_uniq<MemoryStream>(buffer.get(), buffer.GetSize())) {
+	}
+
+public:
+	void BeginWrite(WriteStream &writer, const string_t &first_value) {
+		dbp_encoder.BeginWrite(writer, UnsafeNumericCast<int64_t>(first_value.GetSize()));
+		stream->WriteData(const_data_ptr_cast(first_value.GetData()), first_value.GetSize());
+	}
+
+	void WriteValue(WriteStream &writer, const string_t &value) {
+		dbp_encoder.WriteValue(writer, UnsafeNumericCast<int64_t>(value.GetSize()));
+		stream->WriteData(const_data_ptr_cast(value.GetData()), value.GetSize());
+	}
+
+	void FinishWrite(WriteStream &writer) {
+		D_ASSERT(stream->GetPosition() == total_string_size);
+		dbp_encoder.FinishWrite(writer);
+		writer.WriteData(buffer.get(), total_string_size);
+	}
+
+private:
+	DbpEncoder dbp_encoder;
+	const idx_t total_string_size;
+	AllocatedData buffer;
+	unsafe_unique_ptr<MemoryStream> stream;
+};
+
+} // namespace duckdb

package/src/duckdb/extension/parquet/include/parquet_extension.hpp

@@ -1,3 +1,11 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// parquet_extension.hpp
+//
+//
+//===----------------------------------------------------------------------===/
+
 #pragma once
 
 #include "duckdb.hpp"

package/src/duckdb/extension/parquet/include/parquet_file_metadata_cache.hpp

@@ -20,7 +20,7 @@ class ParquetFileMetadataCache : public ObjectCacheEntry {
 public:
 	ParquetFileMetadataCache() : metadata(nullptr) {
 	}
-	ParquetFileMetadataCache(unique_ptr<duckdb_parquet::format::FileMetaData> file_metadata, time_t r_time,
+	ParquetFileMetadataCache(unique_ptr<duckdb_parquet::FileMetaData> file_metadata, time_t r_time,
 	                         unique_ptr<GeoParquetFileMetadata> geo_metadata)
 	    : metadata(std::move(file_metadata)), read_time(r_time), geo_metadata(std::move(geo_metadata)) {
 	}
@@ -28,7 +28,7 @@ public:
 	~ParquetFileMetadataCache() override = default;
 
 	//! Parquet file metadata
-	unique_ptr<const duckdb_parquet::format::FileMetaData> metadata;
+	unique_ptr<const duckdb_parquet::FileMetaData> metadata;
 
 	//! read time
 	time_t read_time;

package/src/duckdb/extension/parquet/include/parquet_metadata.hpp

@@ -33,4 +33,9 @@ public:
 	ParquetFileMetadataFunction();
 };
 
+class ParquetBloomProbeFunction : public TableFunction {
+public:
+	ParquetBloomProbeFunction();
+};
+
 } // namespace duckdb

package/src/duckdb/extension/parquet/include/parquet_reader.hpp

@@ -71,10 +71,12 @@ public:
 	static ParquetColumnDefinition FromSchemaValue(ClientContext &context, const Value &column_value);
 
 public:
+	// DEPRECATED, use 'identifier' instead
 	int32_t field_id;
 	string name;
 	LogicalType type;
 	Value default_value;
+	Value identifier;
 
 public:
 	void Serialize(Serializer &serializer) const;
@@ -127,18 +129,15 @@ public:
 	FileSystem &fs;
 	Allocator &allocator;
 	string file_name;
-	vector<LogicalType> return_types;
-	vector<string> names;
+	vector<MultiFileReaderColumnDefinition> columns;
 	shared_ptr<ParquetFileMetadataCache> metadata;
 	ParquetOptions parquet_options;
 	MultiFileReaderData reader_data;
 	unique_ptr<ColumnReader> root_reader;
 	shared_ptr<EncryptionUtil> encryption_util;
 
-	//! Index of the file_row_number column
-	idx_t file_row_number_idx = DConstants::INVALID_INDEX;
 	//! Parquet schema for the generated columns
-	vector<duckdb_parquet::format::SchemaElement> generated_column_schema;
+	vector<duckdb_parquet::SchemaElement> generated_column_schema;
 	//! Table column names - set when using COPY tbl FROM file.parquet
 	vector<string> table_columns;
 
@@ -150,24 +149,30 @@ public:
 		auto result = make_uniq<ParquetUnionData>();
 		result->file_name = reader_p->file_name;
 		if (file_idx == 0) {
-			result->names = reader_p->names;
-			result->types = reader_p->return_types;
+			for (auto &column : reader_p->columns) {
+				result->names.push_back(column.name);
+				result->types.push_back(column.type);
+			}
 			result->options = reader_p->parquet_options;
 			result->metadata = reader_p->metadata;
 			result->reader = std::move(reader_p);
 		} else {
-			result->names = std::move(reader_p->names);
-			result->types = std::move(reader_p->return_types);
+			for (auto &column : reader_p->columns) {
+				result->names.push_back(column.name);
+				result->types.push_back(column.type);
+			}
+			reader_p->columns.clear();
 			result->options = std::move(reader_p->parquet_options);
 			result->metadata = std::move(reader_p->metadata);
 		}
+
 		return result;
 	}
 
 	idx_t NumRows();
 	idx_t NumRowGroups();
 
-	const duckdb_parquet::format::FileMetaData *GetFileMetadata();
+	const duckdb_parquet::FileMetaData *GetFileMetadata();
 
 	uint32_t Read(duckdb_apache::thrift::TBase &object, TProtocol &iprot);
 	uint32_t ReadData(duckdb_apache::thrift::protocol::TProtocol &iprot, const data_ptr_t buffer,
@@ -183,11 +188,9 @@ public:
 	const string &GetFileName() {
 		return file_name;
 	}
-	const vector<string> &GetNames() {
-		return names;
-	}
-	const vector<LogicalType> &GetTypes() {
-		return return_types;
+
+	const vector<MultiFileReaderColumnDefinition> &GetColumns() {
+		return columns;
 	}
 
 	static unique_ptr<BaseStatistics> ReadStatistics(ClientContext &context, ParquetOptions parquet_options,
@@ -202,9 +205,10 @@ private:
 	bool ScanInternal(ParquetReaderScanState &state, DataChunk &output);
 	unique_ptr<ColumnReader> CreateReader(ClientContext &context);
 
-	unique_ptr<ColumnReader> CreateReaderRecursive(ClientContext &context, idx_t depth, idx_t max_define,
-	                                               idx_t max_repeat, idx_t &next_schema_idx, idx_t &next_file_idx);
-	const duckdb_parquet::format::RowGroup &GetGroup(ParquetReaderScanState &state);
+	unique_ptr<ColumnReader> CreateReaderRecursive(ClientContext &context, const vector<ColumnIndex> &indexes,
+	                                               idx_t depth, idx_t max_define, idx_t max_repeat,
+	                                               idx_t &next_schema_idx, idx_t &next_file_idx);
+	const duckdb_parquet::RowGroup &GetGroup(ParquetReaderScanState &state);
 	uint64_t GetGroupCompressedSize(ParquetReaderScanState &state);
 	idx_t GetGroupOffset(ParquetReaderScanState &state);
 	// Group span is the distance between the min page offset and the max page offset plus the max page compressed size

package/src/duckdb/extension/parquet/include/parquet_rle_bp_decoder.hpp

@@ -40,11 +40,7 @@ public:
 				values_read += repeat_batch;
 			} else if (literal_count_ > 0) {
 				uint32_t literal_batch = MinValue(batch_size - values_read, static_cast<uint32_t>(literal_count_));
-				uint32_t actual_read = ParquetDecodeUtils::BitUnpack<T>(buffer_, bitpack_pos, values + values_read,
-				                                                        literal_batch, bit_width_);
-				if (literal_batch != actual_read) {
-					throw std::runtime_error("Did not find enough values");
-				}
+				ParquetDecodeUtils::BitUnpack<T>(buffer_, bitpack_pos, values + values_read, literal_batch, bit_width_);
 				literal_count_ -= literal_batch;
 				values_read += literal_batch;
 			} else {