duckdb 1.0.1-dev21.0 → 1.0.1-dev27.0

package/src/duckdb/extension/parquet/include/geo_parquet.hpp

@@ -0,0 +1,139 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// geo_parquet.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "column_writer.hpp"
+#include "duckdb/common/string.hpp"
+#include "duckdb/common/types/data_chunk.hpp"
+#include "duckdb/common/unordered_map.hpp"
+#include "duckdb/common/unordered_set.hpp"
+#include "parquet_types.h"
+
+namespace duckdb {
+
+enum class WKBGeometryType : uint16_t {
+	POINT = 1,
+	LINESTRING = 2,
+	POLYGON = 3,
+	MULTIPOINT = 4,
+	MULTILINESTRING = 5,
+	MULTIPOLYGON = 6,
+	GEOMETRYCOLLECTION = 7,
+
+	POINT_Z = 1001,
+	LINESTRING_Z = 1002,
+	POLYGON_Z = 1003,
+	MULTIPOINT_Z = 1004,
+	MULTILINESTRING_Z = 1005,
+	MULTIPOLYGON_Z = 1006,
+	GEOMETRYCOLLECTION_Z = 1007,
+};
+
+struct WKBGeometryTypes {
+	static const char *ToString(WKBGeometryType type);
+};
+
+struct GeometryBounds {
+	double min_x = NumericLimits<double>::Maximum();
+	double max_x = NumericLimits<double>::Minimum();
+	double min_y = NumericLimits<double>::Maximum();
+	double max_y = NumericLimits<double>::Minimum();
+
+	GeometryBounds() = default;
+
+	void Combine(const GeometryBounds &other) {
+		min_x = std::min(min_x, other.min_x);
+		max_x = std::max(max_x, other.max_x);
+		min_y = std::min(min_y, other.min_y);
+		max_y = std::max(max_y, other.max_y);
+	}
+
+	void Combine(const double &x, const double &y) {
+		min_x = std::min(min_x, x);
+		max_x = std::max(max_x, x);
+		min_y = std::min(min_y, y);
+		max_y = std::max(max_y, y);
+	}
+
+	void Combine(const double &min_x, const double &max_x, const double &min_y, const double &max_y) {
+		this->min_x = std::min(this->min_x, min_x);
+		this->max_x = std::max(this->max_x, max_x);
+		this->min_y = std::min(this->min_y, min_y);
+		this->max_y = std::max(this->max_y, max_y);
+	}
+};
+
+//------------------------------------------------------------------------------
+// GeoParquetMetadata
+//------------------------------------------------------------------------------
+class ParquetReader;
+class ColumnReader;
+class ClientContext;
+class ExpressionExecutor;
+
+enum class GeoParquetColumnEncoding : uint8_t {
+	WKB = 1,
+	POINT,
+	LINESTRING,
+	POLYGON,
+	MULTIPOINT,
+	MULTILINESTRING,
+	MULTIPOLYGON,
+};
+
+struct GeoParquetColumnMetadata {
+	// The encoding of the geometry column
+	GeoParquetColumnEncoding geometry_encoding;
+
+	// The geometry types that are present in the column
+	set<WKBGeometryType> geometry_types;
+
+	// The bounds of the geometry column
+	GeometryBounds bbox;
+
+	// The crs of the geometry column (if any) in PROJJSON format
+	string projjson;
+};
+
+class GeoParquetColumnMetadataWriter {
+	unique_ptr<ExpressionExecutor> executor;
+	DataChunk input_chunk;
+	DataChunk result_chunk;
+
+	unique_ptr<Expression> type_expr;
+	unique_ptr<Expression> flag_expr;
+	unique_ptr<Expression> bbox_expr;
+
+public:
+	explicit GeoParquetColumnMetadataWriter(ClientContext &context);
+	void Update(GeoParquetColumnMetadata &meta, Vector &vector, idx_t count);
+};
+
+struct GeoParquetFileMetadata {
+public:
+	// Try to read GeoParquet metadata. Returns nullptr if not found, invalid or the required spatial extension is not
+	// available.
+	static unique_ptr<GeoParquetFileMetadata> TryRead(const duckdb_parquet::format::FileMetaData &file_meta_data,
+	                                                  ClientContext &context);
+	void Write(duckdb_parquet::format::FileMetaData &file_meta_data) const;
+
+public:
+	// Default to 1.1.0 for now
+	string version = "1.1.0";
+	string primary_geometry_column;
+	unordered_map<string, GeoParquetColumnMetadata> geometry_columns;
+
+	unique_ptr<ColumnReader> CreateColumnReader(ParquetReader &reader, const LogicalType &logical_type,
+	                                            const duckdb_parquet::format::SchemaElement &s_ele, idx_t schema_idx_p,
+	                                            idx_t max_define_p, idx_t max_repeat_p, ClientContext &context);
+
+	bool IsGeometryColumn(const string &column_name) const;
+};
+
+} // namespace duckdb

package/src/duckdb/extension/parquet/include/parquet_crypto.hpp

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "parquet_types.h"
+#include "duckdb/common/encryption_state.hpp"
 
 #ifndef DUCKDB_AMALGAMATION
 #include "duckdb/storage/object_cache.hpp"
@@ -62,26 +63,30 @@ private:
 class ParquetCrypto {
 public:
 	//! Encrypted modules
-	static constexpr uint32_t LENGTH_BYTES = 4;
-	static constexpr uint32_t NONCE_BYTES = 12;
-	static constexpr uint32_t TAG_BYTES = 16;
+	static constexpr idx_t LENGTH_BYTES = 4;
+	static constexpr idx_t NONCE_BYTES = 12;
+	static constexpr idx_t TAG_BYTES = 16;
 
 	//! Block size we encrypt/decrypt
-	static constexpr uint32_t CRYPTO_BLOCK_SIZE = 4096;
+	static constexpr idx_t CRYPTO_BLOCK_SIZE = 4096;
+	static constexpr idx_t BLOCK_SIZE = 16;
 
 public:
 	//! Decrypt and read a Thrift object from the transport protocol
-	static uint32_t Read(TBase &object, TProtocol &iprot, const string &key);
+	static uint32_t Read(TBase &object, TProtocol &iprot, const string &key, const EncryptionUtil &encryption_util_p);
 	//! Encrypt and write a Thrift object to the transport protocol
-	static uint32_t Write(const TBase &object, TProtocol &oprot, const string &key);
+	static uint32_t Write(const TBase &object, TProtocol &oprot, const string &key,
+	                      const EncryptionUtil &encryption_util_p);
 	//! Decrypt and read a buffer
-	static uint32_t ReadData(TProtocol &iprot, const data_ptr_t buffer, const uint32_t buffer_size, const string &key);
+	static uint32_t ReadData(TProtocol &iprot, const data_ptr_t buffer, const uint32_t buffer_size, const string &key,
+	                         const EncryptionUtil &encryption_util_p);
 	//! Encrypt and write a buffer to a file
 	static uint32_t WriteData(TProtocol &oprot, const const_data_ptr_t buffer, const uint32_t buffer_size,
-	                          const string &key);
+	                          const string &key, const EncryptionUtil &encryption_util_p);
 
 public:
 	static void AddKey(ClientContext &context, const FunctionParameters &parameters);
+	static bool ValidKey(const std::string &key);
 };
 
 } // namespace duckdb

package/src/duckdb/extension/parquet/include/parquet_decimal_utils.hpp

@@ -32,6 +32,9 @@ public:
 		if (size > sizeof(PHYSICAL_TYPE)) {
 			for (idx_t i = sizeof(PHYSICAL_TYPE); i < size; i++) {
 				auto byte = *(pointer + (size - i - 1));
+				if (!positive) {
+					byte ^= 0xFF;
+				}
 				if (byte != 0) {
 					throw InvalidInputException("Invalid decimal encoding in Parquet file");
 				}

package/src/duckdb/extension/parquet/include/parquet_file_metadata_cache.hpp

@@ -10,9 +10,9 @@
 #include "duckdb.hpp"
 #ifndef DUCKDB_AMALGAMATION
 #include "duckdb/storage/object_cache.hpp"
+#include "geo_parquet.hpp"
 #endif
 #include "parquet_types.h"
-
 namespace duckdb {
 
 //! ParquetFileMetadataCache
@@ -20,8 +20,9 @@ class ParquetFileMetadataCache : public ObjectCacheEntry {
 public:
 	ParquetFileMetadataCache() : metadata(nullptr) {
 	}
-	ParquetFileMetadataCache(unique_ptr<duckdb_parquet::format::FileMetaData> file_metadata, time_t r_time)
-	    : metadata(std::move(file_metadata)), read_time(r_time) {
+	ParquetFileMetadataCache(unique_ptr<duckdb_parquet::format::FileMetaData> file_metadata, time_t r_time,
+	                         unique_ptr<GeoParquetFileMetadata> geo_metadata)
+	    : metadata(std::move(file_metadata)), read_time(r_time), geo_metadata(std::move(geo_metadata)) {
 	}
 
 	~ParquetFileMetadataCache() override = default;
@@ -32,6 +33,9 @@ public:
 	//! read time
 	time_t read_time;
 
+	//! GeoParquet metadata
+	unique_ptr<GeoParquetFileMetadata> geo_metadata;
+
 public:
 	static string ObjectType() {
 		return "parquet_metadata";

package/src/duckdb/extension/parquet/include/parquet_reader.hpp

@@ -11,6 +11,7 @@
 #include "duckdb.hpp"
 #ifndef DUCKDB_AMALGAMATION
 #include "duckdb/common/common.hpp"
+#include "duckdb/common/encryption_state.hpp"
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/multi_file_reader.hpp"
 #include "duckdb/common/multi_file_reader_options.hpp"
@@ -88,6 +89,7 @@ struct ParquetOptions {
 	bool binary_as_string = false;
 	bool file_row_number = false;
 	shared_ptr<ParquetEncryptionConfig> encryption_config;
+	bool debug_use_openssl = true;
 
 	MultiFileReaderOptions file_options;
 	vector<ParquetColumnDefinition> schema;
@@ -97,11 +99,28 @@ public:
 	static ParquetOptions Deserialize(Deserializer &deserializer);
 };
 
+struct ParquetUnionData {
+	~ParquetUnionData();
+
+	string file_name;
+	vector<string> names;
+	vector<LogicalType> types;
+	ParquetOptions options;
+	shared_ptr<ParquetFileMetadataCache> metadata;
+	unique_ptr<ParquetReader> reader;
+
+	const string &GetFileName() {
+		return file_name;
+	}
+};
+
 class ParquetReader {
 public:
-	ParquetReader(ClientContext &context, string file_name, ParquetOptions parquet_options);
-	ParquetReader(ClientContext &context, ParquetOptions parquet_options,
-	              shared_ptr<ParquetFileMetadataCache> metadata);
+	using UNION_READER_DATA = unique_ptr<ParquetUnionData>;
+
+public:
+	ParquetReader(ClientContext &context, string file_name, ParquetOptions parquet_options,
+	              shared_ptr<ParquetFileMetadataCache> metadata = nullptr);
 	~ParquetReader();
 
 	FileSystem &fs;
@@ -113,16 +132,37 @@ public:
 	ParquetOptions parquet_options;
 	MultiFileReaderData reader_data;
 	unique_ptr<ColumnReader> root_reader;
+	shared_ptr<EncryptionUtil> encryption_util;
 
 	//! Index of the file_row_number column
 	idx_t file_row_number_idx = DConstants::INVALID_INDEX;
 	//! Parquet schema for the generated columns
 	vector<duckdb_parquet::format::SchemaElement> generated_column_schema;
+	//! Table column names - set when using COPY tbl FROM file.parquet
+	vector<string> table_columns;
 
 public:
-	void InitializeScan(ParquetReaderScanState &state, vector<idx_t> groups_to_read);
+	void InitializeScan(ClientContext &context, ParquetReaderScanState &state, vector<idx_t> groups_to_read);
 	void Scan(ParquetReaderScanState &state, DataChunk &output);
 
+	static unique_ptr<ParquetUnionData> StoreUnionReader(unique_ptr<ParquetReader> reader_p, idx_t file_idx) {
+		auto result = make_uniq<ParquetUnionData>();
+		result->file_name = reader_p->file_name;
+		if (file_idx == 0) {
+			result->names = reader_p->names;
+			result->types = reader_p->return_types;
+			result->options = reader_p->parquet_options;
+			result->metadata = reader_p->metadata;
+			result->reader = std::move(reader_p);
+		} else {
+			result->names = std::move(reader_p->names);
+			result->types = std::move(reader_p->return_types);
+			result->options = std::move(reader_p->parquet_options);
+			result->metadata = std::move(reader_p->metadata);
+		}
+		return result;
+	}
+
 	idx_t NumRows();
 	idx_t NumRowGroups();
 
@@ -149,13 +189,20 @@ public:
 		return return_types;
 	}
 
+	static unique_ptr<BaseStatistics> ReadStatistics(ClientContext &context, ParquetOptions parquet_options,
+	                                                 shared_ptr<ParquetFileMetadataCache> metadata, const string &name);
+
 private:
-	void InitializeSchema();
+	//! Construct a parquet reader but **do not** open a file, used in ReadStatistics only
+	ParquetReader(ClientContext &context, ParquetOptions parquet_options,
+	              shared_ptr<ParquetFileMetadataCache> metadata);
+
+	void InitializeSchema(ClientContext &context);
 	bool ScanInternal(ParquetReaderScanState &state, DataChunk &output);
-	unique_ptr<ColumnReader> CreateReader();
+	unique_ptr<ColumnReader> CreateReader(ClientContext &context);
 
-	unique_ptr<ColumnReader> CreateReaderRecursive(idx_t depth, idx_t max_define, idx_t max_repeat,
-	                                               idx_t &next_schema_idx, idx_t &next_file_idx);
+	unique_ptr<ColumnReader> CreateReaderRecursive(ClientContext &context, idx_t depth, idx_t max_define,
+	                                               idx_t max_repeat, idx_t &next_schema_idx, idx_t &next_file_idx);
 	const duckdb_parquet::format::RowGroup &GetGroup(ParquetReaderScanState &state);
 	uint64_t GetGroupCompressedSize(ParquetReaderScanState &state);
 	idx_t GetGroupOffset(ParquetReaderScanState &state);

package/src/duckdb/extension/parquet/include/parquet_rle_bp_decoder.hpp

@@ -7,10 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #pragma once
+#include "decode_utils.hpp"
 #include "parquet_types.h"
-#include "thrift_tools.hpp"
 #include "resizable_buffer.hpp"
-#include "decode_utils.hpp"
+#include "thrift_tools.hpp"
 
 namespace duckdb {
 
@@ -35,7 +35,7 @@ public:
 		while (values_read < batch_size) {
 			if (repeat_count_ > 0) {
 				int repeat_batch = MinValue(batch_size - values_read, static_cast<uint32_t>(repeat_count_));
-				std::fill(values + values_read, values + values_read + repeat_batch, static_cast<T>(current_value_));
+				std::fill_n(values + values_read, repeat_batch, static_cast<T>(current_value_));
 				repeat_count_ -= repeat_batch;
 				values_read += repeat_batch;
 			} else if (literal_count_ > 0) {

package/src/duckdb/extension/parquet/include/parquet_rle_bp_encoder.hpp

@@ -16,7 +16,7 @@ namespace duckdb {
 
 class RleBpEncoder {
 public:
-	RleBpEncoder(uint32_t bit_width);
+	explicit RleBpEncoder(uint32_t bit_width);
 
 public:
 	//! NOTE: Prepare is only required if a byte count is required BEFORE writing

package/src/duckdb/extension/parquet/include/parquet_timestamp.hpp

@@ -17,14 +17,22 @@ struct Int96 {
 };
 
 timestamp_t ImpalaTimestampToTimestamp(const Int96 &raw_ts);
+timestamp_ns_t ImpalaTimestampToTimestampNS(const Int96 &raw_ts);
 Int96 TimestampToImpalaTimestamp(timestamp_t &ts);
+
 timestamp_t ParquetTimestampMicrosToTimestamp(const int64_t &raw_ts);
 timestamp_t ParquetTimestampMsToTimestamp(const int64_t &raw_ts);
 timestamp_t ParquetTimestampNsToTimestamp(const int64_t &raw_ts);
+
+timestamp_ns_t ParquetTimestampMsToTimestampNs(const int64_t &raw_ms);
+timestamp_ns_t ParquetTimestampUsToTimestampNs(const int64_t &raw_us);
+timestamp_ns_t ParquetTimestampNsToTimestampNs(const int64_t &raw_ns);
+
 date_t ParquetIntToDate(const int32_t &raw_date);
 dtime_t ParquetIntToTimeMs(const int32_t &raw_time);
 dtime_t ParquetIntToTime(const int64_t &raw_time);
 dtime_t ParquetIntToTimeNs(const int64_t &raw_time);
+
 dtime_tz_t ParquetIntToTimeMsTZ(const int32_t &raw_time);
 dtime_tz_t ParquetIntToTimeTZ(const int64_t &raw_time);
 dtime_tz_t ParquetIntToTimeNsTZ(const int64_t &raw_time);

package/src/duckdb/extension/parquet/include/parquet_writer.hpp

@@ -11,6 +11,7 @@
 #include "duckdb.hpp"
 #ifndef DUCKDB_AMALGAMATION
 #include "duckdb/common/common.hpp"
+#include "duckdb/common/encryption_state.hpp"
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/mutex.hpp"
 #include "duckdb/common/serializer/buffered_file_writer.hpp"
@@ -20,6 +21,7 @@
 
 #include "column_writer.hpp"
 #include "parquet_types.h"
+#include "geo_parquet.hpp"
 #include "thrift/protocol/TCompactProtocol.h"
 
 namespace duckdb {
@@ -61,11 +63,11 @@ struct FieldID {
 
 class ParquetWriter {
 public:
-	ParquetWriter(FileSystem &fs, string file_name, vector<LogicalType> types, vector<string> names,
-	              duckdb_parquet::format::CompressionCodec::type codec, ChildFieldIDs field_ids,
+	ParquetWriter(ClientContext &context, FileSystem &fs, string file_name, vector<LogicalType> types,
+	              vector<string> names, duckdb_parquet::format::CompressionCodec::type codec, ChildFieldIDs field_ids,
 	              const vector<pair<string, string>> &kv_metadata,
 	              shared_ptr<ParquetEncryptionConfig> encryption_config, double dictionary_compression_ratio_threshold,
-	              optional_idx compression_level);
+	              optional_idx compression_level, bool debug_use_openssl);
 
 public:
 	void PrepareRowGroup(ColumnDataCollection &buffer, PreparedRowGroup &result);
@@ -85,6 +87,9 @@ public:
 	duckdb_parquet::format::Type::type GetType(idx_t schema_idx) {
 		return file_meta_data.schema[schema_idx].type;
 	}
+	LogicalType GetSQLType(idx_t schema_idx) const {
+		return sql_types[schema_idx];
+	}
 	BufferedFileWriter &GetWriter() {
 		return *writer;
 	}
@@ -98,15 +103,20 @@ public:
 	optional_idx CompressionLevel() const {
 		return compression_level;
 	}
-
-	static CopyTypeSupport TypeIsSupported(const LogicalType &type);
+	idx_t NumberOfRowGroups() {
+		lock_guard<mutex> glock(lock);
+		return file_meta_data.row_groups.size();
+	}
 
 	uint32_t Write(const duckdb_apache::thrift::TBase &object);
 	uint32_t WriteData(const const_data_ptr_t buffer, const uint32_t buffer_size);
 
+	GeoParquetFileMetadata &GetGeoParquetData();
+
+	static bool TryGetParquetType(const LogicalType &duckdb_type,
+	                              optional_ptr<duckdb_parquet::format::Type::type> type = nullptr);
+
 private:
-	static CopyTypeSupport DuckDBTypeToParquetTypeInternal(const LogicalType &duckdb_type,
-	                                                       duckdb_parquet::format::Type::type &type);
 	string file_name;
 	vector<LogicalType> sql_types;
 	vector<string> column_names;
@@ -115,6 +125,8 @@ private:
 	shared_ptr<ParquetEncryptionConfig> encryption_config;
 	double dictionary_compression_ratio_threshold;
 	optional_idx compression_level;
+	bool debug_use_openssl;
+	shared_ptr<EncryptionUtil> encryption_util;
 
 	unique_ptr<BufferedFileWriter> writer;
 	std::shared_ptr<duckdb_apache::thrift::protocol::TProtocol> protocol;
@@ -122,6 +134,8 @@ private:
 	std::mutex lock;
 
 	vector<unique_ptr<ColumnWriter>> column_writers;
+
+	unique_ptr<GeoParquetFileMetadata> geoparquet_data;
 };
 
 } // namespace duckdb

package/src/duckdb/extension/parquet/include/resizable_buffer.hpp

@@ -25,50 +25,72 @@ public:
 	uint64_t len = 0;
 
 public:
-	void inc(uint64_t increment) {
+	void inc(const uint64_t increment) {
 		available(increment);
+		unsafe_inc(increment);
+	}
+
+	void unsafe_inc(const uint64_t increment) {
 		len -= increment;
 		ptr += increment;
 	}
 
 	template <class T>
 	T read() {
-		T val = get<T>();
-		inc(sizeof(T));
+		available(sizeof(T));
+		return unsafe_read<T>();
+	}
+
+	template <class T>
+	T unsafe_read() {
+		T val = unsafe_get<T>();
+		unsafe_inc(sizeof(T));
 		return val;
 	}
 
 	template <class T>
 	T get() {
 		available(sizeof(T));
-		T val = Load<T>(ptr);
-		return val;
+		return unsafe_get<T>();
 	}
 
-	void copy_to(char *dest, uint64_t len) {
+	template <class T>
+	T unsafe_get() {
+		return Load<T>(ptr);
+	}
+
+	void copy_to(char *dest, const uint64_t len) const {
 		available(len);
+		unsafe_copy_to(dest, len);
+	}
+
+	void unsafe_copy_to(char *dest, const uint64_t len) const {
 		std::memcpy(dest, ptr, len);
 	}
 
-	void zero() {
+	void zero() const {
 		std::memset(ptr, 0, len);
 	}
 
-	void available(uint64_t req_len) {
-		if (req_len > len) {
+	void available(const uint64_t req_len) const {
+		if (!check_available(req_len)) {
 			throw std::runtime_error("Out of buffer");
 		}
 	}
+
+	bool check_available(const uint64_t req_len) const {
+		return req_len <= len;
+	}
 };
 
 class ResizeableBuffer : public ByteBuffer {
 public:
 	ResizeableBuffer() {
 	}
-	ResizeableBuffer(Allocator &allocator, uint64_t new_size) {
+	ResizeableBuffer(Allocator &allocator, const uint64_t new_size) {
 		resize(allocator, new_size);
 	}
-	void resize(Allocator &allocator, uint64_t new_size) {
+	void resize(Allocator &allocator, const uint64_t new_size) {
 		len = new_size;
 		if (new_size == 0) {
 			return;

package/src/duckdb/extension/parquet/include/string_column_reader.hpp

@@ -16,8 +16,11 @@ struct StringParquetValueConversion {
 	static string_t DictRead(ByteBuffer &dict, uint32_t &offset, ColumnReader &reader);
 
 	static string_t PlainRead(ByteBuffer &plain_data, ColumnReader &reader);
-
 	static void PlainSkip(ByteBuffer &plain_data, ColumnReader &reader);
+
+	static bool PlainAvailable(const ByteBuffer &plain_data, const idx_t count);
+	static string_t UnsafePlainRead(ByteBuffer &plain_data, ColumnReader &reader);
+	static void UnsafePlainSkip(ByteBuffer &plain_data, ColumnReader &reader);
 };
 
 class StringColumnReader : public TemplatedColumnReader<string_t, StringParquetValueConversion> {
@@ -28,7 +31,7 @@ public:
 	StringColumnReader(ParquetReader &reader, LogicalType type_p, const SchemaElement &schema_p, idx_t schema_idx_p,
 	                   idx_t max_define_p, idx_t max_repeat_p);
 
-	unique_ptr<string_t[]> dict_strings;
+	unsafe_unique_ptr<string_t[]> dict_strings;
 	idx_t fixed_width_string_length;
 	idx_t delta_offset = 0;
 

package/src/duckdb/extension/parquet/include/templated_column_reader.hpp

@@ -27,6 +27,18 @@ struct TemplatedParquetValueConversion {
 	static void PlainSkip(ByteBuffer &plain_data, ColumnReader &reader) {
 		plain_data.inc(sizeof(VALUE_TYPE));
 	}
+
+	static bool PlainAvailable(const ByteBuffer &plain_data, const idx_t count) {
+		return plain_data.check_available(count * sizeof(VALUE_TYPE));
+	}
+
+	static VALUE_TYPE UnsafePlainRead(ByteBuffer &plain_data, ColumnReader &reader) {
+		return plain_data.unsafe_read<VALUE_TYPE>();
+	}
+
+	static void UnsafePlainSkip(ByteBuffer &plain_data, ColumnReader &reader) {
+		plain_data.unsafe_inc(sizeof(VALUE_TYPE));
+	}
 };
 
 template <class VALUE_TYPE, class VALUE_CONVERSION>
@@ -60,29 +72,39 @@ public:
 			throw IOException(
 			    "Parquet file is likely corrupted, cannot have dictionary offsets without seeing a dictionary first.");
 		}
-		auto result_ptr = FlatVector::GetData<VALUE_TYPE>(result);
-		auto &result_mask = FlatVector::Validity(result);
+		if (HasDefines()) {
+			OffsetsInternal<true>(*dict, offsets, defines, num_values, filter, result_offset, result);
+		} else {
+			OffsetsInternal<false>(*dict, offsets, defines, num_values, filter, result_offset, result);
+		}
+	}
 
+	void Plain(shared_ptr<ByteBuffer> plain_data, uint8_t *defines, uint64_t num_values, parquet_filter_t &filter,
+	           idx_t result_offset, Vector &result) override {
+		PlainTemplated<VALUE_TYPE, VALUE_CONVERSION>(std::move(plain_data), defines, num_values, filter, result_offset,
+		                                             result);
+	}
+
+private:
+	template <bool HAS_DEFINES>
+	void OffsetsInternal(ResizeableBuffer &dict_ref, uint32_t *__restrict offsets, const uint8_t *__restrict defines,
+	                     const uint64_t num_values, const parquet_filter_t &filter, const idx_t result_offset,
+	                     Vector &result) {
+		const auto result_ptr = FlatVector::GetData<VALUE_TYPE>(result);
+		auto &result_mask = FlatVector::Validity(result);
 		idx_t offset_idx = 0;
-		for (idx_t row_idx = 0; row_idx < num_values; row_idx++) {
-			if (HasDefines() && defines[row_idx + result_offset] != max_define) {
-				result_mask.SetInvalid(row_idx + result_offset);
+		for (idx_t row_idx = result_offset; row_idx < result_offset + num_values; row_idx++) {
+			if (HAS_DEFINES && defines[row_idx] != max_define) {
+				result_mask.SetInvalid(row_idx);
 				continue;
 			}
-			if (filter[row_idx + result_offset]) {
-				VALUE_TYPE val = VALUE_CONVERSION::DictRead(*dict, offsets[offset_idx++], *this);
-				result_ptr[row_idx + result_offset] = val;
+			if (filter.test(row_idx)) {
+				result_ptr[row_idx] = VALUE_CONVERSION::DictRead(dict_ref, offsets[offset_idx++], *this);
 			} else {
 				offset_idx++;
 			}
 		}
 	}
-
-	void Plain(shared_ptr<ByteBuffer> plain_data, uint8_t *defines, uint64_t num_values, parquet_filter_t &filter,
-	           idx_t result_offset, Vector &result) override {
-		PlainTemplated<VALUE_TYPE, VALUE_CONVERSION>(std::move(plain_data), defines, num_values, filter, result_offset,
-		                                             result);
-	}
 };
 
 template <class PARQUET_PHYSICAL_TYPE, class DUCKDB_PHYSICAL_TYPE,
@@ -99,6 +121,18 @@ struct CallbackParquetValueConversion {
 	static void PlainSkip(ByteBuffer &plain_data, ColumnReader &reader) {
 		plain_data.inc(sizeof(PARQUET_PHYSICAL_TYPE));
 	}
+
+	static bool PlainAvailable(const ByteBuffer &plain_data, const idx_t count) {
+		return plain_data.check_available(count * sizeof(PARQUET_PHYSICAL_TYPE));
+	}
+
+	static DUCKDB_PHYSICAL_TYPE UnsafePlainRead(ByteBuffer &plain_data, ColumnReader &reader) {
+		return FUNC(plain_data.unsafe_read<PARQUET_PHYSICAL_TYPE>());
+	}
+
+	static void UnsafePlainSkip(ByteBuffer &plain_data, ColumnReader &reader) {
+		plain_data.unsafe_inc(sizeof(PARQUET_PHYSICAL_TYPE));
+	}
 };
 
 } // namespace duckdb