duckdb 0.3.5-dev992.0 → 0.4.1-dev1019.0

package/src/parquet-amalgamation.hpp

@@ -91,8 +91,8 @@ public:
  * under the License.
  */
 
-#ifndef _THRIFT_THRIFT_H_
-#define _THRIFT_THRIFT_H_ 1
+#ifndef _DUCKDB_THRIFT_THRIFT_H_
+#define _DUCKDB_THRIFT_THRIFT_H_ 1
 
 
 
@@ -121,8 +121,8 @@ public:
 
 // clang-format off
 
-#ifndef _THRIFT_TRANSPORT_PLATFORM_SOCKET_H_
-#  define _THRIFT_TRANSPORT_PLATFORM_SOCKET_H_
+#ifndef _DUCKDB_THRIFT_TRANSPORT_PLATFORM_SOCKET_H_
+#  define _DUCKDB_THRIFT_TRANSPORT_PLATFORM_SOCKET_H_
 
 #ifdef _WIN32
 #ifdef _WINSOCKAPI_
@@ -236,7 +236,7 @@ public:
 #  define THRIFT_SHUT_RDWR SHUT_RDWR
 #endif
 
-#endif // _THRIFT_TRANSPORT_PLATFORM_SOCKET_H_
+#endif // _DUCKDB_THRIFT_TRANSPORT_PLATFORM_SOCKET_H_
 
 
 // LICENSE_CHANGE_END
@@ -329,8 +329,8 @@ public:
  * under the License.
  */
 
-#ifndef _THRIFT_TLOGGING_H_
-#define _THRIFT_TLOGGING_H_ 1
+#ifndef _DUCKDB_THRIFT_TLOGGING_H_
+#define _DUCKDB_THRIFT_TLOGGING_H_ 1
 
 
 
@@ -454,7 +454,7 @@ public:
 #define T_GENERIC_PROTOCOL(template_class, generic_prot, specific_prot)
 #endif
 
-#endif // #ifndef _THRIFT_TLOGGING_H_
+#endif // #ifndef _DUCKDB_THRIFT_TLOGGING_H_
 
 
 // LICENSE_CHANGE_END
@@ -546,7 +546,7 @@ void profile_write_pprof(FILE* gen_calls_f, FILE* virtual_calls_f);
 }
 } // duckdb_apache::thrift
 
-#endif // #ifndef _THRIFT_THRIFT_H_
+#endif // #ifndef _DUCKDB_THRIFT_THRIFT_H_
 
 
 // LICENSE_CHANGE_END
@@ -576,8 +576,8 @@ void profile_write_pprof(FILE* gen_calls_f, FILE* virtual_calls_f);
  * under the License.
  */
 
-#ifndef _THRIFT_TAPPLICATIONEXCEPTION_H_
-#define _THRIFT_TAPPLICATIONEXCEPTION_H_ 1
+#ifndef _DUCKDB_THRIFT_TAPPLICATIONEXCEPTION_H_
+#define _DUCKDB_THRIFT_TAPPLICATIONEXCEPTION_H_ 1
 
 
 
@@ -671,7 +671,7 @@ protected:
 }
 } // duckdb_apache::thrift
 
-#endif // #ifndef _THRIFT_TAPPLICATIONEXCEPTION_H_
+#endif // #ifndef _DUCKDB_THRIFT_TAPPLICATIONEXCEPTION_H_
 
 
 // LICENSE_CHANGE_END
@@ -701,8 +701,8 @@ protected:
  * under the License.
  */
 
-#ifndef _THRIFT_TBASE_H_
-#define _THRIFT_TBASE_H_ 1
+#ifndef _DUCKDB_THRIFT_TBASE_H_
+#define _DUCKDB_THRIFT_TBASE_H_ 1
 
 
 
@@ -730,8 +730,8 @@ protected:
  * under the License.
  */
 
-#ifndef _THRIFT_PROTOCOL_TPROTOCOL_H_
-#define _THRIFT_PROTOCOL_TPROTOCOL_H_ 1
+#ifndef _DUCKDB_THRIFT_PROTOCOL_TPROTOCOL_H_
+#define _DUCKDB_THRIFT_PROTOCOL_TPROTOCOL_H_ 1
 
 #ifdef _WIN32
 // Need to come before any Windows.h includes
@@ -763,8 +763,8 @@ protected:
  * under the License.
  */
 
-#ifndef _THRIFT_TRANSPORT_TTRANSPORT_H_
-#define _THRIFT_TRANSPORT_TTRANSPORT_H_ 1
+#ifndef _DUCKDB_THRIFT_TRANSPORT_TTRANSPORT_H_
+#define _DUCKDB_THRIFT_TRANSPORT_TTRANSPORT_H_ 1
 
 
 
@@ -792,8 +792,8 @@ protected:
  * under the License.
  */
 
-#ifndef _THRIFT_TRANSPORT_TTRANSPORTEXCEPTION_H_
-#define _THRIFT_TRANSPORT_TTRANSPORTEXCEPTION_H_ 1
+#ifndef _DUCKDB_THRIFT_TRANSPORT_TTRANSPORTEXCEPTION_H_
+#define _DUCKDB_THRIFT_TRANSPORT_TTRANSPORTEXCEPTION_H_ 1
 
 // FUCK OFF #include <boost/numeric/conversion/cast.hpp>
 #include <string>
@@ -878,7 +878,7 @@ protected:
 }
 } // duckdb_apache::thrift::transport
 
-#endif // #ifndef _THRIFT_TRANSPORT_TTRANSPORTEXCEPTION_H_
+#endif // #ifndef _DUCKDB_THRIFT_TRANSPORT_TTRANSPORTEXCEPTION_H_
 
 
 // LICENSE_CHANGE_END
@@ -1129,7 +1129,7 @@ public:
 }
 } // duckdb_apache::thrift::transport
 
-#endif // #ifndef _THRIFT_TRANSPORT_TTRANSPORT_H_
+#endif // #ifndef _DUCKDB_THRIFT_TRANSPORT_TTRANSPORT_H_
 
 
 // LICENSE_CHANGE_END
@@ -1159,8 +1159,8 @@ public:
  * under the License.
  */
 
-#ifndef _THRIFT_PROTOCOL_TPROTOCOLEXCEPTION_H_
-#define _THRIFT_PROTOCOL_TPROTOCOLEXCEPTION_H_ 1
+#ifndef _DUCKDB_THRIFT_PROTOCOL_TPROTOCOLEXCEPTION_H_
+#define _DUCKDB_THRIFT_PROTOCOL_TPROTOCOLEXCEPTION_H_ 1
 
 #include <string>
 
@@ -1244,7 +1244,7 @@ protected:
 }
 } // duckdb_apache::thrift::protocol
 
-#endif // #ifndef _THRIFT_PROTOCOL_TPROTOCOLEXCEPTION_H_
+#endif // #ifndef _DUCKDB_THRIFT_PROTOCOL_TPROTOCOLEXCEPTION_H_
 
 
 // LICENSE_CHANGE_END
@@ -1267,37 +1267,39 @@ protected:
 // but that doesn't work.
 // For a pretty in-depth explanation of the problem, see
 // http://cellperformance.beyond3d.com/articles/2006/06/understanding-strict-aliasing.html
+namespace duckdb_apache { namespace thrift {
 template <typename To, typename From>
 static inline To bitwise_cast(From from) {
-  static_assert(sizeof(From) == sizeof(To), "sizeof(From) == sizeof(To)");
-
-  // BAD!!!  These are all broken with -O2.
-  //return *reinterpret_cast<To*>(&from);  // BAD!!!
-  //return *static_cast<To*>(static_cast<void*>(&from));  // BAD!!!
-  //return *(To*)(void*)&from;  // BAD!!!
-
-  // Super clean and paritally blessed by section 3.9 of the standard.
-  //unsigned char c[sizeof(from)];
-  //memcpy(c, &from, sizeof(from));
-  //To to;
-  //memcpy(&to, c, sizeof(c));
-  //return to;
-
-  // Slightly more questionable.
-  // Same code emitted by GCC.
-  //To to;
-  //memcpy(&to, &from, sizeof(from));
-  //return to;
-
-  // Technically undefined, but almost universally supported,
-  // and the most efficient implementation.
-  union {
-    From f;
-    To t;
-  } u;
-  u.f = from;
-  return u.t;
+	static_assert(sizeof(From) == sizeof(To), "sizeof(From) == sizeof(To)");
+
+	// BAD!!!  These are all broken with -O2.
+	// return *reinterpret_cast<To*>(&from);  // BAD!!!
+	// return *static_cast<To*>(static_cast<void*>(&from));  // BAD!!!
+	// return *(To*)(void*)&from;  // BAD!!!
+
+	// Super clean and paritally blessed by section 3.9 of the standard.
+	// unsigned char c[sizeof(from)];
+	// memcpy(c, &from, sizeof(from));
+	// To to;
+	// memcpy(&to, c, sizeof(c));
+	// return to;
+
+	// Slightly more questionable.
+	// Same code emitted by GCC.
+	// To to;
+	// memcpy(&to, &from, sizeof(from));
+	// return to;
+
+	// Technically undefined, but almost universally supported,
+	// and the most efficient implementation.
+	union {
+		From f;
+		To t;
+	} u;
+	u.f = from;
+	return u.t;
 }
+}} // namespace duckdb_apache::thrift
 
 
 #ifdef HAVE_SYS_PARAM_H
@@ -1983,7 +1985,7 @@ uint32_t skip(Protocol_& prot, TType type) {
 
 }}} // duckdb_apache::thrift::protocol
 
-#endif // #define _THRIFT_PROTOCOL_TPROTOCOL_H_ 1
+#endif // #define _DUCKDB_THRIFT_PROTOCOL_TPROTOCOL_H_ 1
 
 
 // LICENSE_CHANGE_END
@@ -2001,7 +2003,7 @@ public:
 }
 } // duckdb_apache::thrift
 
-#endif // #ifndef _THRIFT_TBASE_H_
+#endif // #ifndef _DUCKDB_THRIFT_TBASE_H_
 
 
 // LICENSE_CHANGE_END
@@ -4625,7 +4627,7 @@ std::ostream& operator<<(std::ostream& out, const FileCryptoMetaData& obj);
 // LICENSE_CHANGE_END
 
 
-
+#include <list>
 
 
 // LICENSE_CHANGE_BEGIN
@@ -4651,8 +4653,8 @@ std::ostream& operator<<(std::ostream& out, const FileCryptoMetaData& obj);
  * under the License.
  */
 
-#ifndef _THRIFT_PROTOCOL_TCOMPACTPROTOCOL_H_
-#define _THRIFT_PROTOCOL_TCOMPACTPROTOCOL_H_ 1
+#ifndef _DUCKDB_THRIFT_PROTOCOL_TCOMPACTPROTOCOL_H_
+#define _DUCKDB_THRIFT_PROTOCOL_TCOMPACTPROTOCOL_H_ 1
 
 
 
@@ -4679,8 +4681,8 @@ std::ostream& operator<<(std::ostream& out, const FileCryptoMetaData& obj);
  * under the License.
  */
 
-#ifndef _THRIFT_PROTOCOL_TVIRTUALPROTOCOL_H_
-#define _THRIFT_PROTOCOL_TVIRTUALPROTOCOL_H_ 1
+#ifndef _DUCKDB_THRIFT_PROTOCOL_TVIRTUALPROTOCOL_H_
+#define _DUCKDB_THRIFT_PROTOCOL_TVIRTUALPROTOCOL_H_ 1
 
 
 
@@ -5172,7 +5174,7 @@ protected:
 }
 } // duckdb_apache::thrift::protocol
 
-#endif // #define _THRIFT_PROTOCOL_TVIRTUALPROTOCOL_H_ 1
+#endif // #define _DUCKDB_THRIFT_PROTOCOL_TVIRTUALPROTOCOL_H_ 1
 
 
 // LICENSE_CHANGE_END
@@ -5441,8 +5443,8 @@ typedef TCompactProtocolFactoryT<TTransport> TCompactProtocolFactory;
  * specific language governing permissions and limitations
  * under the License.
  */
-#ifndef _THRIFT_PROTOCOL_TCOMPACTPROTOCOL_TCC_
-#define _THRIFT_PROTOCOL_TCOMPACTPROTOCOL_TCC_ 1
+#ifndef _DUCKDB_THRIFT_PROTOCOL_TCOMPACTPROTOCOL_TCC_
+#define _DUCKDB_THRIFT_PROTOCOL_TCOMPACTPROTOCOL_TCC_ 1
 
 #include <limits>
 
@@ -6248,7 +6250,7 @@ TType TCompactProtocolT<Transport_>::getTType(int8_t type) {
 
 }}} // duckdb_apache::thrift::protocol
 
-#endif // _THRIFT_PROTOCOL_TCOMPACTPROTOCOL_TCC_
+#endif // _DUCKDB_THRIFT_PROTOCOL_TCOMPACTPROTOCOL_TCC_
 
 
 // LICENSE_CHANGE_END
@@ -6284,8 +6286,8 @@ TType TCompactProtocolT<Transport_>::getTType(int8_t type) {
  * under the License.
  */
 
-#ifndef _THRIFT_TRANSPORT_TBUFFERTRANSPORTS_H_
-#define _THRIFT_TRANSPORT_TBUFFERTRANSPORTS_H_ 1
+#ifndef _DUCKDB_THRIFT_TRANSPORT_TBUFFERTRANSPORTS_H_
+#define _DUCKDB_THRIFT_TRANSPORT_TBUFFERTRANSPORTS_H_ 1
 
 #include <cstdlib>
 #include <cstddef>
@@ -6319,8 +6321,8 @@ TType TCompactProtocolT<Transport_>::getTType(int8_t type) {
  * under the License.
  */
 
-#ifndef _THRIFT_TRANSPORT_TVIRTUALTRANSPORT_H_
-#define _THRIFT_TRANSPORT_TVIRTUALTRANSPORT_H_ 1
+#ifndef _DUCKDB_THRIFT_TRANSPORT_TVIRTUALTRANSPORT_H_
+#define _DUCKDB_THRIFT_TRANSPORT_TVIRTUALTRANSPORT_H_ 1
 
 
 
@@ -6439,7 +6441,7 @@ protected:
 }
 } // duckdb_apache::thrift::transport
 
-#endif // #ifndef _THRIFT_TRANSPORT_TVIRTUALTRANSPORT_H_
+#endif // #ifndef _DUCKDB_THRIFT_TRANSPORT_TVIRTUALTRANSPORT_H_
 
 
 // LICENSE_CHANGE_END
@@ -6902,7 +6904,7 @@ protected:
 }
 } // duckdb_apache::thrift::transport
 
-#endif // #ifndef _THRIFT_TRANSPORT_TBUFFERTRANSPORTS_H_
+#endif // #ifndef _DUCKDB_THRIFT_TRANSPORT_TBUFFERTRANSPORTS_H_
 
 
 // LICENSE_CHANGE_END
@@ -6916,31 +6918,174 @@ protected:
 
 namespace duckdb {
 
+// A ReadHead for prefetching data in a specific range
+struct ReadHead {
+	ReadHead(idx_t location, uint64_t size) : location(location), size(size) {};
+	// Hint info
+	idx_t location;
+	uint64_t size;
+
+	// Current info
+	unique_ptr<AllocatedData> data;
+	bool data_isset = false;
+
+	idx_t GetEnd() const {
+		return size + location;
+	}
+
+	void Allocate(Allocator &allocator) {
+		data = allocator.Allocate(size);
+	}
+};
+
+// Comparator for ReadHeads that are either overlapping, adjacent, or within ALLOW_GAP bytes from each other
+struct ReadHeadComparator {
+	static constexpr uint64_t ALLOW_GAP = 1 << 14; // 16 KiB
+	bool operator()(const ReadHead *a, const ReadHead *b) const {
+		auto a_start = a->location;
+		auto a_end = a->location + a->size;
+		auto b_start = b->location;
+
+		if (a_end <= NumericLimits<idx_t>::Maximum() - ALLOW_GAP) {
+			a_end += ALLOW_GAP;
+		}
+
+		return a_start < b_start && a_end < b_start;
+	}
+};
+
+// Two-step read ahead buffer
+// 1: register all ranges that will be read, merging ranges that are consecutive
+// 2: prefetch all registered ranges
+struct ReadAheadBuffer {
+	ReadAheadBuffer(Allocator &allocator, FileHandle &handle, FileOpener &opener)
+	    : allocator(allocator), handle(handle), file_opener(opener) {
+	}
+
+	// The list of read heads
+	std::list<ReadHead> read_heads;
+	// Set for merging consecutive ranges
+	std::set<ReadHead *, ReadHeadComparator> merge_set;
+
+	Allocator &allocator;
+	FileHandle &handle;
+	FileOpener &file_opener;
+
+	idx_t total_size = 0;
+
+	// Add a read head to the prefetching list
+	void AddReadHead(idx_t pos, uint64_t len, bool merge_buffers = true) {
+		// Attempt to merge with existing
+		if (merge_buffers) {
+			ReadHead new_read_head {pos, len};
+			auto lookup_set = merge_set.find(&new_read_head);
+			if (lookup_set != merge_set.end()) {
+				auto existing_head = *lookup_set;
+				auto new_start = MinValue<idx_t>(existing_head->location, new_read_head.location);
+				auto new_length = MaxValue<idx_t>(existing_head->GetEnd(), new_read_head.GetEnd()) - new_start;
+				existing_head->location = new_start;
+				existing_head->size = new_length;
+				return;
+			}
+		}
+
+		read_heads.emplace_front(ReadHead(pos, len));
+		total_size += len;
+		auto &read_head = read_heads.front();
+
+		if (merge_buffers) {
+			merge_set.insert(&read_head);
+		}
+
+		if (read_head.GetEnd() > handle.GetFileSize()) {
+			throw std::runtime_error("Prefetch registered for bytes outside file");
+		}
+	}
+
+	// Returns the relevant read head
+	ReadHead *GetReadHead(idx_t pos) {
+		for (auto &read_head : read_heads) {
+			if (pos >= read_head.location && pos < read_head.GetEnd()) {
+				return &read_head;
+			}
+		}
+		return nullptr;
+	}
+
+	// Prefetch all read heads
+	void Prefetch() {
+		for (auto &read_head : read_heads) {
+			read_head.Allocate(allocator);
+
+			if (read_head.GetEnd() > handle.GetFileSize()) {
+				throw std::runtime_error("Prefetch registered requested for bytes outside file");
+			}
+
+			handle.Read(read_head.data->get(), read_head.size, read_head.location);
+			read_head.data_isset = true;
+		}
+	}
+};
+
 class ThriftFileTransport : public duckdb_apache::thrift::transport::TVirtualTransport<ThriftFileTransport> {
 public:
-	ThriftFileTransport(Allocator &allocator, FileHandle &handle_p)
-	    : allocator(allocator), handle(handle_p), location(0) {
+	static constexpr uint64_t PREFETCH_FALLBACK_BUFFERSIZE = 1000000;
+
+	ThriftFileTransport(Allocator &allocator, FileHandle &handle_p, FileOpener &opener, bool prefetch_mode_p)
+	    : handle(handle_p), location(0), allocator(allocator), ra_buffer(ReadAheadBuffer(allocator, handle_p, opener)),
+	      prefetch_mode(prefetch_mode_p) {
 	}
 
 	uint32_t read(uint8_t *buf, uint32_t len) {
-		if (prefetched_data && location >= prefetch_location &&
-		    location + len < prefetch_location + prefetched_data->GetSize()) {
-			memcpy(buf, prefetched_data->get() + location - prefetch_location, len);
+		auto prefetch_buffer = ra_buffer.GetReadHead(location);
+		if (prefetch_buffer != nullptr && location - prefetch_buffer->location + len <= prefetch_buffer->size) {
+			D_ASSERT(location - prefetch_buffer->location + len <= prefetch_buffer->size);
+
+			if (!prefetch_buffer->data_isset) {
+				prefetch_buffer->Allocate(allocator);
+				handle.Read(prefetch_buffer->data->get(), prefetch_buffer->size, prefetch_buffer->location);
+				prefetch_buffer->data_isset = true;
+			}
+			memcpy(buf, prefetch_buffer->data->get() + location - prefetch_buffer->location, len);
 		} else {
-			handle.Read(buf, len, location);
+			if (prefetch_mode && len < PREFETCH_FALLBACK_BUFFERSIZE && len > 0) {
+				Prefetch(location, MinValue<uint64_t>(PREFETCH_FALLBACK_BUFFERSIZE, handle.GetFileSize() - location));
+				auto prefetch_buffer_fallback = ra_buffer.GetReadHead(location);
+				D_ASSERT(location - prefetch_buffer_fallback->location + len <= prefetch_buffer_fallback->size);
+				memcpy(buf, prefetch_buffer_fallback->data->get() + location - prefetch_buffer_fallback->location, len);
+			} else {
+				handle.Read(buf, len, location);
+			}
 		}
 		location += len;
 		return len;
 	}
 
-	void Prefetch(idx_t pos, idx_t len) {
-		prefetch_location = pos;
-		prefetched_data = allocator.Allocate(len);
-		handle.Read(prefetched_data->get(), len, prefetch_location);
+	// Prefetch a single buffer
+	void Prefetch(idx_t pos, uint64_t len) {
+		RegisterPrefetch(pos, len, false);
+		FinalizeRegistration();
+		PrefetchRegistered();
+	}
+
+	// Register a buffer for prefixing
+	void RegisterPrefetch(idx_t pos, uint64_t len, bool can_merge = true) {
+		ra_buffer.AddReadHead(pos, len, can_merge);
+	}
+
+	// Prevents any further merges, should be called before PrefetchRegistered
+	void FinalizeRegistration() {
+		ra_buffer.merge_set.clear();
+	}
+
+	// Prefetch all previously registered ranges
+	void PrefetchRegistered() {
+		ra_buffer.Prefetch();
 	}
 
 	void ClearPrefetch() {
-		prefetched_data.reset();
+		ra_buffer.read_heads.clear();
+		ra_buffer.merge_set.clear();
 	}
 
 	void SetLocation(idx_t location_p) {
@@ -6955,12 +7100,17 @@ public:
 	}
 
 private:
-	Allocator &allocator;
 	FileHandle &handle;
 	idx_t location;
 
-	unique_ptr<AllocatedData> prefetched_data;
-	idx_t prefetch_location;
+	Allocator &allocator;
+
+	// Multi-buffer prefetch
+	ReadAheadBuffer ra_buffer;
+
+	// Whether the prefetch mode is enabled. In this mode the DirectIO flag of the handle will be set and the parquet
+	// reader will manage the read buffering.
+	bool prefetch_mode;
 };
 
 } // namespace duckdb
@@ -7417,8 +7567,13 @@ public:
 	idx_t MaxDefine() const;
 	idx_t MaxRepeat() const;
 
+	virtual idx_t FileOffset() const;
+	virtual uint64_t TotalCompressedSize();
 	virtual idx_t GroupRowsAvailable();
 
+	// register the range this reader will touch for prefetching
+	virtual void RegisterPrefetch(ThriftFileTransport &transport, bool allow_merge);
+
 	virtual unique_ptr<BaseStatistics> Stats(const std::vector<ColumnChunk> &columns);
 
 protected:
@@ -7433,6 +7588,9 @@ protected:
 	virtual void DictReference(Vector &result);
 	virtual void PlainReference(shared_ptr<ByteBuffer>, Vector &result);
 
+	// applies any skips that were registered using Skip()
+	virtual void ApplyPendingSkips(idx_t num_values);
+
 	bool HasDefines() {
 		return max_define > 0;
 	}
@@ -7451,13 +7609,15 @@ protected:
 	ParquetReader &reader;
 	LogicalType type;
 
+	idx_t pending_skips = 0;
+
 private:
 	void PrepareRead(parquet_filter_t &filter);
 	void PreparePage(idx_t compressed_page_size, idx_t uncompressed_page_size);
 	void PrepareDataPage(PageHeader &page_hdr);
 	void PreparePageV2(PageHeader &page_hdr);
 
-	const duckdb_parquet::format::ColumnChunk *chunk;
+	const duckdb_parquet::format::ColumnChunk *chunk = nullptr;
 
 	duckdb_apache::thrift::protocol::TProtocol *protocol;
 	idx_t page_rows_available;
@@ -7545,6 +7705,11 @@ class ChunkCollection;
 class BaseStatistics;
 class TableFilterSet;
 
+struct ParquetReaderPrefetchConfig {
+	// Percentage of data in a row group span that should be scanned for enabling whole group prefetch
+	static constexpr double WHOLE_GROUP_PREFETCH_MINIMUM_SCAN = 0.95;
+};
+
 struct ParquetReaderScanState {
 	vector<idx_t> group_idx_list;
 	int64_t current_group;
@@ -7560,6 +7725,9 @@ struct ParquetReaderScanState {
 
 	ResizeableBuffer define_buf;
 	ResizeableBuffer repeat_buf;
+
+	bool prefetch_mode = false;
+	bool current_group_prefetched = false;
 };
 
 struct ParquetOptions {
@@ -7568,6 +7736,8 @@ struct ParquetOptions {
 	explicit ParquetOptions(ClientContext &context);
 
 	bool binary_as_string = false;
+	bool filename = false;
+	bool hive_partitioning = false;
 };
 
 class ParquetReader {
@@ -7624,6 +7794,10 @@ private:
 	                                               idx_t depth, idx_t max_define, idx_t max_repeat,
 	                                               idx_t &next_schema_idx, idx_t &next_file_idx);
 	const duckdb_parquet::format::RowGroup &GetGroup(ParquetReaderScanState &state);
+	uint64_t GetGroupCompressedSize(ParquetReaderScanState &state);
+	idx_t GetGroupOffset(ParquetReaderScanState &state);
+	// Group span is the distance between the min page offset and the max page offset plus the max page compressed size
+	uint64_t GetGroupSpan(ParquetReaderScanState &state);
 	void PrepareRowGroupBuffer(ParquetReaderScanState &state, idx_t out_col_idx);
 	LogicalType DeriveLogicalType(const SchemaElement &s_ele);
 
@@ -7681,7 +7855,7 @@ namespace duckdb {
 class BufferedSerializer;
 class ParquetWriter;
 class ColumnWriterPageState;
-class StandardColumnWriterState;
+class BasicColumnWriterState;
 
 class ColumnWriterState {
 public:
@@ -7703,9 +7877,6 @@ public:
 };
 
 class ColumnWriter {
-	//! We limit the uncompressed page size to 100MB
-	// The max size in Parquet is 2GB, but we choose a more conservative limit
-	static constexpr const idx_t MAX_UNCOMPRESSED_PAGE_SIZE = 100000000;
 
 public:
 	ColumnWriter(ParquetWriter &writer, idx_t schema_idx, vector<string> schema_path, idx_t max_repeat,
@@ -7729,46 +7900,35 @@ public:
 	                                                      idx_t max_repeat = 0, idx_t max_define = 1,
 	                                                      bool can_have_nulls = true);
 
-	virtual unique_ptr<ColumnWriterState> InitializeWriteState(duckdb_parquet::format::RowGroup &row_group);
-	virtual void Prepare(ColumnWriterState &state, ColumnWriterState *parent, Vector &vector, idx_t count);
+	virtual unique_ptr<ColumnWriterState> InitializeWriteState(duckdb_parquet::format::RowGroup &row_group) = 0;
 
-	virtual void BeginWrite(ColumnWriterState &state);
-	virtual void Write(ColumnWriterState &state, Vector &vector, idx_t count);
-	virtual void FinalizeWrite(ColumnWriterState &state);
-
-protected:
-	void HandleDefineLevels(ColumnWriterState &state, ColumnWriterState *parent, ValidityMask &validity, idx_t count,
-	                        uint16_t define_value, uint16_t null_value);
-	void HandleRepeatLevels(ColumnWriterState &state_p, ColumnWriterState *parent, idx_t count, idx_t max_repeat);
-
-	void WriteLevels(Serializer &temp_writer, const vector<uint16_t> &levels, idx_t max_value, idx_t start_offset,
-	                 idx_t count);
+	//! indicates whether the write need to analyse the data before preparing it
+	virtual bool HasAnalyze() {
+		return false;
+	}
 
-	virtual duckdb_parquet::format::Encoding::type GetEncoding();
+	virtual void Analyze(ColumnWriterState &state, ColumnWriterState *parent, Vector &vector, idx_t count) {
+		throw NotImplementedException("Writer does not need analysis");
+	}
 
-	void NextPage(ColumnWriterState &state_p);
-	void FlushPage(ColumnWriterState &state_p);
-	void WriteDictionary(ColumnWriterState &state_p, unique_ptr<BufferedSerializer> temp_writer, idx_t row_count);
+	//! Called after all data has been passed to Analyze
+	virtual void FinalizeAnalyze(ColumnWriterState &state) {
+		throw NotImplementedException("Writer does not need analysis");
+	}
 
-	virtual void FlushDictionary(ColumnWriterState &state, ColumnWriterStatistics *stats);
+	virtual void Prepare(ColumnWriterState &state, ColumnWriterState *parent, Vector &vector, idx_t count) = 0;
 
-	//! Initializes the state used to track statistics during writing. Only used for scalar types.
-	virtual unique_ptr<ColumnWriterStatistics> InitializeStatsState();
-	//! Retrieves the row size of a vector at the specified location. Only used for scalar types.
-	virtual idx_t GetRowSize(Vector &vector, idx_t index);
-	//! Writes a (subset of a) vector to the specified serializer. Only used for scalar types.
-	virtual void WriteVector(Serializer &temp_writer, ColumnWriterStatistics *stats, ColumnWriterPageState *page_state,
-	                         Vector &vector, idx_t chunk_start, idx_t chunk_end);
+	virtual void BeginWrite(ColumnWriterState &state) = 0;
+	virtual void Write(ColumnWriterState &state, Vector &vector, idx_t count) = 0;
+	virtual void FinalizeWrite(ColumnWriterState &state) = 0;
 
-	//! Initialize the writer for a specific page. Only used for scalar types.
-	virtual unique_ptr<ColumnWriterPageState> InitializePageState();
-	//! Flushes the writer for a specific page. Only used for scalar types.
-	virtual void FlushPageState(Serializer &temp_writer, ColumnWriterPageState *state);
+protected:
+	void HandleDefineLevels(ColumnWriterState &state, ColumnWriterState *parent, ValidityMask &validity, idx_t count,
+	                        uint16_t define_value, uint16_t null_value);
+	void HandleRepeatLevels(ColumnWriterState &state_p, ColumnWriterState *parent, idx_t count, idx_t max_repeat);
 
 	void CompressPage(BufferedSerializer &temp_writer, size_t &compressed_size, data_ptr_t &compressed_data,
 	                  unique_ptr<data_t[]> &compressed_buf);
-
-	void SetParquetStatistics(StandardColumnWriterState &state, duckdb_parquet::format::ColumnChunk &column);
 };
 
 } // namespace duckdb
@@ -7781,6 +7941,7 @@ class FileOpener;
 
 class ParquetWriter {
 	friend class ColumnWriter;
+	friend class BasicColumnWriter;
 	friend class ListColumnWriter;
 	friend class StructColumnWriter;