duckdb 0.4.1-dev182.0 → 0.4.1-dev1896.0

package/src/parquet-amalgamation.hpp

@@ -34,6 +34,10 @@ public:
 
 #include "duckdb.hpp"
 #ifndef DUCKDB_AMALGAMATION
+#include "duckdb/planner/table_filter.hpp"
+#include "duckdb/planner/filter/constant_filter.hpp"
+#include "duckdb/planner/filter/null_filter.hpp"
+#include "duckdb/planner/filter/conjunction_filter.hpp"
 #include "duckdb/common/common.hpp"
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/string_util.hpp"
@@ -91,8 +95,8 @@ public:
  * under the License.
  */
 
-#ifndef _THRIFT_THRIFT_H_
-#define _THRIFT_THRIFT_H_ 1
+#ifndef _DUCKDB_THRIFT_THRIFT_H_
+#define _DUCKDB_THRIFT_THRIFT_H_ 1
 
 
 
@@ -121,8 +125,8 @@ public:
 
 // clang-format off
 
-#ifndef _THRIFT_TRANSPORT_PLATFORM_SOCKET_H_
-#  define _THRIFT_TRANSPORT_PLATFORM_SOCKET_H_
+#ifndef _DUCKDB_THRIFT_TRANSPORT_PLATFORM_SOCKET_H_
+#  define _DUCKDB_THRIFT_TRANSPORT_PLATFORM_SOCKET_H_
 
 #ifdef _WIN32
 #ifdef _WINSOCKAPI_
@@ -236,7 +240,7 @@ public:
 #  define THRIFT_SHUT_RDWR SHUT_RDWR
 #endif
 
-#endif // _THRIFT_TRANSPORT_PLATFORM_SOCKET_H_
+#endif // _DUCKDB_THRIFT_TRANSPORT_PLATFORM_SOCKET_H_
 
 
 // LICENSE_CHANGE_END
@@ -329,8 +333,8 @@ public:
  * under the License.
  */
 
-#ifndef _THRIFT_TLOGGING_H_
-#define _THRIFT_TLOGGING_H_ 1
+#ifndef _DUCKDB_THRIFT_TLOGGING_H_
+#define _DUCKDB_THRIFT_TLOGGING_H_ 1
 
 
 
@@ -454,7 +458,7 @@ public:
 #define T_GENERIC_PROTOCOL(template_class, generic_prot, specific_prot)
 #endif
 
-#endif // #ifndef _THRIFT_TLOGGING_H_
+#endif // #ifndef _DUCKDB_THRIFT_TLOGGING_H_
 
 
 // LICENSE_CHANGE_END
@@ -546,7 +550,7 @@ void profile_write_pprof(FILE* gen_calls_f, FILE* virtual_calls_f);
 }
 } // duckdb_apache::thrift
 
-#endif // #ifndef _THRIFT_THRIFT_H_
+#endif // #ifndef _DUCKDB_THRIFT_THRIFT_H_
 
 
 // LICENSE_CHANGE_END
@@ -576,8 +580,8 @@ void profile_write_pprof(FILE* gen_calls_f, FILE* virtual_calls_f);
  * under the License.
  */
 
-#ifndef _THRIFT_TAPPLICATIONEXCEPTION_H_
-#define _THRIFT_TAPPLICATIONEXCEPTION_H_ 1
+#ifndef _DUCKDB_THRIFT_TAPPLICATIONEXCEPTION_H_
+#define _DUCKDB_THRIFT_TAPPLICATIONEXCEPTION_H_ 1
 
 
 
@@ -671,7 +675,7 @@ protected:
 }
 } // duckdb_apache::thrift
 
-#endif // #ifndef _THRIFT_TAPPLICATIONEXCEPTION_H_
+#endif // #ifndef _DUCKDB_THRIFT_TAPPLICATIONEXCEPTION_H_
 
 
 // LICENSE_CHANGE_END
@@ -701,8 +705,8 @@ protected:
  * under the License.
  */
 
-#ifndef _THRIFT_TBASE_H_
-#define _THRIFT_TBASE_H_ 1
+#ifndef _DUCKDB_THRIFT_TBASE_H_
+#define _DUCKDB_THRIFT_TBASE_H_ 1
 
 
 
@@ -730,8 +734,8 @@ protected:
  * under the License.
  */
 
-#ifndef _THRIFT_PROTOCOL_TPROTOCOL_H_
-#define _THRIFT_PROTOCOL_TPROTOCOL_H_ 1
+#ifndef _DUCKDB_THRIFT_PROTOCOL_TPROTOCOL_H_
+#define _DUCKDB_THRIFT_PROTOCOL_TPROTOCOL_H_ 1
 
 #ifdef _WIN32
 // Need to come before any Windows.h includes
@@ -763,8 +767,8 @@ protected:
  * under the License.
  */
 
-#ifndef _THRIFT_TRANSPORT_TTRANSPORT_H_
-#define _THRIFT_TRANSPORT_TTRANSPORT_H_ 1
+#ifndef _DUCKDB_THRIFT_TRANSPORT_TTRANSPORT_H_
+#define _DUCKDB_THRIFT_TRANSPORT_TTRANSPORT_H_ 1
 
 
 
@@ -792,8 +796,8 @@ protected:
  * under the License.
  */
 
-#ifndef _THRIFT_TRANSPORT_TTRANSPORTEXCEPTION_H_
-#define _THRIFT_TRANSPORT_TTRANSPORTEXCEPTION_H_ 1
+#ifndef _DUCKDB_THRIFT_TRANSPORT_TTRANSPORTEXCEPTION_H_
+#define _DUCKDB_THRIFT_TRANSPORT_TTRANSPORTEXCEPTION_H_ 1
 
 // FUCK OFF #include <boost/numeric/conversion/cast.hpp>
 #include <string>
@@ -878,7 +882,7 @@ protected:
 }
 } // duckdb_apache::thrift::transport
 
-#endif // #ifndef _THRIFT_TRANSPORT_TTRANSPORTEXCEPTION_H_
+#endif // #ifndef _DUCKDB_THRIFT_TRANSPORT_TTRANSPORTEXCEPTION_H_
 
 
 // LICENSE_CHANGE_END
@@ -1129,7 +1133,7 @@ public:
 }
 } // duckdb_apache::thrift::transport
 
-#endif // #ifndef _THRIFT_TRANSPORT_TTRANSPORT_H_
+#endif // #ifndef _DUCKDB_THRIFT_TRANSPORT_TTRANSPORT_H_
 
 
 // LICENSE_CHANGE_END
@@ -1159,8 +1163,8 @@ public:
  * under the License.
  */
 
-#ifndef _THRIFT_PROTOCOL_TPROTOCOLEXCEPTION_H_
-#define _THRIFT_PROTOCOL_TPROTOCOLEXCEPTION_H_ 1
+#ifndef _DUCKDB_THRIFT_PROTOCOL_TPROTOCOLEXCEPTION_H_
+#define _DUCKDB_THRIFT_PROTOCOL_TPROTOCOLEXCEPTION_H_ 1
 
 #include <string>
 
@@ -1244,7 +1248,7 @@ protected:
 }
 } // duckdb_apache::thrift::protocol
 
-#endif // #ifndef _THRIFT_PROTOCOL_TPROTOCOLEXCEPTION_H_
+#endif // #ifndef _DUCKDB_THRIFT_PROTOCOL_TPROTOCOLEXCEPTION_H_
 
 
 // LICENSE_CHANGE_END
@@ -1267,37 +1271,39 @@ protected:
 // but that doesn't work.
 // For a pretty in-depth explanation of the problem, see
 // http://cellperformance.beyond3d.com/articles/2006/06/understanding-strict-aliasing.html
+namespace duckdb_apache { namespace thrift {
 template <typename To, typename From>
 static inline To bitwise_cast(From from) {
-  static_assert(sizeof(From) == sizeof(To), "sizeof(From) == sizeof(To)");
-
-  // BAD!!!  These are all broken with -O2.
-  //return *reinterpret_cast<To*>(&from);  // BAD!!!
-  //return *static_cast<To*>(static_cast<void*>(&from));  // BAD!!!
-  //return *(To*)(void*)&from;  // BAD!!!
-
-  // Super clean and paritally blessed by section 3.9 of the standard.
-  //unsigned char c[sizeof(from)];
-  //memcpy(c, &from, sizeof(from));
-  //To to;
-  //memcpy(&to, c, sizeof(c));
-  //return to;
-
-  // Slightly more questionable.
-  // Same code emitted by GCC.
-  //To to;
-  //memcpy(&to, &from, sizeof(from));
-  //return to;
-
-  // Technically undefined, but almost universally supported,
-  // and the most efficient implementation.
-  union {
-    From f;
-    To t;
-  } u;
-  u.f = from;
-  return u.t;
+	static_assert(sizeof(From) == sizeof(To), "sizeof(From) == sizeof(To)");
+
+	// BAD!!!  These are all broken with -O2.
+	// return *reinterpret_cast<To*>(&from);  // BAD!!!
+	// return *static_cast<To*>(static_cast<void*>(&from));  // BAD!!!
+	// return *(To*)(void*)&from;  // BAD!!!
+
+	// Super clean and paritally blessed by section 3.9 of the standard.
+	// unsigned char c[sizeof(from)];
+	// memcpy(c, &from, sizeof(from));
+	// To to;
+	// memcpy(&to, c, sizeof(c));
+	// return to;
+
+	// Slightly more questionable.
+	// Same code emitted by GCC.
+	// To to;
+	// memcpy(&to, &from, sizeof(from));
+	// return to;
+
+	// Technically undefined, but almost universally supported,
+	// and the most efficient implementation.
+	union {
+		From f;
+		To t;
+	} u;
+	u.f = from;
+	return u.t;
 }
+}} // namespace duckdb_apache::thrift
 
 
 #ifdef HAVE_SYS_PARAM_H
@@ -1983,7 +1989,7 @@ uint32_t skip(Protocol_& prot, TType type) {
 
 }}} // duckdb_apache::thrift::protocol
 
-#endif // #define _THRIFT_PROTOCOL_TPROTOCOL_H_ 1
+#endif // #define _DUCKDB_THRIFT_PROTOCOL_TPROTOCOL_H_ 1
 
 
 // LICENSE_CHANGE_END
@@ -2001,7 +2007,7 @@ public:
 }
 } // duckdb_apache::thrift
 
-#endif // #ifndef _THRIFT_TBASE_H_
+#endif // #ifndef _DUCKDB_THRIFT_TBASE_H_
 
 
 // LICENSE_CHANGE_END
@@ -4651,8 +4657,8 @@ std::ostream& operator<<(std::ostream& out, const FileCryptoMetaData& obj);
  * under the License.
  */
 
-#ifndef _THRIFT_PROTOCOL_TCOMPACTPROTOCOL_H_
-#define _THRIFT_PROTOCOL_TCOMPACTPROTOCOL_H_ 1
+#ifndef _DUCKDB_THRIFT_PROTOCOL_TCOMPACTPROTOCOL_H_
+#define _DUCKDB_THRIFT_PROTOCOL_TCOMPACTPROTOCOL_H_ 1
 
 
 
@@ -4679,8 +4685,8 @@ std::ostream& operator<<(std::ostream& out, const FileCryptoMetaData& obj);
  * under the License.
  */
 
-#ifndef _THRIFT_PROTOCOL_TVIRTUALPROTOCOL_H_
-#define _THRIFT_PROTOCOL_TVIRTUALPROTOCOL_H_ 1
+#ifndef _DUCKDB_THRIFT_PROTOCOL_TVIRTUALPROTOCOL_H_
+#define _DUCKDB_THRIFT_PROTOCOL_TVIRTUALPROTOCOL_H_ 1
 
 
 
@@ -5172,7 +5178,7 @@ protected:
 }
 } // duckdb_apache::thrift::protocol
 
-#endif // #define _THRIFT_PROTOCOL_TVIRTUALPROTOCOL_H_ 1
+#endif // #define _DUCKDB_THRIFT_PROTOCOL_TVIRTUALPROTOCOL_H_ 1
 
 
 // LICENSE_CHANGE_END
@@ -5441,8 +5447,8 @@ typedef TCompactProtocolFactoryT<TTransport> TCompactProtocolFactory;
  * specific language governing permissions and limitations
  * under the License.
  */
-#ifndef _THRIFT_PROTOCOL_TCOMPACTPROTOCOL_TCC_
-#define _THRIFT_PROTOCOL_TCOMPACTPROTOCOL_TCC_ 1
+#ifndef _DUCKDB_THRIFT_PROTOCOL_TCOMPACTPROTOCOL_TCC_
+#define _DUCKDB_THRIFT_PROTOCOL_TCOMPACTPROTOCOL_TCC_ 1
 
 #include <limits>
 
@@ -6248,7 +6254,7 @@ TType TCompactProtocolT<Transport_>::getTType(int8_t type) {
 
 }}} // duckdb_apache::thrift::protocol
 
-#endif // _THRIFT_PROTOCOL_TCOMPACTPROTOCOL_TCC_
+#endif // _DUCKDB_THRIFT_PROTOCOL_TCOMPACTPROTOCOL_TCC_
 
 
 // LICENSE_CHANGE_END
@@ -6284,8 +6290,8 @@ TType TCompactProtocolT<Transport_>::getTType(int8_t type) {
  * under the License.
  */
 
-#ifndef _THRIFT_TRANSPORT_TBUFFERTRANSPORTS_H_
-#define _THRIFT_TRANSPORT_TBUFFERTRANSPORTS_H_ 1
+#ifndef _DUCKDB_THRIFT_TRANSPORT_TBUFFERTRANSPORTS_H_
+#define _DUCKDB_THRIFT_TRANSPORT_TBUFFERTRANSPORTS_H_ 1
 
 #include <cstdlib>
 #include <cstddef>
@@ -6319,8 +6325,8 @@ TType TCompactProtocolT<Transport_>::getTType(int8_t type) {
  * under the License.
  */
 
-#ifndef _THRIFT_TRANSPORT_TVIRTUALTRANSPORT_H_
-#define _THRIFT_TRANSPORT_TVIRTUALTRANSPORT_H_ 1
+#ifndef _DUCKDB_THRIFT_TRANSPORT_TVIRTUALTRANSPORT_H_
+#define _DUCKDB_THRIFT_TRANSPORT_TVIRTUALTRANSPORT_H_ 1
 
 
 
@@ -6439,7 +6445,7 @@ protected:
 }
 } // duckdb_apache::thrift::transport
 
-#endif // #ifndef _THRIFT_TRANSPORT_TVIRTUALTRANSPORT_H_
+#endif // #ifndef _DUCKDB_THRIFT_TRANSPORT_TVIRTUALTRANSPORT_H_
 
 
 // LICENSE_CHANGE_END
@@ -6902,7 +6908,7 @@ protected:
 }
 } // duckdb_apache::thrift::transport
 
-#endif // #ifndef _THRIFT_TRANSPORT_TBUFFERTRANSPORTS_H_
+#endif // #ifndef _DUCKDB_THRIFT_TRANSPORT_TBUFFERTRANSPORTS_H_
 
 
 // LICENSE_CHANGE_END
@@ -6924,7 +6930,7 @@ struct ReadHead {
 	uint64_t size;
 
 	// Current info
-	unique_ptr<AllocatedData> data;
+	AllocatedData data;
 	bool data_isset = false;
 
 	idx_t GetEnd() const {
@@ -7019,7 +7025,7 @@ struct ReadAheadBuffer {
 				throw std::runtime_error("Prefetch registered requested for bytes outside file");
 			}
 
-			handle.Read(read_head.data->get(), read_head.size, read_head.location);
+			handle.Read(read_head.data.get(), read_head.size, read_head.location);
 			read_head.data_isset = true;
 		}
 	}
@@ -7041,16 +7047,16 @@ public:
 
 			if (!prefetch_buffer->data_isset) {
 				prefetch_buffer->Allocate(allocator);
-				handle.Read(prefetch_buffer->data->get(), prefetch_buffer->size, prefetch_buffer->location);
+				handle.Read(prefetch_buffer->data.get(), prefetch_buffer->size, prefetch_buffer->location);
 				prefetch_buffer->data_isset = true;
 			}
-			memcpy(buf, prefetch_buffer->data->get() + location - prefetch_buffer->location, len);
+			memcpy(buf, prefetch_buffer->data.get() + location - prefetch_buffer->location, len);
 		} else {
 			if (prefetch_mode && len < PREFETCH_FALLBACK_BUFFERSIZE && len > 0) {
 				Prefetch(location, MinValue<uint64_t>(PREFETCH_FALLBACK_BUFFERSIZE, handle.GetFileSize() - location));
 				auto prefetch_buffer_fallback = ra_buffer.GetReadHead(location);
 				D_ASSERT(location - prefetch_buffer_fallback->location + len <= prefetch_buffer_fallback->size);
-				memcpy(buf, prefetch_buffer_fallback->data->get() + location - prefetch_buffer_fallback->location, len);
+				memcpy(buf, prefetch_buffer_fallback->data.get() + location - prefetch_buffer_fallback->location, len);
 			} else {
 				handle.Read(buf, len, location);
 			}
@@ -7191,12 +7197,12 @@ public:
 		if (new_size > alloc_len) {
 			alloc_len = new_size;
 			allocated_data = allocator.Allocate(alloc_len);
-			ptr = (char *)allocated_data->get();
+			ptr = (char *)allocated_data.get();
 		}
 	}
 
 private:
-	unique_ptr<AllocatedData> allocated_data;
+	AllocatedData allocated_data;
 	idx_t alloc_len = 0;
 };
 
@@ -7699,7 +7705,6 @@ class FileMetaData;
 namespace duckdb {
 class Allocator;
 class ClientContext;
-class ChunkCollection;
 class BaseStatistics;
 class TableFilterSet;
 
@@ -7734,6 +7739,12 @@ struct ParquetOptions {
 	explicit ParquetOptions(ClientContext &context);
 
 	bool binary_as_string = false;
+	bool filename = false;
+	bool hive_partitioning = false;
+
+public:
+	void Serialize(FieldWriter &writer) const;
+	void Deserialize(FieldReader &reader);
 };
 
 class ParquetReader {
@@ -7830,7 +7841,7 @@ private:
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/mutex.hpp"
 #include "duckdb/common/serializer/buffered_file_writer.hpp"
-#include "duckdb/common/types/chunk_collection.hpp"
+#include "duckdb/common/types/column_data_collection.hpp"
 #endif
 
 
@@ -7851,7 +7862,7 @@ namespace duckdb {
 class BufferedSerializer;
 class ParquetWriter;
 class ColumnWriterPageState;
-class StandardColumnWriterState;
+class BasicColumnWriterState;
 
 class ColumnWriterState {
 public:
@@ -7873,9 +7884,6 @@ public:
 };
 
 class ColumnWriter {
-	//! We limit the uncompressed page size to 100MB
-	// The max size in Parquet is 2GB, but we choose a more conservative limit
-	static constexpr const idx_t MAX_UNCOMPRESSED_PAGE_SIZE = 100000000;
 
 public:
 	ColumnWriter(ParquetWriter &writer, idx_t schema_idx, vector<string> schema_path, idx_t max_repeat,
@@ -7899,46 +7907,35 @@ public:
 	                                                      idx_t max_repeat = 0, idx_t max_define = 1,
 	                                                      bool can_have_nulls = true);
 
-	virtual unique_ptr<ColumnWriterState> InitializeWriteState(duckdb_parquet::format::RowGroup &row_group);
-	virtual void Prepare(ColumnWriterState &state, ColumnWriterState *parent, Vector &vector, idx_t count);
+	virtual unique_ptr<ColumnWriterState> InitializeWriteState(duckdb_parquet::format::RowGroup &row_group) = 0;
 
-	virtual void BeginWrite(ColumnWriterState &state);
-	virtual void Write(ColumnWriterState &state, Vector &vector, idx_t count);
-	virtual void FinalizeWrite(ColumnWriterState &state);
-
-protected:
-	void HandleDefineLevels(ColumnWriterState &state, ColumnWriterState *parent, ValidityMask &validity, idx_t count,
-	                        uint16_t define_value, uint16_t null_value);
-	void HandleRepeatLevels(ColumnWriterState &state_p, ColumnWriterState *parent, idx_t count, idx_t max_repeat);
-
-	void WriteLevels(Serializer &temp_writer, const vector<uint16_t> &levels, idx_t max_value, idx_t start_offset,
-	                 idx_t count);
+	//! indicates whether the write need to analyse the data before preparing it
+	virtual bool HasAnalyze() {
+		return false;
+	}
 
-	virtual duckdb_parquet::format::Encoding::type GetEncoding();
+	virtual void Analyze(ColumnWriterState &state, ColumnWriterState *parent, Vector &vector, idx_t count) {
+		throw NotImplementedException("Writer does not need analysis");
+	}
 
-	void NextPage(ColumnWriterState &state_p);
-	void FlushPage(ColumnWriterState &state_p);
-	void WriteDictionary(ColumnWriterState &state_p, unique_ptr<BufferedSerializer> temp_writer, idx_t row_count);
+	//! Called after all data has been passed to Analyze
+	virtual void FinalizeAnalyze(ColumnWriterState &state) {
+		throw NotImplementedException("Writer does not need analysis");
+	}
 
-	virtual void FlushDictionary(ColumnWriterState &state, ColumnWriterStatistics *stats);
+	virtual void Prepare(ColumnWriterState &state, ColumnWriterState *parent, Vector &vector, idx_t count) = 0;
 
-	//! Initializes the state used to track statistics during writing. Only used for scalar types.
-	virtual unique_ptr<ColumnWriterStatistics> InitializeStatsState();
-	//! Retrieves the row size of a vector at the specified location. Only used for scalar types.
-	virtual idx_t GetRowSize(Vector &vector, idx_t index);
-	//! Writes a (subset of a) vector to the specified serializer. Only used for scalar types.
-	virtual void WriteVector(Serializer &temp_writer, ColumnWriterStatistics *stats, ColumnWriterPageState *page_state,
-	                         Vector &vector, idx_t chunk_start, idx_t chunk_end);
+	virtual void BeginWrite(ColumnWriterState &state) = 0;
+	virtual void Write(ColumnWriterState &state, Vector &vector, idx_t count) = 0;
+	virtual void FinalizeWrite(ColumnWriterState &state) = 0;
 
-	//! Initialize the writer for a specific page. Only used for scalar types.
-	virtual unique_ptr<ColumnWriterPageState> InitializePageState();
-	//! Flushes the writer for a specific page. Only used for scalar types.
-	virtual void FlushPageState(Serializer &temp_writer, ColumnWriterPageState *state);
+protected:
+	void HandleDefineLevels(ColumnWriterState &state, ColumnWriterState *parent, ValidityMask &validity, idx_t count,
+	                        uint16_t define_value, uint16_t null_value);
+	void HandleRepeatLevels(ColumnWriterState &state_p, ColumnWriterState *parent, idx_t count, idx_t max_repeat);
 
 	void CompressPage(BufferedSerializer &temp_writer, size_t &compressed_size, data_ptr_t &compressed_data,
 	                  unique_ptr<data_t[]> &compressed_buf);
-
-	void SetParquetStatistics(StandardColumnWriterState &state, duckdb_parquet::format::ColumnChunk &column);
 };
 
 } // namespace duckdb
@@ -7951,6 +7948,7 @@ class FileOpener;
 
 class ParquetWriter {
 	friend class ColumnWriter;
+	friend class BasicColumnWriter;
 	friend class ListColumnWriter;
 	friend class StructColumnWriter;
 
@@ -7959,7 +7957,7 @@ public:
 	              vector<string> names, duckdb_parquet::format::CompressionCodec::type codec);
 
 public:
-	void Flush(ChunkCollection &buffer);
+	void Flush(ColumnDataCollection &buffer);
 	void Finalize();
 
 	static duckdb_parquet::format::Type::type DuckDBTypeToParquetType(const LogicalType &duckdb_type);