duckdb 0.3.2-dev816.0 → 0.3.2-dev838.0

package/src/parquet-amalgamation.hpp

@@ -7232,8 +7232,14 @@ using duckdb_parquet::format::SchemaElement;
 
 struct LogicalType;
 
-unique_ptr<BaseStatistics> ParquetTransformColumnStatistics(const SchemaElement &s_ele, const LogicalType &type,
-                                                            const ColumnChunk &column_chunk);
+struct ParquetStatisticsUtils {
+
+	static unique_ptr<BaseStatistics> TransformColumnStatistics(const SchemaElement &s_ele, const LogicalType &type,
+	                                                            const ColumnChunk &column_chunk);
+
+	static Value ConvertValue(const LogicalType &type, const duckdb_parquet::format::SchemaElement &schema_ele,
+	                          const std::string &stats);
+};
 
 } // namespace duckdb
 
@@ -7264,80 +7270,38 @@ typedef std::bitset<STANDARD_VECTOR_SIZE> parquet_filter_t;
 
 class ColumnReader {
 public:
-	static unique_ptr<ColumnReader> CreateReader(ParquetReader &reader, const LogicalType &type_p,
-	                                             const SchemaElement &schema_p, idx_t schema_idx_p, idx_t max_define,
-	                                             idx_t max_repeat);
-
 	ColumnReader(ParquetReader &reader, LogicalType type_p, const SchemaElement &schema_p, idx_t file_idx_p,
 	             idx_t max_define_p, idx_t max_repeat_p);
-
-	virtual void InitializeRead(const std::vector<ColumnChunk> &columns, TProtocol &protocol_p) {
-		D_ASSERT(file_idx < columns.size());
-		chunk = &columns[file_idx];
-		protocol = &protocol_p;
-		D_ASSERT(chunk);
-		D_ASSERT(chunk->__isset.meta_data);
-
-		if (chunk->__isset.file_path) {
-			throw std::runtime_error("Only inlined data files are supported (no references)");
-		}
-
-		// ugh. sometimes there is an extra offset for the dict. sometimes it's wrong.
-		chunk_read_offset = chunk->meta_data.data_page_offset;
-		if (chunk->meta_data.__isset.dictionary_page_offset && chunk->meta_data.dictionary_page_offset >= 4) {
-			// this assumes the data pages follow the dict pages directly.
-			chunk_read_offset = chunk->meta_data.dictionary_page_offset;
-		}
-		group_rows_available = chunk->meta_data.num_values;
-	}
 	virtual ~ColumnReader();
 
+public:
+	static unique_ptr<ColumnReader> CreateReader(ParquetReader &reader, const LogicalType &type_p,
+	                                             const SchemaElement &schema_p, idx_t schema_idx_p, idx_t max_define,
+	                                             idx_t max_repeat);
+	virtual void InitializeRead(const std::vector<ColumnChunk> &columns, TProtocol &protocol_p);
 	virtual idx_t Read(uint64_t num_values, parquet_filter_t &filter, uint8_t *define_out, uint8_t *repeat_out,
 	                   Vector &result_out);
 
 	virtual void Skip(idx_t num_values);
 
-	const LogicalType &Type() {
-		return type;
-	}
-
-	const SchemaElement &Schema() {
-		return schema;
-	}
+	const LogicalType &Type();
+	const SchemaElement &Schema();
 
-	virtual idx_t GroupRowsAvailable() {
-		return group_rows_available;
-	}
+	virtual idx_t GroupRowsAvailable();
 
-	unique_ptr<BaseStatistics> Stats(const std::vector<ColumnChunk> &columns) {
-		if (Type().id() == LogicalTypeId::LIST || Type().id() == LogicalTypeId::STRUCT ||
-		    Type().id() == LogicalTypeId::MAP) {
-			return nullptr;
-		}
-		return ParquetTransformColumnStatistics(Schema(), Type(), columns[file_idx]);
-	}
+	unique_ptr<BaseStatistics> Stats(const std::vector<ColumnChunk> &columns);
 
 protected:
 	// readers that use the default Read() need to implement those
 	virtual void Plain(shared_ptr<ByteBuffer> plain_data, uint8_t *defines, idx_t num_values, parquet_filter_t &filter,
-	                   idx_t result_offset, Vector &result) {
-		throw NotImplementedException("Plain");
-	}
-
-	virtual void Dictionary(shared_ptr<ByteBuffer> dictionary_data, idx_t num_entries) {
-		throw NotImplementedException("Dictionary");
-	}
-
+	                   idx_t result_offset, Vector &result);
+	virtual void Dictionary(shared_ptr<ByteBuffer> dictionary_data, idx_t num_entries);
 	virtual void Offsets(uint32_t *offsets, uint8_t *defines, idx_t num_values, parquet_filter_t &filter,
-	                     idx_t result_offset, Vector &result) {
-		throw NotImplementedException("Offsets");
-	}
+	                     idx_t result_offset, Vector &result);
 
 	// these are nops for most types, but not for strings
-	virtual void DictReference(Vector &result) {
-	}
-	virtual void PlainReference(shared_ptr<ByteBuffer>, Vector &result) {
-	}
+	virtual void DictReference(Vector &result);
+	virtual void PlainReference(shared_ptr<ByteBuffer>, Vector &result);
 
 	bool HasDefines() {
 		return max_define > 0;
@@ -7509,6 +7473,7 @@ public:
 
 	static unique_ptr<BaseStatistics> ReadStatistics(ParquetReader &reader, LogicalType &type, column_t column_index,
 	                                                 const duckdb_parquet::format::FileMetaData *file_meta_data);
+	static LogicalType DeriveLogicalType(const SchemaElement &s_ele, bool binary_as_string);
 
 private:
 	void InitializeSchema(const vector<LogicalType> &expected_types_p, const string &initial_filename_p);
@@ -7570,6 +7535,7 @@ namespace duckdb {
 class BufferedSerializer;
 class ParquetWriter;
 class ColumnWriterPageState;
+class StandardColumnWriterState;
 
 class ColumnWriterState {
 public:
@@ -7580,30 +7546,44 @@ public:
 	vector<bool> is_empty;
 };
 
+class ColumnWriterStatistics {
+public:
+	virtual ~ColumnWriterStatistics();
+
+	virtual string GetMin();
+	virtual string GetMax();
+	virtual string GetMinValue();
+	virtual string GetMaxValue();
+};
+
 class ColumnWriter {
 	//! We limit the uncompressed page size to 100MB
 	// The max size in Parquet is 2GB, but we choose a more conservative limit
 	static constexpr const idx_t MAX_UNCOMPRESSED_PAGE_SIZE = 100000000;
 
 public:
-	ColumnWriter(ParquetWriter &writer, idx_t schema_idx, idx_t max_repeat, idx_t max_define, bool can_have_nulls);
+	ColumnWriter(ParquetWriter &writer, idx_t schema_idx, vector<string> schema_path, idx_t max_repeat,
+	             idx_t max_define, bool can_have_nulls);
 	virtual ~ColumnWriter();
 
 	ParquetWriter &writer;
 	idx_t schema_idx;
+	vector<string> schema_path;
 	idx_t max_repeat;
 	idx_t max_define;
 	bool can_have_nulls;
+	// collected stats
+	idx_t null_count;
 
 public:
 	//! Create the column writer for a specific type recursively
 	static unique_ptr<ColumnWriter> CreateWriterRecursive(vector<duckdb_parquet::format::SchemaElement> &schemas,
 	                                                      ParquetWriter &writer, const LogicalType &type,
-	                                                      const string &name, idx_t max_repeat = 0,
-	                                                      idx_t max_define = 1, bool can_have_nulls = true);
+	                                                      const string &name, vector<string> schema_path,
+	                                                      idx_t max_repeat = 0, idx_t max_define = 1,
+	                                                      bool can_have_nulls = true);
 
-	virtual unique_ptr<ColumnWriterState> InitializeWriteState(duckdb_parquet::format::RowGroup &row_group,
-	                                                           vector<string> schema_path);
+	virtual unique_ptr<ColumnWriterState> InitializeWriteState(duckdb_parquet::format::RowGroup &row_group);
 	virtual void Prepare(ColumnWriterState &state, ColumnWriterState *parent, Vector &vector, idx_t count);
 
 	virtual void BeginWrite(ColumnWriterState &state);
@@ -7618,14 +7598,22 @@ protected:
 	void WriteLevels(Serializer &temp_writer, const vector<uint16_t> &levels, idx_t max_value, idx_t start_offset,
 	                 idx_t count);
 
+	virtual duckdb_parquet::format::Encoding::type GetEncoding();
+
 	void NextPage(ColumnWriterState &state_p);
 	void FlushPage(ColumnWriterState &state_p);
+	void WriteDictionary(ColumnWriterState &state_p, unique_ptr<BufferedSerializer> temp_writer, idx_t row_count);
+
+	virtual void FlushDictionary(ColumnWriterState &state, ColumnWriterStatistics *stats);
 
+	//! Initializes the state used to track statistics during writing. Only used for scalar types.
+	virtual unique_ptr<ColumnWriterStatistics> InitializeStatsState();
 	//! Retrieves the row size of a vector at the specified location. Only used for scalar types.
-	virtual idx_t GetRowSize(Vector &vector, idx_t index) = 0;
+	virtual idx_t GetRowSize(Vector &vector, idx_t index);
 	//! Writes a (subset of a) vector to the specified serializer. Only used for scalar types.
-	virtual void WriteVector(Serializer &temp_writer, ColumnWriterPageState *page_state, Vector &vector,
-	                         idx_t chunk_start, idx_t chunk_end) = 0;
+	virtual void WriteVector(Serializer &temp_writer, ColumnWriterStatistics *stats, ColumnWriterPageState *page_state,
+	                         Vector &vector, idx_t chunk_start, idx_t chunk_end);
+
 	//! Initialize the writer for a specific page. Only used for scalar types.
 	virtual unique_ptr<ColumnWriterPageState> InitializePageState();
 	//! Flushes the writer for a specific page. Only used for scalar types.
@@ -7633,6 +7621,8 @@ protected:
 
 	void CompressPage(BufferedSerializer &temp_writer, size_t &compressed_size, data_ptr_t &compressed_data,
 	                  unique_ptr<data_t[]> &compressed_buf);
+
+	void SetParquetStatistics(StandardColumnWriterState &state, duckdb_parquet::format::ColumnChunk &column);
 };
 
 } // namespace duckdb
@@ -7657,8 +7647,7 @@ public:
 	void Finalize();
 
 	static duckdb_parquet::format::Type::type DuckDBTypeToParquetType(const LogicalType &duckdb_type);
-	static bool DuckDBTypeToConvertedType(const LogicalType &duckdb_type,
-	                                      duckdb_parquet::format::ConvertedType::type &result);
+	static void SetSchemaProperties(const LogicalType &duckdb_type, duckdb_parquet::format::SchemaElement &schema_ele);
 
 private:
 	string file_name;