duckdb 0.8.2-dev87.0 → 0.9.0

package/src/duckdb/extension/parquet/include/cast_column_reader.hpp

@@ -19,9 +19,9 @@ public:
 	static constexpr const PhysicalType TYPE = PhysicalType::INVALID;
 
 public:
-	CastColumnReader(duckdb::unique_ptr<ColumnReader> child_reader, LogicalType target_type);
+	CastColumnReader(unique_ptr<ColumnReader> child_reader, LogicalType target_type);
 
-	duckdb::unique_ptr<ColumnReader> child_reader;
+	unique_ptr<ColumnReader> child_reader;
 	DataChunk intermediate_chunk;
 
 public:

package/src/duckdb/extension/parquet/include/column_reader.hpp

@@ -8,21 +8,19 @@
 
 #pragma once
 
-#include "parquet_types.h"
-#include "thrift_tools.hpp"
-#include "resizable_buffer.hpp"
-
-#include "parquet_rle_bp_decoder.hpp"
+#include "duckdb.hpp"
 #include "parquet_dbp_decoder.hpp"
+#include "parquet_rle_bp_decoder.hpp"
 #include "parquet_statistics.hpp"
-
-#include "duckdb.hpp"
+#include "parquet_types.h"
+#include "resizable_buffer.hpp"
+#include "thrift_tools.hpp"
 #ifndef DUCKDB_AMALGAMATION
 
-#include "duckdb/common/types/vector.hpp"
-#include "duckdb/common/types/string_type.hpp"
-#include "duckdb/common/types/chunk_collection.hpp"
 #include "duckdb/common/operator/cast_operators.hpp"
+#include "duckdb/common/types/chunk_collection.hpp"
+#include "duckdb/common/types/string_type.hpp"
+#include "duckdb/common/types/vector.hpp"
 #include "duckdb/common/types/vector_cache.hpp"
 #endif
 
@@ -129,7 +127,7 @@ protected:
 
 	ParquetReader &reader;
 	LogicalType type;
-	duckdb::unique_ptr<Vector> byte_array_data;
+	unique_ptr<Vector> byte_array_data;
 	idx_t byte_array_count = 0;
 
 	idx_t pending_skips = 0;
@@ -158,11 +156,11 @@ private:
 	ResizeableBuffer compressed_buffer;
 	ResizeableBuffer offset_buffer;
 
-	duckdb::unique_ptr<RleBpDecoder> dict_decoder;
-	duckdb::unique_ptr<RleBpDecoder> defined_decoder;
-	duckdb::unique_ptr<RleBpDecoder> repeated_decoder;
-	duckdb::unique_ptr<DbpDecoder> dbp_decoder;
-	duckdb::unique_ptr<RleBpDecoder> rle_decoder;
+	unique_ptr<RleBpDecoder> dict_decoder;
+	unique_ptr<RleBpDecoder> defined_decoder;
+	unique_ptr<RleBpDecoder> repeated_decoder;
+	unique_ptr<DbpDecoder> dbp_decoder;
+	unique_ptr<RleBpDecoder> rle_decoder;
 
 	// dummies for Skip()
 	parquet_filter_t none_filter;

package/src/duckdb/extension/parquet/include/column_writer.hpp

@@ -12,10 +12,11 @@
 #include "parquet_types.h"
 
 namespace duckdb {
-class BufferedSerializer;
+class MemoryStream;
 class ParquetWriter;
 class ColumnWriterPageState;
 class BasicColumnWriterState;
+struct ChildFieldIDs;
 
 class ColumnWriterState {
 public:
@@ -78,13 +79,14 @@ public:
 
 public:
 	//! Create the column writer for a specific type recursively
-	static duckdb::unique_ptr<ColumnWriter>
-	CreateWriterRecursive(vector<duckdb_parquet::format::SchemaElement> &schemas, ParquetWriter &writer,
-	                      const LogicalType &type, const string &name, vector<string> schema_path, idx_t max_repeat = 0,
-	                      idx_t max_define = 1, bool can_have_nulls = true);
+	static unique_ptr<ColumnWriter> CreateWriterRecursive(vector<duckdb_parquet::format::SchemaElement> &schemas,
+	                                                      ParquetWriter &writer, const LogicalType &type,
+	                                                      const string &name, vector<string> schema_path,
+	                                                      optional_ptr<const ChildFieldIDs> field_ids,
+	                                                      idx_t max_repeat = 0, idx_t max_define = 1,
+	                                                      bool can_have_nulls = true);
 
-	virtual duckdb::unique_ptr<ColumnWriterState> InitializeWriteState(duckdb_parquet::format::RowGroup &row_group,
-	                                                                   Allocator &allocator) = 0;
+	virtual unique_ptr<ColumnWriterState> InitializeWriteState(duckdb_parquet::format::RowGroup &row_group) = 0;
 
 	//! indicates whether the write need to analyse the data before preparing it
 	virtual bool HasAnalyze() {
@@ -111,8 +113,8 @@ protected:
 	                        uint16_t define_value, uint16_t null_value);
 	void HandleRepeatLevels(ColumnWriterState &state_p, ColumnWriterState *parent, idx_t count, idx_t max_repeat);
 
-	void CompressPage(BufferedSerializer &temp_writer, size_t &compressed_size, data_ptr_t &compressed_data,
-	                  duckdb::unique_ptr<data_t[]> &compressed_buf);
+	void CompressPage(MemoryStream &temp_writer, size_t &compressed_size, data_ptr_t &compressed_data,
+	                  unique_ptr<data_t[]> &compressed_buf);
 };
 
 } // namespace duckdb

package/src/duckdb/extension/parquet/include/list_column_reader.hpp

@@ -19,7 +19,7 @@ public:
 
 public:
 	ListColumnReader(ParquetReader &reader, LogicalType type_p, const SchemaElement &schema_p, idx_t schema_idx_p,
-	                 idx_t max_define_p, idx_t max_repeat_p, duckdb::unique_ptr<ColumnReader> child_column_reader_p);
+	                 idx_t max_define_p, idx_t max_repeat_p, unique_ptr<ColumnReader> child_column_reader_p);
 
 	idx_t Read(uint64_t num_values, parquet_filter_t &filter, data_ptr_t define_out, data_ptr_t repeat_out,
 	           Vector &result_out) override;
@@ -43,7 +43,7 @@ public:
 	}
 
 private:
-	duckdb::unique_ptr<ColumnReader> child_column_reader;
+	unique_ptr<ColumnReader> child_column_reader;
 	ResizeableBuffer child_defines;
 	ResizeableBuffer child_repeats;
 	uint8_t *child_defines_ptr;

package/src/duckdb/extension/parquet/include/parquet_dbp_decoder.hpp

@@ -15,7 +15,7 @@ public:
 		// some derivatives
 		D_ASSERT(miniblocks_per_block > 0);
 		values_per_miniblock = block_value_count / miniblocks_per_block;
-		miniblock_bit_widths = duckdb::unique_ptr<uint8_t[]>(new data_t[miniblocks_per_block]);
+		miniblock_bit_widths = unique_ptr<uint8_t[]>(new data_t[miniblocks_per_block]);
 
 		// init state to something sane
 		values_left_in_block = 0;
@@ -96,7 +96,7 @@ public:
 		if (values_left_in_miniblock == 0) {
 			return;
 		}
-		auto data = duckdb::unique_ptr<uint32_t[]>(new uint32_t[values_left_in_miniblock]);
+		auto data = unique_ptr<uint32_t[]>(new uint32_t[values_left_in_miniblock]);
 		GetBatch<uint32_t>(data_ptr_cast(data.get()), values_left_in_miniblock);
 	}
 
@@ -112,7 +112,7 @@ private:
 	int64_t start_value;
 	idx_t values_per_miniblock;
 
-	duckdb::unique_ptr<uint8_t[]> miniblock_bit_widths;
+	unique_ptr<uint8_t[]> miniblock_bit_widths;
 	idx_t values_left_in_block;
 	idx_t values_left_in_miniblock;
 	idx_t miniblock_offset;

package/src/duckdb/extension/parquet/include/parquet_decimal_utils.hpp

@@ -35,9 +35,9 @@ public:
 		return res;
 	}
 
-	static duckdb::unique_ptr<ColumnReader> CreateReader(ParquetReader &reader, const LogicalType &type_p,
-	                                                     const SchemaElement &schema_p, idx_t file_idx_p,
-	                                                     idx_t max_define, idx_t max_repeat);
+	static unique_ptr<ColumnReader> CreateReader(ParquetReader &reader, const LogicalType &type_p,
+	                                             const SchemaElement &schema_p, idx_t file_idx_p, idx_t max_define,
+	                                             idx_t max_repeat);
 };
 
 } // namespace duckdb

package/src/duckdb/extension/parquet/include/parquet_file_metadata_cache.hpp

@@ -20,14 +20,14 @@ class ParquetFileMetadataCache : public ObjectCacheEntry {
 public:
 	ParquetFileMetadataCache() : metadata(nullptr) {
 	}
-	ParquetFileMetadataCache(duckdb::unique_ptr<duckdb_parquet::format::FileMetaData> file_metadata, time_t r_time)
+	ParquetFileMetadataCache(unique_ptr<duckdb_parquet::format::FileMetaData> file_metadata, time_t r_time)
 	    : metadata(std::move(file_metadata)), read_time(r_time) {
 	}
 
 	~ParquetFileMetadataCache() override = default;
 
 	//! Parquet file metadata
-	duckdb::unique_ptr<const duckdb_parquet::format::FileMetaData> metadata;
+	unique_ptr<const duckdb_parquet::format::FileMetaData> metadata;
 
 	//! read time
 	time_t read_time;

package/src/duckdb/extension/parquet/include/parquet_reader.hpp

@@ -74,8 +74,8 @@ struct ParquetOptions {
 	MultiFileReaderOptions file_options;
 
 public:
-	void Serialize(FieldWriter &writer) const;
-	void Deserialize(FieldReader &reader);
+	void Serialize(Serializer &serializer) const;
+	static ParquetOptions Deserialize(Deserializer &deserializer);
 };
 
 class ParquetReader {
@@ -93,6 +93,7 @@ public:
 	shared_ptr<ParquetFileMetadataCache> metadata;
 	ParquetOptions parquet_options;
 	MultiFileReaderData reader_data;
+	unique_ptr<ColumnReader> root_reader;
 
 public:
 	void InitializeScan(ParquetReaderScanState &state, vector<idx_t> groups_to_read);

package/src/duckdb/extension/parquet/include/parquet_rle_bp_encoder.hpp

@@ -26,9 +26,9 @@ public:
 	void PrepareValue(uint32_t value);
 	void FinishPrepare();
 
-	void BeginWrite(Serializer &writer, uint32_t first_value);
-	void WriteValue(Serializer &writer, uint32_t value);
-	void FinishWrite(Serializer &writer);
+	void BeginWrite(WriteStream &writer, uint32_t first_value);
+	void WriteValue(WriteStream &writer, uint32_t value);
+	void FinishWrite(WriteStream &writer);
 
 	idx_t GetByteCount();
 
@@ -43,7 +43,7 @@ private:
 
 private:
 	void FinishRun();
-	void WriteRun(Serializer &writer);
+	void WriteRun(WriteStream &writer);
 };
 
 } // namespace duckdb

package/src/duckdb/extension/parquet/include/parquet_statistics.hpp

@@ -15,8 +15,8 @@ struct LogicalType;
 
 struct ParquetStatisticsUtils {
 
-	static duckdb::unique_ptr<BaseStatistics>
-	TransformColumnStatistics(const SchemaElement &s_ele, const LogicalType &type, const ColumnChunk &column_chunk);
+	static unique_ptr<BaseStatistics> TransformColumnStatistics(const SchemaElement &s_ele, const LogicalType &type,
+	                                                            const ColumnChunk &column_chunk);
 
 	static Value ConvertValue(const LogicalType &type, const duckdb_parquet::format::SchemaElement &schema_ele,
 	                          const std::string &stats);

package/src/duckdb/extension/parquet/include/parquet_support.hpp

@@ -37,8 +37,7 @@ public:
 		 * @param throwIfNotFound fail if a stream is required and not found
 		 * @return the new stream
 		 */
-		virtual duckdb::unique_ptr<SeekableInputStream> getStream(const StreamIdentifier &si, bool throwIfNotFound)
-		    const = 0;
+		virtual unique_ptr<SeekableInputStream> getStream(const StreamIdentifier &si, bool throwIfNotFound) const = 0;
 
 		/**
 		 * visit all streams of given node and execute visitor logic
@@ -63,7 +62,7 @@ public:
 		 * Get the RowGroupIndex.
 		 * @return a vector of RowIndex belonging to the stripe
 		 */
-		virtual duckdb::unique_ptr<proto::RowIndex> getRowGroupIndex(const StreamIdentifier &si) const = 0;
+		virtual unique_ptr<proto::RowIndex> getRowGroupIndex(const StreamIdentifier &si) const = 0;
 
 		/**
 		 * Get stride index provider which is used by string dictionary reader to
@@ -84,8 +83,7 @@ public:
 	 * @param throwIfNotFound fail if a stream is required and not found
 	 * @return the new stream
 	 */
-	virtual duckdb::unique_ptr<SeekableInputStream> getStream(const StreamIdentifier &si,
-	                                                          bool throwIfNotFound) const = 0;
+	virtual unique_ptr<SeekableInputStream> getStream(const StreamIdentifier &si, bool throwIfNotFound) const = 0;
 
 	/**
 	 * visit all streams of given node and execute visitor logic
@@ -110,7 +108,7 @@ public:
 	 * Get the RowGroupIndex.
 	 * @return a vector of RowIndex belonging to the stripe
 	 */
-	virtual duckdb::unique_ptr<proto::RowIndex> getRowGroupIndex(const StreamIdentifier &si) const = 0;
+	virtual unique_ptr<proto::RowIndex> getRowGroupIndex(const StreamIdentifier &si) const = 0;
 
 	/**
 	 * Get stride index provider which is used by string dictionary reader to
@@ -157,10 +155,10 @@ public:
 	}
 
 	// Creates a reader for the given stripe.
-	static duckdb::unique_ptr<SelectiveColumnReader>
-	build(const std::shared_ptr<const dwio::common::TypeWithId> &requestedType,
-	      const std::shared_ptr<const dwio::common::TypeWithId> &dataType, StripeStreams &stripe,
-	      common::ScanSpec *scanSpec, uint32_t sequence = 0);
+	static unique_ptr<SelectiveColumnReader> build(const std::shared_ptr<const dwio::common::TypeWithId> &requestedType,
+	                                               const std::shared_ptr<const dwio::common::TypeWithId> &dataType,
+	                                               StripeStreams &stripe, common::ScanSpec *scanSpec,
+	                                               uint32_t sequence = 0);
 
 	// Seeks to offset and reads the rows in 'rows' and applies
 	// filters and value processing as given by 'scanSpec supplied at
@@ -336,7 +334,7 @@ public:
 		return kind_;
 	}
 
-	virtual duckdb::unique_ptr<Filter> clone() const = 0;
+	virtual unique_ptr<Filter> clone() const = 0;
 
 	/**
 	 * A filter becomes non-deterministic when applies to nested column,

package/src/duckdb/extension/parquet/include/parquet_timestamp.hpp

@@ -25,5 +25,6 @@ date_t ParquetIntToDate(const int32_t &raw_date);
 dtime_t ParquetIntToTimeMs(const int32_t &raw_time);
 dtime_t ParquetIntToTime(const int64_t &raw_time);
 dtime_t ParquetIntToTimeNs(const int64_t &raw_time);
+dtime_tz_t ParquetIntToTimeTZ(const int64_t &raw_time);
 
 } // namespace duckdb

package/src/duckdb/extension/parquet/include/parquet_writer.hpp

@@ -15,10 +15,11 @@
 #include "duckdb/common/mutex.hpp"
 #include "duckdb/common/serializer/buffered_file_writer.hpp"
 #include "duckdb/common/types/column/column_data_collection.hpp"
+#include "duckdb/function/copy_function.hpp"
 #endif
 
-#include "parquet_types.h"
 #include "column_writer.hpp"
+#include "parquet_types.h"
 #include "thrift/protocol/TCompactProtocol.h"
 
 namespace duckdb {
@@ -27,13 +28,31 @@ class FileOpener;
 
 struct PreparedRowGroup {
 	duckdb_parquet::format::RowGroup row_group;
-	vector<duckdb::unique_ptr<ColumnWriterState>> states;
+	vector<unique_ptr<ColumnWriterState>> states;
+	vector<shared_ptr<StringHeap>> heaps;
+};
+
+struct FieldID;
+struct ChildFieldIDs {
+	ChildFieldIDs();
+	ChildFieldIDs Copy() const;
+	unique_ptr<case_insensitive_map_t<FieldID>> ids;
+};
+
+struct FieldID {
+	static constexpr const auto DUCKDB_FIELD_ID = "__duckdb_field_id";
+	FieldID();
+	explicit FieldID(int32_t field_id);
+	FieldID Copy() const;
+	bool set;
+	int32_t field_id;
+	ChildFieldIDs child_field_ids;
 };
 
 class ParquetWriter {
 public:
 	ParquetWriter(FileSystem &fs, string file_name, vector<LogicalType> types, vector<string> names,
-	              duckdb_parquet::format::CompressionCodec::type codec);
+	              duckdb_parquet::format::CompressionCodec::type codec, ChildFieldIDs field_ids);
 
 public:
 	void PrepareRowGroup(ColumnDataCollection &buffer, PreparedRowGroup &result);
@@ -57,18 +76,23 @@ public:
 		return *writer;
 	}
 
+	static CopyTypeSupport TypeIsSupported(const LogicalType &type);
+
 private:
+	static CopyTypeSupport DuckDBTypeToParquetTypeInternal(const LogicalType &duckdb_type,
+	                                                       duckdb_parquet::format::Type::type &type);
 	string file_name;
 	vector<LogicalType> sql_types;
 	vector<string> column_names;
 	duckdb_parquet::format::CompressionCodec::type codec;
+	ChildFieldIDs field_ids;
 
-	duckdb::unique_ptr<BufferedFileWriter> writer;
+	unique_ptr<BufferedFileWriter> writer;
 	shared_ptr<duckdb_apache::thrift::protocol::TProtocol> protocol;
 	duckdb_parquet::format::FileMetaData file_meta_data;
 	std::mutex lock;
 
-	vector<duckdb::unique_ptr<ColumnWriter>> column_writers;
+	vector<unique_ptr<ColumnWriter>> column_writers;
 };
 
 } // namespace duckdb

package/src/duckdb/extension/parquet/include/string_column_reader.hpp

@@ -28,7 +28,7 @@ public:
 	StringColumnReader(ParquetReader &reader, LogicalType type_p, const SchemaElement &schema_p, idx_t schema_idx_p,
 	                   idx_t max_define_p, idx_t max_repeat_p);
 
-	duckdb::unique_ptr<string_t[]> dict_strings;
+	unique_ptr<string_t[]> dict_strings;
 	idx_t fixed_width_string_length;
 	idx_t delta_offset = 0;
 

package/src/duckdb/extension/parquet/include/struct_column_reader.hpp

@@ -19,10 +19,9 @@ public:
 
 public:
 	StructColumnReader(ParquetReader &reader, LogicalType type_p, const SchemaElement &schema_p, idx_t schema_idx_p,
-	                   idx_t max_define_p, idx_t max_repeat_p,
-	                   vector<duckdb::unique_ptr<ColumnReader>> child_readers_p);
+	                   idx_t max_define_p, idx_t max_repeat_p, vector<unique_ptr<ColumnReader>> child_readers_p);
 
-	vector<duckdb::unique_ptr<ColumnReader>> child_readers;
+	vector<unique_ptr<ColumnReader>> child_readers;
 
 public:
 	ColumnReader *GetChildReader(idx_t child_idx);

package/src/duckdb/extension/parquet/include/zstd_file_system.hpp

@@ -17,13 +17,13 @@ namespace duckdb {
 
 class ZStdFileSystem : public CompressedFileSystem {
 public:
-	duckdb::unique_ptr<FileHandle> OpenCompressedFile(duckdb::unique_ptr<FileHandle> handle, bool write) override;
+	unique_ptr<FileHandle> OpenCompressedFile(unique_ptr<FileHandle> handle, bool write) override;
 
 	std::string GetName() const override {
 		return "ZStdFileSystem";
 	}
 
-	duckdb::unique_ptr<StreamWrapper> CreateStream() override;
+	unique_ptr<StreamWrapper> CreateStream() override;
 	idx_t InBufferSize() override;
 	idx_t OutBufferSize() override;
 };