duckdb 0.6.2-dev32.0 → 0.6.2-dev416.0

package/src/parquet-amalgamation.hpp

@@ -7193,7 +7193,7 @@ public:
 			return;
 		}
 		if (new_size > alloc_len) {
-			alloc_len = new_size;
+			alloc_len = NextPowerOfTwo(new_size);
 			allocated_data = allocator.Allocate(alloc_len);
 			ptr = (char *)allocated_data.get();
 		}
@@ -7387,7 +7387,6 @@ namespace duckdb {
 class DbpDecoder {
 public:
 	DbpDecoder(const uint8_t *buffer, uint32_t buffer_len) : buffer_((char *)buffer, buffer_len) {
-
 		//<block size in values> <number of miniblocks in a block> <total value count> <first value>
 		// overall header
 		block_value_count = ParquetDecodeUtils::VarintDecode<uint64_t>(buffer_);
@@ -7396,6 +7395,7 @@ public:
 		start_value = ParquetDecodeUtils::ZigzagToInt(ParquetDecodeUtils::VarintDecode<int64_t>(buffer_));
 
 		// some derivatives
+		D_ASSERT(miniblocks_per_block > 0);
 		values_per_miniblock = block_value_count / miniblocks_per_block;
 		miniblock_bit_widths = std::unique_ptr<uint8_t[]>(new data_t[miniblocks_per_block]);
 
@@ -7474,6 +7474,17 @@ public:
 		}
 		start_value = values[batch_size - 1];
 	}
+	void Finalize() {
+		if (values_left_in_miniblock == 0) {
+			return;
+		}
+		auto data = unique_ptr<uint32_t[]>(new uint32_t[values_left_in_miniblock]);
+		GetBatch<uint32_t>((char *)data.get(), values_left_in_miniblock);
+	}
+
+	uint64_t TotalValues() {
+		return total_value_count;
+	}
 
 private:
 	ByteBuffer buffer_;
@@ -7579,11 +7590,31 @@ public:
 
 	virtual unique_ptr<BaseStatistics> Stats(idx_t row_group_idx_p, const std::vector<ColumnChunk> &columns);
 
+	template <class VALUE_TYPE, class CONVERSION>
+	void PlainTemplated(shared_ptr<ByteBuffer> plain_data, uint8_t *defines, uint64_t num_values,
+	                    parquet_filter_t &filter, idx_t result_offset, Vector &result) {
+		auto result_ptr = FlatVector::GetData<VALUE_TYPE>(result);
+		auto &result_mask = FlatVector::Validity(result);
+		for (idx_t row_idx = 0; row_idx < num_values; row_idx++) {
+			if (HasDefines() && defines[row_idx + result_offset] != max_define) {
+				result_mask.SetInvalid(row_idx + result_offset);
+				continue;
+			}
+			if (filter[row_idx + result_offset]) {
+				VALUE_TYPE val = CONVERSION::PlainRead(*plain_data, *this);
+				result_ptr[row_idx + result_offset] = val;
+			} else { // there is still some data there that we have to skip over
+				CONVERSION::PlainSkip(*plain_data, *this);
+			}
+		}
+	}
+
 protected:
+	Allocator &GetAllocator();
 	// readers that use the default Read() need to implement those
 	virtual void Plain(shared_ptr<ByteBuffer> plain_data, uint8_t *defines, idx_t num_values, parquet_filter_t &filter,
 	                   idx_t result_offset, Vector &result);
-	virtual void Dictionary(shared_ptr<ByteBuffer> dictionary_data, idx_t num_entries);
+	virtual void Dictionary(shared_ptr<ResizeableBuffer> dictionary_data, idx_t num_entries);
 	virtual void Offsets(uint32_t *offsets, uint8_t *defines, idx_t num_values, parquet_filter_t &filter,
 	                     idx_t result_offset, Vector &result);
 
@@ -7591,6 +7622,11 @@ protected:
 	virtual void DictReference(Vector &result);
 	virtual void PlainReference(shared_ptr<ByteBuffer>, Vector &result);
 
+	virtual void PrepareDeltaLengthByteArray(ResizeableBuffer &buffer);
+	virtual void PrepareDeltaByteArray(ResizeableBuffer &buffer);
+	virtual void DeltaByteArray(uint8_t *defines, idx_t num_values, parquet_filter_t &filter, idx_t result_offset,
+	                            Vector &result);
+
 	// applies any skips that were registered using Skip()
 	virtual void ApplyPendingSkips(idx_t num_values);
 
@@ -7611,10 +7647,13 @@ protected:
 
 	ParquetReader &reader;
 	LogicalType type;
+	unique_ptr<Vector> byte_array_data;
 
 	idx_t pending_skips = 0;
 
 private:
+	void AllocateBlock(idx_t size);
+	void AllocateCompressed(idx_t size);
 	void PrepareRead(parquet_filter_t &filter);
 	void PreparePage(PageHeader &page_hdr);
 	void PrepareDataPage(PageHeader &page_hdr);
@@ -7630,12 +7669,14 @@ private:
 
 	shared_ptr<ResizeableBuffer> block;
 
+	ResizeableBuffer compressed_buffer;
 	ResizeableBuffer offset_buffer;
 
 	unique_ptr<RleBpDecoder> dict_decoder;
 	unique_ptr<RleBpDecoder> defined_decoder;
 	unique_ptr<RleBpDecoder> repeated_decoder;
 	unique_ptr<DbpDecoder> dbp_decoder;
+	unique_ptr<RleBpDecoder> rle_decoder;
 
 	// dummies for Skip()
 	parquet_filter_t none_filter;
@@ -7908,7 +7949,8 @@ public:
 	                                                      idx_t max_repeat = 0, idx_t max_define = 1,
 	                                                      bool can_have_nulls = true);
 
-	virtual unique_ptr<ColumnWriterState> InitializeWriteState(duckdb_parquet::format::RowGroup &row_group) = 0;
+	virtual unique_ptr<ColumnWriterState> InitializeWriteState(duckdb_parquet::format::RowGroup &row_group,
+	                                                           Allocator &allocator) = 0;
 
 	//! indicates whether the write need to analyse the data before preparing it
 	virtual bool HasAnalyze() {