duckdb 1.1.2-dev6.0 → 1.1.4-dev11.0

package/src/duckdb/extension/parquet/geo_parquet.cpp

@@ -176,9 +176,8 @@ void GeoParquetColumnMetadataWriter::Update(GeoParquetColumnMetadata &meta, Vect
 // GeoParquetFileMetadata
 //------------------------------------------------------------------------------
 
-unique_ptr<GeoParquetFileMetadata>
-GeoParquetFileMetadata::TryRead(const duckdb_parquet::format::FileMetaData &file_meta_data,
-                                const ClientContext &context) {
+unique_ptr<GeoParquetFileMetadata> GeoParquetFileMetadata::TryRead(const duckdb_parquet::FileMetaData &file_meta_data,
+                                                                   const ClientContext &context) {
 
 	// Conversion not enabled, or spatial is not loaded!
 	if (!IsGeoParquetConversionEnabled(context)) {
@@ -289,7 +288,7 @@ void GeoParquetFileMetadata::FlushColumnMeta(const string &column_name, const Ge
 	column.bbox.Combine(meta.bbox);
 }
 
-void GeoParquetFileMetadata::Write(duckdb_parquet::format::FileMetaData &file_meta_data) const {
+void GeoParquetFileMetadata::Write(duckdb_parquet::FileMetaData &file_meta_data) const {
 
 	yyjson_mut_doc *doc = yyjson_mut_doc_new(nullptr);
 	yyjson_mut_val *root = yyjson_mut_obj(doc);
@@ -343,7 +342,7 @@ void GeoParquetFileMetadata::Write(duckdb_parquet::format::FileMetaData &file_me
 	}
 
 	// Create a string from the JSON
-	duckdb_parquet::format::KeyValue kv;
+	duckdb_parquet::KeyValue kv;
 	kv.__set_key("geo");
 	kv.__set_value(string(json, len));
 

package/src/duckdb/extension/parquet/include/boolean_column_reader.hpp

@@ -40,10 +40,6 @@ public:
 };
 
 struct BooleanParquetValueConversion {
-	static bool DictRead(ByteBuffer &dict, uint32_t &offset, ColumnReader &reader) {
-		throw std::runtime_error("Dicts for booleans make no sense");
-	}
-
 	static bool PlainRead(ByteBuffer &plain_data, ColumnReader &reader) {
 		plain_data.available(1);
 		return UnsafePlainRead(plain_data, reader);

package/src/duckdb/extension/parquet/include/column_reader.hpp

@@ -29,12 +29,12 @@ class ParquetReader;
 
 using duckdb_apache::thrift::protocol::TProtocol;
 
-using duckdb_parquet::format::ColumnChunk;
-using duckdb_parquet::format::CompressionCodec;
-using duckdb_parquet::format::FieldRepetitionType;
-using duckdb_parquet::format::PageHeader;
-using duckdb_parquet::format::SchemaElement;
-using duckdb_parquet::format::Type;
+using duckdb_parquet::ColumnChunk;
+using duckdb_parquet::CompressionCodec;
+using duckdb_parquet::FieldRepetitionType;
+using duckdb_parquet::PageHeader;
+using duckdb_parquet::SchemaElement;
+using duckdb_parquet::Type;
 
 typedef std::bitset<STANDARD_VECTOR_SIZE> parquet_filter_t;
 
@@ -57,6 +57,9 @@ public:
 	ParquetReader &Reader();
 	const LogicalType &Type() const;
 	const SchemaElement &Schema() const;
+	optional_ptr<const SchemaElement> GetParentSchema() const;
+	void SetParentSchema(const SchemaElement &parent_schema);
+
 	idx_t FileIdx() const;
 	idx_t MaxDefine() const;
 	idx_t MaxRepeat() const;
@@ -72,7 +75,7 @@ public:
 
 	template <class VALUE_TYPE, class CONVERSION>
 	void PlainTemplated(shared_ptr<ByteBuffer> plain_data, uint8_t *defines, uint64_t num_values,
-	                    parquet_filter_t &filter, idx_t result_offset, Vector &result) {
+	                    parquet_filter_t *filter, idx_t result_offset, Vector &result) {
 		if (HasDefines()) {
 			if (CONVERSION::PlainAvailable(*plain_data, num_values)) {
 				PlainTemplatedInternal<VALUE_TYPE, CONVERSION, true, true>(*plain_data, defines, num_values, filter,
@@ -95,13 +98,13 @@ public:
 private:
 	template <class VALUE_TYPE, class CONVERSION, bool HAS_DEFINES, bool UNSAFE>
 	void PlainTemplatedInternal(ByteBuffer &plain_data, const uint8_t *__restrict defines, const uint64_t num_values,
-	                            const parquet_filter_t &filter, const idx_t result_offset, Vector &result) {
+	                            const parquet_filter_t *filter, const idx_t result_offset, Vector &result) {
 		const auto result_ptr = FlatVector::GetData<VALUE_TYPE>(result);
 		auto &result_mask = FlatVector::Validity(result);
 		for (idx_t row_idx = result_offset; row_idx < result_offset + num_values; row_idx++) {
-			if (HAS_DEFINES && defines[row_idx] != max_define) {
+			if (HAS_DEFINES && defines && defines[row_idx] != max_define) {
 				result_mask.SetInvalid(row_idx);
-			} else if (filter.test(row_idx)) {
+			} else if (!filter || filter->test(row_idx)) {
 				result_ptr[row_idx] =
 				    UNSAFE ? CONVERSION::UnsafePlainRead(plain_data, *this) : CONVERSION::PlainRead(plain_data, *this);
 			} else { // there is still some data there that we have to skip over
@@ -117,14 +120,9 @@ private:
 protected:
 	Allocator &GetAllocator();
 	// readers that use the default Read() need to implement those
-	virtual void Plain(shared_ptr<ByteBuffer> plain_data, uint8_t *defines, idx_t num_values, parquet_filter_t &filter,
+	virtual void Plain(shared_ptr<ByteBuffer> plain_data, uint8_t *defines, idx_t num_values, parquet_filter_t *filter,
 	                   idx_t result_offset, Vector &result);
-	virtual void Dictionary(shared_ptr<ResizeableBuffer> dictionary_data, idx_t num_entries);
-	virtual void Offsets(uint32_t *offsets, uint8_t *defines, idx_t num_values, parquet_filter_t &filter,
-	                     idx_t result_offset, Vector &result);
-
 	// these are nops for most types, but not for strings
-	virtual void DictReference(Vector &result);
 	virtual void PlainReference(shared_ptr<ByteBuffer>, Vector &result);
 
 	virtual void PrepareDeltaLengthByteArray(ResizeableBuffer &buffer);
@@ -145,6 +143,7 @@ protected:
 
 protected:
 	const SchemaElement &schema;
+	optional_ptr<const SchemaElement> parent_schema;
 
 	idx_t file_idx;
 	idx_t max_define;
@@ -168,10 +167,11 @@ private:
 	void PreparePageV2(PageHeader &page_hdr);
 	void DecompressInternal(CompressionCodec::type codec, const_data_ptr_t src, idx_t src_size, data_ptr_t dst,
 	                        idx_t dst_size);
+	void ConvertDictToSelVec(uint32_t *offsets, uint8_t *defines, parquet_filter_t &filter, idx_t read_now,
+	                         idx_t result_offset);
+	const ColumnChunk *chunk = nullptr;
 
-	const duckdb_parquet::format::ColumnChunk *chunk = nullptr;
-
-	duckdb_apache::thrift::protocol::TProtocol *protocol;
+	TProtocol *protocol;
 	idx_t page_rows_available;
 	idx_t group_rows_available;
 	idx_t chunk_read_offset;
@@ -193,6 +193,11 @@ private:
 	ResizeableBuffer dummy_define;
 	ResizeableBuffer dummy_repeat;
 
+	SelectionVector dictionary_selection_vector;
+	idx_t dictionary_size;
+	unique_ptr<Vector> dictionary;
+	string dictionary_id;
+
 public:
 	template <class TARGET>
 	TARGET &Cast() {

package/src/duckdb/extension/parquet/include/column_writer.hpp

@@ -17,6 +17,8 @@ class ParquetWriter;
 class ColumnWriterPageState;
 class BasicColumnWriterState;
 struct ChildFieldIDs;
+class ResizeableBuffer;
+class ParquetBloomFilter;
 
 class ColumnWriterState {
 public:
@@ -80,12 +82,12 @@ public:
 public:
 	//! Create the column writer for a specific type recursively
 	static unique_ptr<ColumnWriter>
-	CreateWriterRecursive(ClientContext &context, vector<duckdb_parquet::format::SchemaElement> &schemas,
-	                      ParquetWriter &writer, const LogicalType &type, const string &name,
-	                      vector<string> schema_path, optional_ptr<const ChildFieldIDs> field_ids, idx_t max_repeat = 0,
-	                      idx_t max_define = 1, bool can_have_nulls = true);
+	CreateWriterRecursive(ClientContext &context, vector<duckdb_parquet::SchemaElement> &schemas, ParquetWriter &writer,
+	                      const LogicalType &type, const string &name, vector<string> schema_path,
+	                      optional_ptr<const ChildFieldIDs> field_ids, idx_t max_repeat = 0, idx_t max_define = 1,
+	                      bool can_have_nulls = true);
 
-	virtual unique_ptr<ColumnWriterState> InitializeWriteState(duckdb_parquet::format::RowGroup &row_group) = 0;
+	virtual unique_ptr<ColumnWriterState> InitializeWriteState(duckdb_parquet::RowGroup &row_group) = 0;
 
 	//! indicates whether the write need to analyse the data before preparing it
 	virtual bool HasAnalyze() {

package/src/duckdb/extension/parquet/include/decode_utils.hpp

@@ -1,40 +1,160 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// decode_utils.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
 #pragma once
 
+#include "duckdb/common/bitpacking.hpp"
 #include "resizable_buffer.hpp"
 
 namespace duckdb {
-class ParquetDecodeUtils {
-
-public:
-	template <class T>
-	static T ZigzagToInt(const uint64_t n) {
-		return T(n >> 1) ^ -T(n & 1);
-	}
 
+class ParquetDecodeUtils {
+	//===--------------------------------------------------------------------===//
+	// Bitpacking
+	//===--------------------------------------------------------------------===//
+private:
 	static const uint64_t BITPACK_MASKS[];
 	static const uint64_t BITPACK_MASKS_SIZE;
 	static const uint8_t BITPACK_DLEN;
 
-	template <typename T>
-	static uint32_t BitUnpack(ByteBuffer &buffer, uint8_t &bitpack_pos, T *dest, uint32_t count, uint8_t width) {
-		if (width >= ParquetDecodeUtils::BITPACK_MASKS_SIZE) {
+	static void CheckWidth(const uint8_t width) {
+		if (width >= BITPACK_MASKS_SIZE) {
 			throw InvalidInputException("The width (%d) of the bitpacked data exceeds the supported max width (%d), "
 			                            "the file might be corrupted.",
-			                            width, ParquetDecodeUtils::BITPACK_MASKS_SIZE);
+			                            width, BITPACK_MASKS_SIZE);
 		}
-		auto mask = BITPACK_MASKS[width];
+	}
 
-		for (uint32_t i = 0; i < count; i++) {
-			T val = (buffer.get<uint8_t>() >> bitpack_pos) & mask;
+public:
+	template <class T>
+	static void BitUnpack(ByteBuffer &src, bitpacking_width_t &bitpack_pos, T *dst, idx_t count,
+	                      const bitpacking_width_t width) {
+		CheckWidth(width);
+		const auto mask = BITPACK_MASKS[width];
+		src.available(count * width / BITPACK_DLEN); // check if buffer has enough space available once
+		if (bitpack_pos == 0 && count >= BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE) {
+			idx_t remainder = count % BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE;
+			idx_t aligned_count = count - remainder;
+			BitUnpackAlignedInternal(src, dst, aligned_count, width);
+			dst += aligned_count;
+			count = remainder;
+		}
+		for (idx_t i = 0; i < count; i++) {
+			auto val = (src.unsafe_get<uint8_t>() >> bitpack_pos) & mask;
 			bitpack_pos += width;
 			while (bitpack_pos > BITPACK_DLEN) {
-				buffer.inc(1);
-				val |= (T(buffer.get<uint8_t>()) << T(BITPACK_DLEN - (bitpack_pos - width))) & mask;
+				src.unsafe_inc(1);
+				val |= (static_cast<T>(src.unsafe_get<uint8_t>())
+				        << static_cast<T>(BITPACK_DLEN - (bitpack_pos - width))) &
+				       mask;
 				bitpack_pos -= BITPACK_DLEN;
 			}
-			dest[i] = val;
+			dst[i] = val;
+		}
+	}
+
+	template <class T>
+	static void BitPackAligned(T *src, data_ptr_t dst, const idx_t count, const bitpacking_width_t width) {
+		D_ASSERT(width < BITPACK_MASKS_SIZE);
+		D_ASSERT(count % BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE == 0);
+		BitpackingPrimitives::PackBuffer<T, true>(dst, src, count, width);
+	}
+
+	template <class T>
+	static void BitUnpackAlignedInternal(ByteBuffer &src, T *dst, const idx_t count, const bitpacking_width_t width) {
+		for (idx_t i = 0; i < count; i += BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE) {
+			const auto next_read = BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE * width / 8;
+
+			// Buffer for alignment
+			T aligned_data[BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE];
+
+			// Copy over to aligned buffer
+			memcpy(aligned_data, src.ptr, next_read);
+
+			// Unpack
+			BitpackingPrimitives::UnPackBlock<T>(data_ptr_cast(dst), data_ptr_cast(aligned_data), width, true);
+
+			src.unsafe_inc(next_read);
+			dst += BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE;
 		}
-		return count;
+	}
+
+	template <class T>
+	static void BitUnpackAligned(ByteBuffer &src, T *dst, const idx_t count, const bitpacking_width_t width) {
+		CheckWidth(width);
+		if (count % BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE != 0) {
+			throw InvalidInputException("Aligned bitpacking count must be a multiple of %llu",
+			                            BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE);
+		}
+		const auto read_size = count * width / BITPACK_DLEN;
+		src.available(read_size); // check if buffer has enough space available once
+		BitUnpackAlignedInternal(src, dst, count, width);
+	}
+
+	//===--------------------------------------------------------------------===//
+	// Zigzag
+	//===--------------------------------------------------------------------===//
+private:
+	//! https://lemire.me/blog/2022/11/25/making-all-your-integers-positive-with-zigzag-encoding/
+	template <class UNSIGNED>
+	static typename std::enable_if<std::is_unsigned<UNSIGNED>::value, typename std::make_signed<UNSIGNED>::type>::type
+	ZigzagToIntInternal(UNSIGNED x) {
+		return (x >> 1) ^ (-(x & 1));
+	}
+
+	template <typename SIGNED>
+	static typename std::enable_if<std::is_signed<SIGNED>::value, typename std::make_unsigned<SIGNED>::type>::type
+	IntToZigzagInternal(SIGNED x) {
+		using UNSIGNED = typename std::make_unsigned<SIGNED>::type;
+		return (static_cast<UNSIGNED>(x) << 1) ^ static_cast<UNSIGNED>(x >> (sizeof(SIGNED) * 8 - 1));
+	}
+
+public:
+	template <class UNSIGNED>
+	static typename std::enable_if<std::is_unsigned<UNSIGNED>::value, typename std::make_signed<UNSIGNED>::type>::type
+	ZigzagToInt(UNSIGNED x) {
+		auto integer = ZigzagToIntInternal(x);
+		D_ASSERT(x == IntToZigzagInternal(integer)); // test roundtrip
+		return integer;
+	}
+
+	template <typename SIGNED>
+	static typename std::enable_if<std::is_signed<SIGNED>::value, typename std::make_unsigned<SIGNED>::type>::type
+	IntToZigzag(SIGNED x) {
+		auto zigzag = IntToZigzagInternal(x);
+		D_ASSERT(x == ZigzagToIntInternal(zigzag)); // test roundtrip
+		return zigzag;
+	}
+
+	//===--------------------------------------------------------------------===//
+	// Varint
+	//===--------------------------------------------------------------------===//
+public:
+	template <class T>
+	static uint8_t GetVarintSize(T val) {
+		uint8_t res = 0;
+		do {
+			val >>= 7;
+			res++;
+		} while (val != 0);
+		return res;
+	}
+
+	template <class T>
+	static void VarintEncode(T val, WriteStream &ser) {
+		do {
+			uint8_t byte = val & 127;
+			val >>= 7;
+			if (val != 0) {
+				byte |= 128;
+			}
+			ser.Write<uint8_t>(byte);
+		} while (val != 0);
 	}
 
 	template <class T>

package/src/duckdb/extension/parquet/include/geo_parquet.hpp

@@ -119,15 +119,16 @@ class GeoParquetFileMetadata {
 public:
 	// Try to read GeoParquet metadata. Returns nullptr if not found, invalid or the required spatial extension is not
 	// available.
-	static unique_ptr<GeoParquetFileMetadata> TryRead(const duckdb_parquet::format::FileMetaData &file_meta_data,
+
+	static unique_ptr<GeoParquetFileMetadata> TryRead(const duckdb_parquet::FileMetaData &file_meta_data,
 	                                                  const ClientContext &context);
-	void Write(duckdb_parquet::format::FileMetaData &file_meta_data) const;
+	void Write(duckdb_parquet::FileMetaData &file_meta_data) const;
 
 	void FlushColumnMeta(const string &column_name, const GeoParquetColumnMetadata &meta);
 	const unordered_map<string, GeoParquetColumnMetadata> &GetColumnMeta() const;
 
 	unique_ptr<ColumnReader> CreateColumnReader(ParquetReader &reader, const LogicalType &logical_type,
-	                                            const duckdb_parquet::format::SchemaElement &s_ele, idx_t schema_idx_p,
+	                                            const duckdb_parquet::SchemaElement &s_ele, idx_t schema_idx_p,
 	                                            idx_t max_define_p, idx_t max_repeat_p, ClientContext &context);
 
 	bool IsGeometryColumn(const string &column_name) const;

package/src/duckdb/extension/parquet/include/null_column_reader.hpp

@@ -25,20 +25,7 @@ public:
 	shared_ptr<ResizeableBuffer> dict;
 
 public:
-	void Dictionary(shared_ptr<ResizeableBuffer> data, idx_t num_entries) override {
-		dict = std::move(data);
-	}
-
-	void Offsets(uint32_t *offsets, uint8_t *defines, uint64_t num_values, parquet_filter_t &filter,
-	             idx_t result_offset, Vector &result) override {
-		auto &result_mask = FlatVector::Validity(result);
-
-		for (idx_t row_idx = 0; row_idx < num_values; row_idx++) {
-			result_mask.SetInvalid(row_idx + result_offset);
-		}
-	}
-
-	void Plain(shared_ptr<ByteBuffer> plain_data, uint8_t *defines, uint64_t num_values, parquet_filter_t &filter,
+	void Plain(shared_ptr<ByteBuffer> plain_data, uint8_t *defines, uint64_t num_values, parquet_filter_t *filter,
 	           idx_t result_offset, Vector &result) override {
 		(void)defines;
 		(void)plain_data;

package/src/duckdb/extension/parquet/include/parquet_bss_encoder.hpp

@@ -0,0 +1,45 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// parquet_bss_encoder.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "decode_utils.hpp"
+
+namespace duckdb {
+
+class BssEncoder {
+public:
+	explicit BssEncoder(const idx_t total_value_count_p, const idx_t bit_width_p)
+	    : total_value_count(total_value_count_p), bit_width(bit_width_p), count(0),
+	      buffer(Allocator::DefaultAllocator().Allocate(total_value_count * bit_width + 1)) {
+	}
+
+public:
+	template <class T>
+	void WriteValue(const T &value) {
+		D_ASSERT(sizeof(T) == bit_width);
+		for (idx_t i = 0; i < sizeof(T); i++) {
+			buffer.get()[i * total_value_count + count] = reinterpret_cast<const_data_ptr_t>(&value)[i];
+		}
+		count++;
+	}
+
+	void FinishWrite(WriteStream &writer) {
+		D_ASSERT(count == total_value_count);
+		writer.WriteData(buffer.get(), total_value_count * bit_width);
+	}
+
+private:
+	const idx_t total_value_count;
+	const idx_t bit_width;
+
+	idx_t count;
+	AllocatedData buffer;
+};
+
+} // namespace duckdb

package/src/duckdb/extension/parquet/include/parquet_crypto.hpp

@@ -4,7 +4,7 @@
 // parquet_crypto.hpp
 //
 //
-//===----------------------------------------------------------------------===/
+//===----------------------------------------------------------------------===//
 
 #pragma once