duckdb 1.3.4 → 1.4.0

package/src/duckdb/extension/parquet/include/geo_parquet.hpp

@@ -16,58 +16,169 @@
 #include "parquet_types.h"
 
 namespace duckdb {
+
 struct ParquetColumnSchema;
 
-enum class WKBGeometryType : uint16_t {
-	POINT = 1,
-	LINESTRING = 2,
-	POLYGON = 3,
-	MULTIPOINT = 4,
-	MULTILINESTRING = 5,
-	MULTIPOLYGON = 6,
-	GEOMETRYCOLLECTION = 7,
-
-	POINT_Z = 1001,
-	LINESTRING_Z = 1002,
-	POLYGON_Z = 1003,
-	MULTIPOINT_Z = 1004,
-	MULTILINESTRING_Z = 1005,
-	MULTIPOLYGON_Z = 1006,
-	GEOMETRYCOLLECTION_Z = 1007,
-};
+struct GeometryKindSet {
+
+	uint8_t bits[4] = {0, 0, 0, 0};
+
+	void Add(uint32_t wkb_type) {
+		auto kind = wkb_type % 1000;
+		auto dims = wkb_type / 1000;
+		if (kind < 1 || kind > 7 || (dims) > 3) {
+			return;
+		}
+		bits[dims] |= (1 << (kind - 1));
+	}
+
+	void Combine(const GeometryKindSet &other) {
+		for (uint32_t d = 0; d < 4; d++) {
+			bits[d] |= other.bits[d];
+		}
+	}
+
+	bool IsEmpty() const {
+		for (uint32_t d = 0; d < 4; d++) {
+			if (bits[d] != 0) {
+				return false;
+			}
+		}
+		return true;
+	}
+
+	template <class T>
+	vector<T> ToList() const {
+		vector<T> result;
+		for (uint32_t d = 0; d < 4; d++) {
+			for (uint32_t i = 1; i <= 7; i++) {
+				if (bits[d] & (1 << (i - 1))) {
+					result.push_back(i + d * 1000);
+				}
+			}
+		}
+		return result;
+	}
 
-struct WKBGeometryTypes {
-	static const char *ToString(WKBGeometryType type);
+	vector<string> ToString(bool snake_case) const {
+		vector<string> result;
+		for (uint32_t d = 0; d < 4; d++) {
+			for (uint32_t i = 1; i <= 7; i++) {
+				if (bits[d] & (1 << (i - 1))) {
+					string str;
+					switch (i) {
+					case 1:
+						str = snake_case ? "point" : "Point";
+						break;
+					case 2:
+						str = snake_case ? "linestring" : "LineString";
+						break;
+					case 3:
+						str = snake_case ? "polygon" : "Polygon";
+						break;
+					case 4:
+						str = snake_case ? "multipoint" : "MultiPoint";
+						break;
+					case 5:
+						str = snake_case ? "multilinestring" : "MultiLineString";
+						break;
+					case 6:
+						str = snake_case ? "multipolygon" : "MultiPolygon";
+						break;
+					case 7:
+						str = snake_case ? "geometrycollection" : "GeometryCollection";
+						break;
+					default:
+						str = snake_case ? "unknown" : "Unknown";
+						break;
+					}
+					switch (d) {
+					case 1:
+						str += snake_case ? "_z" : " Z";
+						break;
+					case 2:
+						str += snake_case ? "_m" : " M";
+						break;
+					case 3:
+						str += snake_case ? "_zm" : " ZM";
+						break;
+					default:
+						break;
+					}
+
+					result.push_back(str);
+				}
+			}
+		}
+		return result;
+	}
 };
 
-struct GeometryBounds {
-	double min_x = NumericLimits<double>::Maximum();
-	double max_x = NumericLimits<double>::Minimum();
-	double min_y = NumericLimits<double>::Maximum();
-	double max_y = NumericLimits<double>::Minimum();
+struct GeometryExtent {
+
+	double xmin = NumericLimits<double>::Maximum();
+	double xmax = NumericLimits<double>::Minimum();
+	double ymin = NumericLimits<double>::Maximum();
+	double ymax = NumericLimits<double>::Minimum();
+	double zmin = NumericLimits<double>::Maximum();
+	double zmax = NumericLimits<double>::Minimum();
+	double mmin = NumericLimits<double>::Maximum();
+	double mmax = NumericLimits<double>::Minimum();
+
+	bool IsSet() const {
+		return xmin != NumericLimits<double>::Maximum() && xmax != NumericLimits<double>::Minimum() &&
+		       ymin != NumericLimits<double>::Maximum() && ymax != NumericLimits<double>::Minimum();
+	}
+
+	bool HasZ() const {
+		return zmin != NumericLimits<double>::Maximum() && zmax != NumericLimits<double>::Minimum();
+	}
 
-	GeometryBounds() = default;
+	bool HasM() const {
+		return mmin != NumericLimits<double>::Maximum() && mmax != NumericLimits<double>::Minimum();
+	}
 
-	void Combine(const GeometryBounds &other) {
-		min_x = std::min(min_x, other.min_x);
-		max_x = std::max(max_x, other.max_x);
-		min_y = std::min(min_y, other.min_y);
-		max_y = std::max(max_y, other.max_y);
+	void Combine(const GeometryExtent &other) {
+		xmin = std::min(xmin, other.xmin);
+		xmax = std::max(xmax, other.xmax);
+		ymin = std::min(ymin, other.ymin);
+		ymax = std::max(ymax, other.ymax);
+		zmin = std::min(zmin, other.zmin);
+		zmax = std::max(zmax, other.zmax);
+		mmin = std::min(mmin, other.mmin);
+		mmax = std::max(mmax, other.mmax);
 	}
 
-	void Combine(const double &x, const double &y) {
-		min_x = std::min(min_x, x);
-		max_x = std::max(max_x, x);
-		min_y = std::min(min_y, y);
-		max_y = std::max(max_y, y);
+	void Combine(const double &xmin_p, const double &xmax_p, const double &ymin_p, const double &ymax_p) {
+		xmin = std::min(xmin, xmin_p);
+		xmax = std::max(xmax, xmax_p);
+		ymin = std::min(ymin, ymin_p);
+		ymax = std::max(ymax, ymax_p);
 	}
 
-	void Combine(const double &min_x, const double &max_x, const double &min_y, const double &max_y) {
-		this->min_x = std::min(this->min_x, min_x);
-		this->max_x = std::max(this->max_x, max_x);
-		this->min_y = std::min(this->min_y, min_y);
-		this->max_y = std::max(this->max_y, max_y);
+	void ExtendX(const double &x) {
+		xmin = std::min(xmin, x);
+		xmax = std::max(xmax, x);
+	}
+	void ExtendY(const double &y) {
+		ymin = std::min(ymin, y);
+		ymax = std::max(ymax, y);
+	}
+	void ExtendZ(const double &z) {
+		zmin = std::min(zmin, z);
+		zmax = std::max(zmax, z);
 	}
+	void ExtendM(const double &m) {
+		mmin = std::min(mmin, m);
+		mmax = std::max(mmax, m);
+	}
+};
+
+struct GeometryStats {
+	GeometryKindSet types;
+	GeometryExtent bbox;
+
+	void Update(const string_t &wkb);
 };
 
 //------------------------------------------------------------------------------
@@ -92,47 +203,31 @@ struct GeoParquetColumnMetadata {
 	// The encoding of the geometry column
 	GeoParquetColumnEncoding geometry_encoding;
 
-	// The geometry types that are present in the column
-	set<WKBGeometryType> geometry_types;
-
-	// The bounds of the geometry column
-	GeometryBounds bbox;
+	// The statistics of the geometry column
+	GeometryStats stats;
 
 	// The crs of the geometry column (if any) in PROJJSON format
 	string projjson;
-};
-
-class GeoParquetColumnMetadataWriter {
-	unique_ptr<ExpressionExecutor> executor;
-	DataChunk input_chunk;
-	DataChunk result_chunk;
-
-	unique_ptr<Expression> type_expr;
-	unique_ptr<Expression> flag_expr;
-	unique_ptr<Expression> bbox_expr;
 
-public:
-	explicit GeoParquetColumnMetadataWriter(ClientContext &context);
-	void Update(GeoParquetColumnMetadata &meta, Vector &vector, idx_t count);
+	// Used to track the "primary" geometry column (if any)
+	idx_t insertion_index = 0;
 };
 
 class GeoParquetFileMetadata {
 public:
+	void AddGeoParquetStats(const string &column_name, const LogicalType &type, const GeometryStats &stats);
+	void Write(duckdb_parquet::FileMetaData &file_meta_data);
+
 	// Try to read GeoParquet metadata. Returns nullptr if not found, invalid or the required spatial extension is not
 	// available.
-
 	static unique_ptr<GeoParquetFileMetadata> TryRead(const duckdb_parquet::FileMetaData &file_meta_data,
 	                                                  const ClientContext &context);
-	void Write(duckdb_parquet::FileMetaData &file_meta_data) const;
-
-	void FlushColumnMeta(const string &column_name, const GeoParquetColumnMetadata &meta);
 	const unordered_map<string, GeoParquetColumnMetadata> &GetColumnMeta() const;
 
-	unique_ptr<ColumnReader> CreateColumnReader(ParquetReader &reader, const ParquetColumnSchema &schema,
-	                                            ClientContext &context);
+	static unique_ptr<ColumnReader> CreateColumnReader(ParquetReader &reader, const ParquetColumnSchema &schema,
+	                                                   ClientContext &context);
 
 	bool IsGeometryColumn(const string &column_name) const;
-	void RegisterGeometryColumn(const string &column_name);
 
 	static bool IsGeoParquetConversionEnabled(const ClientContext &context);
 	static LogicalType GeometryType();
@@ -140,7 +235,6 @@ public:
 private:
 	mutex write_lock;
 	string version = "1.1.0";
-	string primary_geometry_column;
 	unordered_map<string, GeoParquetColumnMetadata> geometry_columns;
 };
 

package/src/duckdb/extension/parquet/include/parquet_column_schema.hpp

@@ -11,9 +11,11 @@
 #include "parquet_types.h"
 
 namespace duckdb {
-class ParquetReader;
 
-enum class ParquetColumnSchemaType { COLUMN, FILE_ROW_NUMBER, GEOMETRY, EXPRESSION };
+using duckdb_parquet::FileMetaData;
+struct ParquetOptions;
+
+enum class ParquetColumnSchemaType { COLUMN, FILE_ROW_NUMBER, GEOMETRY, EXPRESSION, VARIANT };
 
 enum class ParquetExtraTypeInfo {
 	NONE,
@@ -49,8 +51,8 @@ struct ParquetColumnSchema {
 	ParquetExtraTypeInfo type_info = ParquetExtraTypeInfo::NONE;
 	vector<ParquetColumnSchema> children;
 
-	unique_ptr<BaseStatistics> Stats(ParquetReader &reader, idx_t row_group_idx_p,
-	                                 const vector<duckdb_parquet::ColumnChunk> &columns) const;
+	unique_ptr<BaseStatistics> Stats(const FileMetaData &file_meta_data, const ParquetOptions &parquet_options,
+	                                 idx_t row_group_idx_p, const vector<duckdb_parquet::ColumnChunk> &columns) const;
 };
 
 } // namespace duckdb

package/src/duckdb/extension/parquet/include/parquet_crypto.hpp

@@ -10,10 +10,7 @@
 
 #include "parquet_types.h"
 #include "duckdb/common/encryption_state.hpp"
-
-#ifndef DUCKDB_AMALGAMATION
 #include "duckdb/storage/object_cache.hpp"
-#endif
 
 namespace duckdb {
 

package/src/duckdb/extension/parquet/include/parquet_extension.hpp

@@ -14,7 +14,7 @@ namespace duckdb {
 
 class ParquetExtension : public Extension {
 public:
-	void Load(DuckDB &db) override;
+	void Load(ExtensionLoader &loader) override;
 	std::string Name() override;
 	std::string Version() const override;
 };

package/src/duckdb/extension/parquet/include/parquet_file_metadata_cache.hpp

@@ -20,7 +20,7 @@ enum class ParquetCacheValidity { VALID, INVALID, UNKNOWN };
 class ParquetFileMetadataCache : public ObjectCacheEntry {
 public:
 	ParquetFileMetadataCache(unique_ptr<duckdb_parquet::FileMetaData> file_metadata, CachingFileHandle &handle,
-	                         unique_ptr<GeoParquetFileMetadata> geo_metadata);
+	                         unique_ptr<GeoParquetFileMetadata> geo_metadata, idx_t footer_size);
 	~ParquetFileMetadataCache() override = default;
 
 	//! Parquet file metadata
@@ -29,6 +29,9 @@ public:
 	//! GeoParquet metadata
 	unique_ptr<GeoParquetFileMetadata> geo_metadata;
 
+	//! Parquet footer size
+	idx_t footer_size;
+
 public:
 	static string ObjectType();
 	string GetObjectType() override;
@@ -40,7 +43,7 @@ public:
 
 private:
 	bool validate;
-	time_t last_modified;
+	timestamp_t last_modified;
 	string version_tag;
 };
 

package/src/duckdb/extension/parquet/include/parquet_multi_file_info.hpp

@@ -24,8 +24,7 @@ public:
 };
 
 struct ParquetMultiFileInfo : MultiFileReaderInterface {
-	static unique_ptr<MultiFileReaderInterface> InitializeInterface(ClientContext &context, MultiFileReader &reader,
-	                                                                MultiFileList &file_list);
+	static unique_ptr<MultiFileReaderInterface> CreateInterface(ClientContext &context);
 
 	unique_ptr<BaseFileReaderOptions> InitializeOptions(ClientContext &context,
 	                                                    optional_ptr<TableFunctionInfo> info) override;
@@ -56,6 +55,7 @@ struct ParquetMultiFileInfo : MultiFileReaderInterface {
 	unique_ptr<NodeStatistics> GetCardinality(const MultiFileBindData &bind_data, idx_t file_count) override;
 	void GetVirtualColumns(ClientContext &context, MultiFileBindData &bind_data, virtual_column_map_t &result) override;
 	unique_ptr<MultiFileReaderInterface> Copy() override;
+	FileGlobInput GetGlobInput() override;
 };
 
 class ParquetScanFunction {

package/src/duckdb/extension/parquet/include/parquet_reader.hpp

@@ -77,6 +77,9 @@ struct ParquetReaderScanState {
 	unique_ptr<AdaptiveFilter> adaptive_filter;
 	//! Table filter list
 	vector<ParquetScanFilter> scan_filters;
+
+	//! (optional) pointer to the PhysicalOperator for logging
+	optional_ptr<const PhysicalOperator> op;
 };
 
 struct ParquetColumnDefinition {
@@ -102,6 +105,7 @@ struct ParquetOptions {
 	explicit ParquetOptions(ClientContext &context);
 
 	bool binary_as_string = false;
+	bool variant_legacy_encoding = false;
 	bool file_row_number = false;
 	shared_ptr<ParquetEncryptionConfig> encryption_config;
 	bool debug_use_openssl = true;
@@ -130,8 +134,11 @@ struct ParquetUnionData : public BaseUnionData {
 	}
 	~ParquetUnionData() override;
 
+	unique_ptr<BaseStatistics> GetStatistics(ClientContext &context, const string &name) override;
+
 	ParquetOptions options;
 	shared_ptr<ParquetFileMetadataCache> metadata;
+	unique_ptr<ParquetColumnSchema> root_schema;
 };
 
 class ParquetReader : public BaseFileReader {
@@ -185,6 +192,7 @@ public:
 
 	static unique_ptr<BaseStatistics> ReadStatistics(ClientContext &context, ParquetOptions parquet_options,
 	                                                 shared_ptr<ParquetFileMetadataCache> metadata, const string &name);
+	static unique_ptr<BaseStatistics> ReadStatistics(const ParquetUnionData &union_data, const string &name);
 
 	LogicalType DeriveLogicalType(const SchemaElement &s_ele, ParquetColumnSchema &schema) const;
 
@@ -203,9 +211,9 @@ private:
 	void InitializeSchema(ClientContext &context);
 	bool ScanInternal(ClientContext &context, ParquetReaderScanState &state, DataChunk &output);
 	//! Parse the schema of the file
-	unique_ptr<ParquetColumnSchema> ParseSchema();
+	unique_ptr<ParquetColumnSchema> ParseSchema(ClientContext &context);
 	ParquetColumnSchema ParseSchemaRecursive(idx_t depth, idx_t max_define, idx_t max_repeat, idx_t &next_schema_idx,
-	                                         idx_t &next_file_idx);
+	                                         idx_t &next_file_idx, ClientContext &context);
 
 	unique_ptr<ColumnReader> CreateReader(ClientContext &context);
 

package/src/duckdb/extension/parquet/include/parquet_rle_bp_encoder.hpp

@@ -142,6 +142,9 @@ private:
 	}
 
 	void WriteCurrentBlockBP(WriteStream &writer) {
+		if (bp_block_count == 0) {
+			return;
+		}
 		ParquetDecodeUtils::VarintEncode(BP_BLOCK_SIZE / 8 << 1 | 1, writer); // (... | 1) signals BP run
 		ParquetDecodeUtils::BitPackAligned(bp_block, data_ptr_cast(bp_block_packed), BP_BLOCK_SIZE, bit_width);
 		writer.WriteData(data_ptr_cast(bp_block_packed), BP_BLOCK_SIZE * bit_width / 8);

package/src/duckdb/extension/parquet/include/parquet_statistics.hpp

@@ -9,9 +9,7 @@
 #pragma once
 
 #include "duckdb.hpp"
-#ifndef DUCKDB_AMALGAMATION
 #include "duckdb/storage/statistics/base_statistics.hpp"
-#endif
 #include "parquet_types.h"
 #include "resizable_buffer.hpp"
 

package/src/duckdb/extension/parquet/include/parquet_timestamp.hpp

@@ -29,12 +29,16 @@ timestamp_ns_t ParquetTimestampUsToTimestampNs(const int64_t &raw_us);
 timestamp_ns_t ParquetTimestampNsToTimestampNs(const int64_t &raw_ns);
 
 date_t ParquetIntToDate(const int32_t &raw_date);
-dtime_t ParquetIntToTimeMs(const int32_t &raw_time);
-dtime_t ParquetIntToTime(const int64_t &raw_time);
-dtime_t ParquetIntToTimeNs(const int64_t &raw_time);
+dtime_t ParquetMsIntToTime(const int32_t &raw_millis);
+dtime_t ParquetIntToTime(const int64_t &raw_micros);
+dtime_t ParquetNsIntToTime(const int64_t &raw_nanos);
 
-dtime_tz_t ParquetIntToTimeMsTZ(const int32_t &raw_time);
-dtime_tz_t ParquetIntToTimeTZ(const int64_t &raw_time);
-dtime_tz_t ParquetIntToTimeNsTZ(const int64_t &raw_time);
+dtime_ns_t ParquetMsIntToTimeNs(const int32_t &raw_millis);
+dtime_ns_t ParquetUsIntToTimeNs(const int64_t &raw_micros);
+dtime_ns_t ParquetIntToTimeNs(const int64_t &raw_nanos);
+
+dtime_tz_t ParquetIntToTimeMsTZ(const int32_t &raw_millis);
+dtime_tz_t ParquetIntToTimeTZ(const int64_t &raw_micros);
+dtime_tz_t ParquetIntToTimeNsTZ(const int64_t &raw_nanos);
 
 } // namespace duckdb

package/src/duckdb/extension/parquet/include/parquet_writer.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb.hpp"
 #include "duckdb/common/common.hpp"
+#include "duckdb/common/optional_idx.hpp"
 #include "duckdb/common/encryption_state.hpp"
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/mutex.hpp"
@@ -81,7 +82,7 @@ public:
 	ParquetWriter(ClientContext &context, FileSystem &fs, string file_name, vector<LogicalType> types,
 	              vector<string> names, duckdb_parquet::CompressionCodec::type codec, ChildFieldIDs field_ids,
 	              const vector<pair<string, string>> &kv_metadata,
-	              shared_ptr<ParquetEncryptionConfig> encryption_config, idx_t dictionary_size_limit,
+	              shared_ptr<ParquetEncryptionConfig> encryption_config, optional_idx dictionary_size_limit,
 	              idx_t string_dictionary_page_size_limit, bool enable_bloom_filters,
 	              double bloom_filter_false_positive_ratio, int64_t compression_level, bool debug_use_openssl,
 	              ParquetVersion parquet_version);
@@ -117,7 +118,7 @@ public:
 	idx_t FileSize() {
 		return total_written;
 	}
-	idx_t DictionarySizeLimit() const {
+	optional_idx DictionarySizeLimit() const {
 		return dictionary_size_limit;
 	}
 	idx_t StringDictionaryPageSizeLimit() const {
@@ -166,7 +167,7 @@ private:
 	duckdb_parquet::CompressionCodec::type codec;
 	ChildFieldIDs field_ids;
 	shared_ptr<ParquetEncryptionConfig> encryption_config;
-	idx_t dictionary_size_limit;
+	optional_idx dictionary_size_limit;
 	idx_t string_dictionary_page_size_limit;
 	bool enable_bloom_filters;
 	double bloom_filter_false_positive_ratio;

package/src/duckdb/extension/parquet/include/reader/row_number_column_reader.hpp

@@ -8,9 +8,7 @@
 
 #pragma once
 
-#ifndef DUCKDB_AMALGAMATION
 #include "duckdb/common/limits.hpp"
-#endif
 #include "column_reader.hpp"
 #include "reader/templated_column_reader.hpp"
 

package/src/duckdb/extension/parquet/include/reader/uuid_column_reader.hpp

@@ -11,24 +11,14 @@
 #include "column_reader.hpp"
 #include "templated_column_reader.hpp"
 #include "parquet_reader.hpp"
+#include "duckdb/common/types/uuid.hpp"
 
 namespace duckdb {
 
 struct UUIDValueConversion {
 	static hugeint_t ReadParquetUUID(const_data_ptr_t input) {
-		hugeint_t result;
-		result.lower = 0;
-		uint64_t unsigned_upper = 0;
-		for (idx_t i = 0; i < sizeof(uint64_t); i++) {
-			unsigned_upper <<= 8;
-			unsigned_upper += input[i];
-		}
-		for (idx_t i = sizeof(uint64_t); i < sizeof(hugeint_t); i++) {
-			result.lower <<= 8;
-			result.lower += input[i];
-		}
-		result.upper = static_cast<int64_t>(unsigned_upper ^ (uint64_t(1) << 63));
-		return result;
+		// Use the utility function from BaseUUID
+		return BaseUUID::FromBlob(input);
 	}
 
 	template <bool CHECKED>