duckdb 1.3.3-dev0.0 → 1.3.3-dev4.0

package/src/duckdb/extension/parquet/parquet_metadata.cpp

@@ -4,13 +4,11 @@
 
 #include <sstream>
 
-#ifndef DUCKDB_AMALGAMATION
 #include "duckdb/common/multi_file/multi_file_reader.hpp"
 #include "duckdb/common/types/blob.hpp"
 #include "duckdb/common/types/column/column_data_collection.hpp"
 #include "duckdb/planner/filter/constant_filter.hpp"
 #include "duckdb/main/config.hpp"
-#endif
 
 namespace duckdb {
 
@@ -62,28 +60,28 @@ public:
 };
 
 template <class T>
-string ConvertParquetElementToString(T &&entry) {
+static string ConvertParquetElementToString(T &&entry) {
 	duckdb::stringstream ss;
 	ss << entry;
 	return ss.str();
 }
 
 template <class T>
-string PrintParquetElementToString(T &&entry) {
+static string PrintParquetElementToString(T &&entry) {
 	duckdb::stringstream ss;
 	entry.printTo(ss);
 	return ss.str();
 }
 
 template <class T>
-Value ParquetElementString(T &&value, bool is_set) {
+static Value ParquetElementString(T &&value, bool is_set) {
 	if (!is_set) {
 		return Value();
 	}
 	return Value(ConvertParquetElementToString(value));
 }
 
-Value ParquetElementStringVal(const string &value, bool is_set) {
+static Value ParquetElementStringVal(const string &value, bool is_set) {
 	if (!is_set) {
 		return Value();
 	}
@@ -91,7 +89,7 @@ Value ParquetElementStringVal(const string &value, bool is_set) {
 }
 
 template <class T>
-Value ParquetElementInteger(T &&value, bool is_iset) {
+static Value ParquetElementInteger(T &&value, bool is_iset) {
 	if (!is_iset) {
 		return Value();
 	}
@@ -99,14 +97,14 @@ Value ParquetElementInteger(T &&value, bool is_iset) {
 }
 
 template <class T>
-Value ParquetElementBigint(T &&value, bool is_iset) {
+static Value ParquetElementBigint(T &&value, bool is_iset) {
 	if (!is_iset) {
 		return Value();
 	}
 	return Value::BIGINT(value);
 }
 
-Value ParquetElementBoolean(bool value, bool is_iset) {
+static Value ParquetElementBoolean(bool value, bool is_iset) {
 	if (!is_iset) {
 		return Value();
 	}
@@ -200,16 +198,77 @@ void ParquetMetaDataOperatorData::BindMetaData(vector<LogicalType> &return_types
 
 	names.emplace_back("max_is_exact");
 	return_types.emplace_back(LogicalType::BOOLEAN);
+
+	names.emplace_back("row_group_compressed_bytes");
+	return_types.emplace_back(LogicalType::BIGINT);
+
+	names.emplace_back("geo_bbox");
+	return_types.emplace_back(LogicalType::STRUCT({
+	    {"xmin", LogicalType::DOUBLE},
+	    {"xmax", LogicalType::DOUBLE},
+	    {"ymin", LogicalType::DOUBLE},
+	    {"ymax", LogicalType::DOUBLE},
+	    {"zmin", LogicalType::DOUBLE},
+	    {"zmax", LogicalType::DOUBLE},
+	    {"mmin", LogicalType::DOUBLE},
+	    {"mmax", LogicalType::DOUBLE},
+	}));
+
+	names.emplace_back("geo_types");
+	return_types.emplace_back(LogicalType::LIST(LogicalType::VARCHAR));
 }
 
-Value ConvertParquetStats(const LogicalType &type, const ParquetColumnSchema &schema_ele, bool stats_is_set,
-                          const std::string &stats) {
+static Value ConvertParquetStats(const LogicalType &type, const ParquetColumnSchema &schema_ele, bool stats_is_set,
+                                 const std::string &stats) {
 	if (!stats_is_set) {
 		return Value(LogicalType::VARCHAR);
 	}
 	return ParquetStatisticsUtils::ConvertValue(type, schema_ele, stats).DefaultCastAs(LogicalType::VARCHAR);
 }
 
+static Value ConvertParquetGeoStatsBBOX(const duckdb_parquet::GeospatialStatistics &stats) {
+	if (!stats.__isset.bbox) {
+		return Value(LogicalType::STRUCT({
+		    {"xmin", LogicalType::DOUBLE},
+		    {"xmax", LogicalType::DOUBLE},
+		    {"ymin", LogicalType::DOUBLE},
+		    {"ymax", LogicalType::DOUBLE},
+		    {"zmin", LogicalType::DOUBLE},
+		    {"zmax", LogicalType::DOUBLE},
+		    {"mmin", LogicalType::DOUBLE},
+		    {"mmax", LogicalType::DOUBLE},
+		}));
+	}
+
+	return Value::STRUCT({
+	    {"xmin", Value::DOUBLE(stats.bbox.xmin)},
+	    {"xmax", Value::DOUBLE(stats.bbox.xmax)},
+	    {"ymin", Value::DOUBLE(stats.bbox.ymin)},
+	    {"ymax", Value::DOUBLE(stats.bbox.ymax)},
+	    {"zmin", stats.bbox.__isset.zmin ? Value::DOUBLE(stats.bbox.zmin) : Value(LogicalTypeId::DOUBLE)},
+	    {"zmax", stats.bbox.__isset.zmax ? Value::DOUBLE(stats.bbox.zmax) : Value(LogicalTypeId::DOUBLE)},
+	    {"mmin", stats.bbox.__isset.mmin ? Value::DOUBLE(stats.bbox.mmin) : Value(LogicalTypeId::DOUBLE)},
+	    {"mmax", stats.bbox.__isset.mmax ? Value::DOUBLE(stats.bbox.mmax) : Value(LogicalTypeId::DOUBLE)},
+	});
+}
+
+static Value ConvertParquetGeoStatsTypes(const duckdb_parquet::GeospatialStatistics &stats) {
+	if (!stats.__isset.geospatial_types) {
+		return Value(LogicalType::LIST(LogicalType::VARCHAR));
+	}
+	vector<Value> types;
+	types.reserve(stats.geospatial_types.size());
+
+	GeometryKindSet kind_set;
+	for (auto &type : stats.geospatial_types) {
+		kind_set.Add(type);
+	}
+	for (auto &type_name : kind_set.ToString(true)) {
+		types.push_back(Value(type_name));
+	}
+	return Value::LIST(LogicalType::VARCHAR, types);
+}
+
 void ParquetMetaDataOperatorData::LoadRowGroupMetadata(ClientContext &context, const vector<LogicalType> &return_types,
                                                        const OpenFileInfo &file) {
 	collection.Reset();
@@ -350,6 +409,17 @@ void ParquetMetaDataOperatorData::LoadRowGroupMetadata(ClientContext &context, c
 			current_chunk.SetValue(27, count,
 			                       ParquetElementBoolean(stats.is_max_value_exact, stats.__isset.is_max_value_exact));
 
+			// row_group_compressed_bytes
+			current_chunk.SetValue(
+			    28, count,
+			    ParquetElementBigint(row_group.__isset.total_compressed_size, row_group.__isset.total_compressed_size));
+
+			// geo_stats_bbox, LogicalType::STRUCT(...)
+			current_chunk.SetValue(29, count, ConvertParquetGeoStatsBBOX(col_meta.geospatial_statistics));
+
+			// geo_stats_types, LogicalType::LIST(LogicalType::VARCHAR)
+			current_chunk.SetValue(30, count, ConvertParquetGeoStatsTypes(col_meta.geospatial_statistics));
+
 			count++;
 			if (count >= STANDARD_VECTOR_SIZE) {
 				current_chunk.SetCardinality(count);
@@ -402,9 +472,12 @@ void ParquetMetaDataOperatorData::BindSchema(vector<LogicalType> &return_types,
 
 	names.emplace_back("logical_type");
 	return_types.emplace_back(LogicalType::VARCHAR);
+
+	names.emplace_back("duckdb_type");
+	return_types.emplace_back(LogicalType::VARCHAR);
 }
 
-Value ParquetLogicalTypeToString(const duckdb_parquet::LogicalType &type, bool is_set) {
+static Value ParquetLogicalTypeToString(const duckdb_parquet::LogicalType &type, bool is_set) {
 	if (!is_set) {
 		return Value();
 	}
@@ -450,6 +523,12 @@ Value ParquetLogicalTypeToString(const duckdb_parquet::LogicalType &type, bool i
 	if (type.__isset.FLOAT16) {
 		return Value(PrintParquetElementToString(type.FLOAT16));
 	}
+	if (type.__isset.GEOMETRY) {
+		return Value(PrintParquetElementToString(type.GEOMETRY));
+	}
+	if (type.__isset.GEOGRAPHY) {
+		return Value(PrintParquetElementToString(type.GEOGRAPHY));
+	}
 	return Value();
 }
 
@@ -498,6 +577,14 @@ void ParquetMetaDataOperatorData::LoadSchemaData(ClientContext &context, const v
 		// logical_type, LogicalType::VARCHAR
 		current_chunk.SetValue(10, count, ParquetLogicalTypeToString(column.logicalType, column.__isset.logicalType));
 
+		// duckdb_type, LogicalType::VARCHAR
+		ParquetColumnSchema column_schema;
+		Value duckdb_type;
+		if (column.__isset.type) {
+			duckdb_type = reader->DeriveLogicalType(column, column_schema).ToString();
+		}
+		current_chunk.SetValue(11, count, duckdb_type);
+
 		count++;
 		if (count >= STANDARD_VECTOR_SIZE) {
 			current_chunk.SetCardinality(count);
@@ -582,6 +669,12 @@ void ParquetMetaDataOperatorData::BindFileMetaData(vector<LogicalType> &return_t
 
 	names.emplace_back("footer_signing_key_metadata");
 	return_types.emplace_back(LogicalType::VARCHAR);
+
+	names.emplace_back("file_size_bytes");
+	return_types.emplace_back(LogicalType::UBIGINT);
+
+	names.emplace_back("footer_size");
+	return_types.emplace_back(LogicalType::UBIGINT);
 }
 
 void ParquetMetaDataOperatorData::LoadFileMetaData(ClientContext &context, const vector<LogicalType> &return_types,
@@ -610,6 +703,11 @@ void ParquetMetaDataOperatorData::LoadFileMetaData(ClientContext &context, const
 	current_chunk.SetValue(6, 0,
 	                       ParquetElementStringVal(meta_data->footer_signing_key_metadata,
 	                                               meta_data->__isset.footer_signing_key_metadata));
+	//  file_size_bytes
+	current_chunk.SetValue(7, 0, Value::UBIGINT(reader->GetHandle().GetFileSize()));
+	//  footer_size
+	current_chunk.SetValue(8, 0, Value::UBIGINT(reader->metadata->footer_size));
+
 	current_chunk.SetCardinality(1);
 	collection.Append(current_chunk);
 	collection.InitializeScan(scan_state);
@@ -651,7 +749,7 @@ void ParquetMetaDataOperatorData::ExecuteBloomProbe(ClientContext &context, cons
 		throw InvalidInputException("Column %s not found in %s", column_name, file.path);
 	}
 
-	auto &allocator = Allocator::DefaultAllocator();
+	auto &allocator = BufferAllocator::Get(context);
 	auto transport = duckdb_base_std::make_shared<ThriftFileTransport>(reader->GetHandle(), false);
 	auto protocol =
 	    make_uniq<duckdb_apache::thrift::protocol::TCompactProtocolT<ThriftFileTransport>>(std::move(transport));
@@ -689,8 +787,8 @@ void ParquetMetaDataOperatorData::ExecuteBloomProbe(ClientContext &context, cons
 // Bind
 //===--------------------------------------------------------------------===//
 template <ParquetMetadataOperatorType TYPE>
-unique_ptr<FunctionData> ParquetMetaDataBind(ClientContext &context, TableFunctionBindInput &input,
-                                             vector<LogicalType> &return_types, vector<string> &names) {
+static unique_ptr<FunctionData> ParquetMetaDataBind(ClientContext &context, TableFunctionBindInput &input,
+                                                    vector<LogicalType> &return_types, vector<string> &names) {
 	auto result = make_uniq<ParquetMetaDataBindData>();
 
 	switch (TYPE) {
@@ -730,7 +828,7 @@ unique_ptr<FunctionData> ParquetMetaDataBind(ClientContext &context, TableFuncti
 }
 
 template <ParquetMetadataOperatorType TYPE>
-unique_ptr<GlobalTableFunctionState> ParquetMetaDataInit(ClientContext &context, TableFunctionInitInput &input) {
+static unique_ptr<GlobalTableFunctionState> ParquetMetaDataInit(ClientContext &context, TableFunctionInitInput &input) {
 	auto &bind_data = input.bind_data->Cast<ParquetMetaDataBindData>();
 
 	auto result = make_uniq<ParquetMetaDataOperatorData>(context, bind_data.return_types);
@@ -767,7 +865,7 @@ unique_ptr<GlobalTableFunctionState> ParquetMetaDataInit(ClientContext &context,
 }
 
 template <ParquetMetadataOperatorType TYPE>
-void ParquetMetaDataImplementation(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) {
+static void ParquetMetaDataImplementation(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) {
 	auto &data = data_p.global_state->Cast<ParquetMetaDataOperatorData>();
 	auto &bind_data = data_p.bind_data->Cast<ParquetMetaDataBindData>();
 

package/src/duckdb/extension/parquet/parquet_multi_file_info.cpp

@@ -33,10 +33,15 @@ struct ParquetReadBindData : public TableFunctionData {
 };
 
 struct ParquetReadGlobalState : public GlobalTableFunctionState {
+	explicit ParquetReadGlobalState(optional_ptr<const PhysicalOperator> op_p)
+	    : row_group_index(0), batch_index(0), op(op_p) {
+	}
 	//! Index of row group within file currently up for scanning
 	idx_t row_group_index;
 	//! Batch index of the next row group to be scanned
 	idx_t batch_index;
+	//! (Optional) pointer to physical operator performing the scan
+	optional_ptr<const PhysicalOperator> op;
 };
 
 struct ParquetReadLocalState : public LocalTableFunctionState {
@@ -100,7 +105,7 @@ static void BindSchema(ClientContext &context, vector<LogicalType> &return_types
 #endif
 
 		res.default_expression = make_uniq<ConstantExpression>(column.default_value);
-		reader_bind.schema.emplace_back(std::move(res));
+		reader_bind.schema.emplace_back(res);
 	}
 	ParseFileRowNumberOption(reader_bind, options, return_types, names);
 	if (options.file_row_number) {
@@ -108,7 +113,7 @@ static void BindSchema(ClientContext &context, vector<LogicalType> &return_types
 		res.identifier = Value::INTEGER(MultiFileReader::ORDINAL_FIELD_ID);
 		schema_col_names.push_back(res.name);
 		schema_col_types.push_back(res.type);
-		reader_bind.schema.emplace_back(std::move(res));
+		reader_bind.schema.emplace_back(res);
 	}
 
 	if (match_by_field_id) {
@@ -126,8 +131,7 @@ static void BindSchema(ClientContext &context, vector<LogicalType> &return_types
 	D_ASSERT(names.size() == return_types.size());
 }
 
-unique_ptr<MultiFileReaderInterface>
-ParquetMultiFileInfo::InitializeInterface(ClientContext &context, MultiFileReader &reader, MultiFileList &file_list) {
+unique_ptr<MultiFileReaderInterface> ParquetMultiFileInfo::CreateInterface(ClientContext &context) {
 	return make_uniq<ParquetMultiFileInfo>();
 }
 
@@ -239,10 +243,10 @@ static unique_ptr<FunctionData> ParquetScanDeserialize(Deserializer &deserialize
 	for (auto &path : files) {
 		file_path.emplace_back(path);
 	}
+	FileGlobInput input(FileGlobOptions::FALLBACK_GLOB, "parquet");
 
 	auto multi_file_reader = MultiFileReader::Create(function);
-	auto file_list = multi_file_reader->CreateFileList(context, Value::LIST(LogicalType::VARCHAR, file_path),
-	                                                   FileGlobOptions::DISALLOW_EMPTY);
+	auto file_list = multi_file_reader->CreateFileList(context, Value::LIST(LogicalType::VARCHAR, file_path), input);
 	auto parquet_options = make_uniq<ParquetFileReaderOptions>(std::move(serialization.parquet_options));
 	auto interface = make_uniq<ParquetMultiFileInfo>();
 	auto bind_data = MultiFileFunction<ParquetMultiFileInfo>::MultiFileBindInternal(
@@ -252,14 +256,14 @@ static unique_ptr<FunctionData> ParquetScanDeserialize(Deserializer &deserialize
 	return bind_data;
 }
 
-vector<column_t> ParquetGetRowIdColumns(ClientContext &context, optional_ptr<FunctionData> bind_data) {
+static vector<column_t> ParquetGetRowIdColumns(ClientContext &context, optional_ptr<FunctionData> bind_data) {
 	vector<column_t> result;
 	result.emplace_back(MultiFileReader::COLUMN_IDENTIFIER_FILE_INDEX);
 	result.emplace_back(MultiFileReader::COLUMN_IDENTIFIER_FILE_ROW_NUMBER);
 	return result;
 }
 
-vector<PartitionStatistics> ParquetGetPartitionStats(ClientContext &context, GetPartitionStatsInput &input) {
+static vector<PartitionStatistics> ParquetGetPartitionStats(ClientContext &context, GetPartitionStatsInput &input) {
 	auto &bind_data = input.bind_data->Cast<MultiFileBindData>();
 	vector<PartitionStatistics> result;
 	if (bind_data.file_list->GetExpandResult() == FileExpandResult::SINGLE_FILE && bind_data.initial_reader) {
@@ -393,6 +397,10 @@ bool ParquetMultiFileInfo::ParseOption(ClientContext &context, const string &ori
 		options.binary_as_string = BooleanValue::Get(val);
 		return true;
 	}
+	if (key == "variant_legacy_encoding") {
+		options.variant_legacy_encoding = BooleanValue::Get(val);
+		return true;
+	}
 	if (key == "file_row_number") {
 		options.file_row_number = BooleanValue::Get(val);
 		return true;
@@ -532,13 +540,14 @@ shared_ptr<BaseUnionData> ParquetReader::GetUnionData(idx_t file_idx) {
 	} else {
 		result->options = std::move(parquet_options);
 		result->metadata = std::move(metadata);
+		result->root_schema = std::move(root_schema);
 	}
 	return std::move(result);
 }
 
 unique_ptr<GlobalTableFunctionState> ParquetMultiFileInfo::InitializeGlobalState(ClientContext &, MultiFileBindData &,
-                                                                                 MultiFileGlobalState &) {
-	return make_uniq<ParquetReadGlobalState>();
+                                                                                 MultiFileGlobalState &global_state) {
+	return make_uniq<ParquetReadGlobalState>(global_state.op);
 }
 
 unique_ptr<LocalTableFunctionState> ParquetMultiFileInfo::InitializeLocalState(ExecutionContext &,
@@ -568,7 +577,9 @@ void ParquetReader::FinishFile(ClientContext &context, GlobalTableFunctionState
 
 void ParquetReader::Scan(ClientContext &context, GlobalTableFunctionState &gstate_p,
                          LocalTableFunctionState &local_state_p, DataChunk &chunk) {
+	auto &gstate = gstate_p.Cast<ParquetReadGlobalState>();
 	auto &local_state = local_state_p.Cast<ParquetReadLocalState>();
+	local_state.scan_state.op = gstate.op;
 	Scan(context, local_state.scan_state, chunk);
 }
 
@@ -576,4 +587,8 @@ unique_ptr<MultiFileReaderInterface> ParquetMultiFileInfo::Copy() {
 	return make_uniq<ParquetMultiFileInfo>();
 }
 
+FileGlobInput ParquetMultiFileInfo::GetGlobInput() {
+	return FileGlobInput(FileGlobOptions::FALLBACK_GLOB, "parquet");
+}
+
 } // namespace duckdb