duckdb 1.3.3-dev4.0 → 1.3.3

package/src/duckdb/extension/parquet/parquet_reader.cpp

@@ -14,7 +14,6 @@
 #include "mbedtls_wrapper.hpp"
 #include "reader/row_number_column_reader.hpp"
 #include "reader/string_column_reader.hpp"
-#include "reader/variant_column_reader.hpp"
 #include "reader/struct_column_reader.hpp"
 #include "reader/templated_column_reader.hpp"
 #include "thrift_tools.hpp"
@@ -29,7 +28,6 @@
 #include "duckdb/optimizer/statistics_propagator.hpp"
 #include "duckdb/planner/table_filter_state.hpp"
 #include "duckdb/common/multi_file/multi_file_reader.hpp"
-#include "duckdb/logging/log_manager.hpp"
 
 #include <cassert>
 #include <chrono>
@@ -48,15 +46,20 @@ using duckdb_parquet::SchemaElement;
 using duckdb_parquet::Statistics;
 using duckdb_parquet::Type;
 
-static unique_ptr<duckdb_apache::thrift::protocol::TProtocol>
-CreateThriftFileProtocol(QueryContext context, CachingFileHandle &file_handle, bool prefetch_mode) {
+static unique_ptr<duckdb_apache::thrift::protocol::TProtocol> CreateThriftFileProtocol(CachingFileHandle &file_handle,
+                                                                                       bool prefetch_mode) {
 	auto transport = duckdb_base_std::make_shared<ThriftFileTransport>(file_handle, prefetch_mode);
 	return make_uniq<duckdb_apache::thrift::protocol::TCompactProtocolT<ThriftFileTransport>>(std::move(transport));
 }
 
 static bool ShouldAndCanPrefetch(ClientContext &context, CachingFileHandle &file_handle) {
 	Value disable_prefetch = false;
-	Value prefetch_all_files = false;
+	Value prefetch_all_files =
+#ifdef DUCKDB_PREFETCH_ALL_PARQUET_FILES
+	    true; // For debugging purposes we can toggle this to always enable
+#else
+	    false; // Defaults to false
+#endif
 	context.TryGetCurrentSetting("disable_parquet_prefetching", disable_prefetch);
 	context.TryGetCurrentSetting("prefetch_all_parquet_files", prefetch_all_files);
 	bool should_prefetch = !file_handle.OnDiskFile() || prefetch_all_files.GetValue<bool>();
@@ -64,9 +67,9 @@ static bool ShouldAndCanPrefetch(ClientContext &context, CachingFileHandle &file
 	return should_prefetch && can_prefetch;
 }
 
-static void ParseParquetFooter(data_ptr_t buffer, const string &file_path, idx_t file_size,
-                               const shared_ptr<const ParquetEncryptionConfig> &encryption_config, uint32_t &footer_len,
-                               bool &footer_encrypted) {
+void ParseParquetFooter(data_ptr_t buffer, const string &file_path, idx_t file_size,
+                        const shared_ptr<const ParquetEncryptionConfig> &encryption_config, uint32_t &footer_len,
+                        bool &footer_encrypted) {
 	if (memcmp(buffer + 4, "PAR1", 4) == 0) {
 		footer_encrypted = false;
 		if (encryption_config) {
@@ -92,7 +95,7 @@ static shared_ptr<ParquetFileMetadataCache>
 LoadMetadata(ClientContext &context, Allocator &allocator, CachingFileHandle &file_handle,
              const shared_ptr<const ParquetEncryptionConfig> &encryption_config, const EncryptionUtil &encryption_util,
              optional_idx footer_size) {
-	auto file_proto = CreateThriftFileProtocol(QueryContext(context), file_handle, false);
+	auto file_proto = CreateThriftFileProtocol(file_handle, false);
 	auto &transport = reinterpret_cast<ThriftFileTransport &>(*file_proto->getTransport());
 	auto file_size = transport.GetSize();
 	if (file_size < 12) {
@@ -100,7 +103,6 @@ LoadMetadata(ClientContext &context, Allocator &allocator, CachingFileHandle &fi
 	}
 
 	bool footer_encrypted;
-	uint32_t footer_len;
 	// footer size is not provided - read it from the back
 	if (!footer_size.IsValid()) {
 		// We have to do two reads here:
@@ -126,6 +128,7 @@ LoadMetadata(ClientContext &context, Allocator &allocator, CachingFileHandle &fi
 		transport.SetLocation(file_size - 8);
 		transport.read(buf.ptr, 8);
 
+		uint32_t footer_len;
 		ParseParquetFooter(buf.ptr, file_handle.GetPath(), file_size, encryption_config, footer_len, footer_encrypted);
 
 		auto metadata_pos = file_size - (footer_len + 8);
@@ -134,7 +137,7 @@ LoadMetadata(ClientContext &context, Allocator &allocator, CachingFileHandle &fi
 			transport.Prefetch(metadata_pos, footer_len);
 		}
 	} else {
-		footer_len = UnsafeNumericCast<uint32_t>(footer_size.GetIndex());
+		auto footer_len = UnsafeNumericCast<uint32_t>(footer_size.GetIndex());
 		if (footer_len == 0 || file_size < 12 + footer_len) {
 			throw InvalidInputException("Invalid footer length provided for file '%s'", file_handle.GetPath());
 		}
@@ -170,8 +173,7 @@ LoadMetadata(ClientContext &context, Allocator &allocator, CachingFileHandle &fi
 
 	// Try to read the GeoParquet metadata (if present)
 	auto geo_metadata = GeoParquetFileMetadata::TryRead(*metadata, context);
-	return make_shared_ptr<ParquetFileMetadataCache>(std::move(metadata), file_handle, std::move(geo_metadata),
-	                                                 footer_len);
+	return make_shared_ptr<ParquetFileMetadataCache>(std::move(metadata), file_handle, std::move(geo_metadata));
 }
 
 LogicalType ParquetReader::DeriveLogicalType(const SchemaElement &s_ele, ParquetColumnSchema &schema) const {
@@ -225,10 +227,6 @@ LogicalType ParquetReader::DeriveLogicalType(const SchemaElement &s_ele, Parquet
 				return LogicalType::TIME_TZ;
 			}
 			return LogicalType::TIME;
-		} else if (s_ele.logicalType.__isset.GEOMETRY) {
-			return LogicalType::BLOB;
-		} else if (s_ele.logicalType.__isset.GEOGRAPHY) {
-			return LogicalType::BLOB;
 		}
 	}
 	if (s_ele.__isset.converted_type) {
@@ -407,7 +405,7 @@ unique_ptr<ColumnReader> ParquetReader::CreateReaderRecursive(ClientContext &con
                                                               const ParquetColumnSchema &schema) {
 	switch (schema.schema_type) {
 	case ParquetColumnSchemaType::GEOMETRY:
-		return GeoParquetFileMetadata::CreateColumnReader(*this, schema, context);
+		return metadata->geo_metadata->CreateColumnReader(*this, schema, context);
 	case ParquetColumnSchemaType::FILE_ROW_NUMBER:
 		return make_uniq<RowNumberColumnReader>(*this, schema);
 	case ParquetColumnSchemaType::COLUMN: {
@@ -439,17 +437,6 @@ unique_ptr<ColumnReader> ParquetReader::CreateReaderRecursive(ClientContext &con
 			throw InternalException("Unsupported schema type for schema with children");
 		}
 	}
-	case ParquetColumnSchemaType::VARIANT: {
-		if (schema.children.size() < 2) {
-			throw InternalException("VARIANT schema type used for a non-variant type column");
-		}
-		vector<unique_ptr<ColumnReader>> children;
-		children.resize(schema.children.size());
-		for (idx_t child_index = 0; child_index < schema.children.size(); child_index++) {
-			children[child_index] = CreateReaderRecursive(context, indexes, schema.children[child_index]);
-		}
-		return make_uniq<VariantColumnReader>(context, *this, schema, std::move(children));
-	}
 	default:
 		throw InternalException("Unsupported ParquetColumnSchemaType");
 	}
@@ -494,15 +481,14 @@ ParquetColumnSchema::ParquetColumnSchema(ParquetColumnSchema parent, LogicalType
 	children.push_back(std::move(parent));
 }
 
-unique_ptr<BaseStatistics> ParquetColumnSchema::Stats(const FileMetaData &file_meta_data,
-                                                      const ParquetOptions &parquet_options, idx_t row_group_idx_p,
+unique_ptr<BaseStatistics> ParquetColumnSchema::Stats(ParquetReader &reader, idx_t row_group_idx_p,
                                                       const vector<ColumnChunk> &columns) const {
 	if (schema_type == ParquetColumnSchemaType::EXPRESSION) {
 		return nullptr;
 	}
 	if (schema_type == ParquetColumnSchemaType::FILE_ROW_NUMBER) {
 		auto stats = NumericStats::CreateUnknown(type);
-		auto &row_groups = file_meta_data.row_groups;
+		auto &row_groups = reader.GetFileMetadata()->row_groups;
 		D_ASSERT(row_group_idx_p < row_groups.size());
 		idx_t row_group_offset_min = 0;
 		for (idx_t i = 0; i < row_group_idx_p; i++) {
@@ -515,58 +501,11 @@ unique_ptr<BaseStatistics> ParquetColumnSchema::Stats(const FileMetaData &file_m
 		stats.Set(StatsInfo::CANNOT_HAVE_NULL_VALUES);
 		return stats.ToUnique();
 	}
-	return ParquetStatisticsUtils::TransformColumnStatistics(*this, columns, parquet_options.can_have_nan);
-}
-
-static bool IsVariantType(const SchemaElement &root, const vector<ParquetColumnSchema> &children) {
-	if (children.size() < 2) {
-		return false;
-	}
-	auto &child0 = children[0];
-	auto &child1 = children[1];
-
-	ParquetColumnSchema const *metadata;
-	ParquetColumnSchema const *value;
-
-	if (child0.name == "metadata" && child1.name == "value") {
-		metadata = &child0;
-		value = &child1;
-	} else if (child1.name == "metadata" && child0.name == "value") {
-		metadata = &child1;
-		value = &child0;
-	} else {
-		return false;
-	}
-
-	//! Verify names
-	if (metadata->name != "metadata") {
-		return false;
-	}
-	if (value->name != "value") {
-		return false;
-	}
-
-	//! Verify types
-	if (metadata->parquet_type != duckdb_parquet::Type::BYTE_ARRAY) {
-		return false;
-	}
-	if (value->parquet_type != duckdb_parquet::Type::BYTE_ARRAY) {
-		return false;
-	}
-	if (children.size() == 3) {
-		auto &typed_value = children[2];
-		if (typed_value.name != "typed_value") {
-			return false;
-		}
-	} else if (children.size() != 2) {
-		return false;
-	}
-	return true;
+	return ParquetStatisticsUtils::TransformColumnStatistics(*this, columns, reader.parquet_options.can_have_nan);
 }
 
 ParquetColumnSchema ParquetReader::ParseSchemaRecursive(idx_t depth, idx_t max_define, idx_t max_repeat,
-                                                        idx_t &next_schema_idx, idx_t &next_file_idx,
-                                                        ClientContext &context) {
+                                                        idx_t &next_schema_idx, idx_t &next_file_idx) {
 
 	auto file_meta_data = GetFileMetadata();
 	D_ASSERT(file_meta_data);
@@ -587,9 +526,8 @@ ParquetColumnSchema ParquetReader::ParseSchemaRecursive(idx_t depth, idx_t max_d
 
 	// Check for geoparquet spatial types
 	if (depth == 1) {
-		// geoparquet types have to be at the root of the schema, and have to be present in the kv metadata.
-		// geoarrow types, although geometry columns, are structs and have children and are handled below.
-		if (metadata->geo_metadata && metadata->geo_metadata->IsGeometryColumn(s_ele.name) && s_ele.num_children == 0) {
+		// geoparquet types have to be at the root of the schema, and have to be present in the kv metadata
+		if (metadata->geo_metadata && metadata->geo_metadata->IsGeometryColumn(s_ele.name)) {
 			auto root_schema = ParseColumnSchema(s_ele, max_define, max_repeat, this_idx, next_file_idx++);
 			return ParquetColumnSchema(std::move(root_schema), GeoParquetFileMetadata::GeometryType(),
 			                           ParquetColumnSchemaType::GEOMETRY);
@@ -603,8 +541,7 @@ ParquetColumnSchema ParquetReader::ParseSchemaRecursive(idx_t depth, idx_t max_d
 		while (c_idx < NumericCast<idx_t>(s_ele.num_children)) {
 			next_schema_idx++;
 
-			auto child_schema =
-			    ParseSchemaRecursive(depth + 1, max_define, max_repeat, next_schema_idx, next_file_idx, context);
+			auto child_schema = ParseSchemaRecursive(depth + 1, max_define, max_repeat, next_schema_idx, next_file_idx);
 			child_schemas.push_back(std::move(child_schema));
 			c_idx++;
 		}
@@ -623,14 +560,9 @@ ParquetColumnSchema ParquetReader::ParseSchemaRecursive(idx_t depth, idx_t max_d
 		}
 
 		bool is_repeated = repetition_type == FieldRepetitionType::REPEATED;
-		const bool is_list = s_ele.__isset.converted_type && s_ele.converted_type == ConvertedType::LIST;
-		const bool is_map = s_ele.__isset.converted_type && s_ele.converted_type == ConvertedType::MAP;
+		bool is_list = s_ele.__isset.converted_type && s_ele.converted_type == ConvertedType::LIST;
+		bool is_map = s_ele.__isset.converted_type && s_ele.converted_type == ConvertedType::MAP;
 		bool is_map_kv = s_ele.__isset.converted_type && s_ele.converted_type == ConvertedType::MAP_KEY_VALUE;
-		bool is_variant = s_ele.__isset.logicalType && s_ele.logicalType.__isset.VARIANT == true;
-		if (!is_variant) {
-			is_variant = parquet_options.variant_legacy_encoding && IsVariantType(s_ele, child_schemas);
-		}
-
 		if (!is_map_kv && this_idx > 0) {
 			// check if the parent node of this is a map
 			auto &p_ele = file_meta_data->schema[this_idx - 1];
@@ -663,18 +595,10 @@ ParquetColumnSchema ParquetReader::ParseSchemaRecursive(idx_t depth, idx_t max_d
 				struct_types.emplace_back(make_pair(child_schema.name, child_schema.type));
 			}
 
-			LogicalType result_type;
-			if (is_variant) {
-				result_type = LogicalType::JSON();
-			} else {
-				result_type = LogicalType::STRUCT(std::move(struct_types));
-			}
+			auto result_type = LogicalType::STRUCT(std::move(struct_types));
 			ParquetColumnSchema struct_schema(s_ele.name, std::move(result_type), max_define, max_repeat, this_idx,
 			                                  next_file_idx);
 			struct_schema.children = std::move(child_schemas);
-			if (is_variant) {
-				struct_schema.schema_type = ParquetColumnSchemaType::VARIANT;
-			}
 			result = std::move(struct_schema);
 		} else {
 			// if we have a struct with only a single type, pull up
@@ -704,24 +628,16 @@ ParquetColumnSchema ParquetReader::ParseSchemaRecursive(idx_t depth, idx_t max_d
 			list_schema.children.push_back(std::move(result));
 			return list_schema;
 		}
-
-		// Convert to geometry type if possible
-		if (s_ele.__isset.logicalType && (s_ele.logicalType.__isset.GEOMETRY || s_ele.logicalType.__isset.GEOGRAPHY) &&
-		    GeoParquetFileMetadata::IsGeoParquetConversionEnabled(context)) {
-			return ParquetColumnSchema(std::move(result), GeoParquetFileMetadata::GeometryType(),
-			                           ParquetColumnSchemaType::GEOMETRY);
-		}
-
 		return result;
 	}
 }
 
-static ParquetColumnSchema FileRowNumberSchema() {
+ParquetColumnSchema FileRowNumberSchema() {
 	return ParquetColumnSchema("file_row_number", LogicalType::BIGINT, 0, 0, 0, 0,
 	                           ParquetColumnSchemaType::FILE_ROW_NUMBER);
 }
 
-unique_ptr<ParquetColumnSchema> ParquetReader::ParseSchema(ClientContext &context) {
+unique_ptr<ParquetColumnSchema> ParquetReader::ParseSchema() {
 	auto file_meta_data = GetFileMetadata();
 	idx_t next_schema_idx = 0;
 	idx_t next_file_idx = 0;
@@ -732,7 +648,7 @@ unique_ptr<ParquetColumnSchema> ParquetReader::ParseSchema(ClientContext &contex
 	if (file_meta_data->schema[0].num_children == 0) {
 		throw IOException("Parquet reader: root schema element has no children");
 	}
-	auto root = ParseSchemaRecursive(0, 0, 0, next_schema_idx, next_file_idx, context);
+	auto root = ParseSchemaRecursive(0, 0, 0, next_schema_idx, next_file_idx);
 	if (root.type.id() != LogicalTypeId::STRUCT) {
 		throw InvalidInputException("Root element of Parquet file must be a struct");
 	}
@@ -788,7 +704,7 @@ void ParquetReader::InitializeSchema(ClientContext &context) {
 		throw InvalidInputException("Failed to read Parquet file '%s': Need at least one non-root column in the file",
 		                            GetFileName());
 	}
-	root_schema = ParseSchema(context);
+	root_schema = ParseSchema();
 	for (idx_t i = 0; i < root_schema->children.size(); i++) {
 		auto &element = root_schema->children[i];
 		columns.push_back(ParseColumnDefinition(*file_meta_data, element));
@@ -804,12 +720,9 @@ void ParquetReader::AddVirtualColumn(column_t virtual_column_id) {
 }
 
 ParquetOptions::ParquetOptions(ClientContext &context) {
-	Value lookup_value;
-	if (context.TryGetCurrentSetting("binary_as_string", lookup_value)) {
-		binary_as_string = lookup_value.GetValue<bool>();
-	}
-	if (context.TryGetCurrentSetting("variant_legacy_encoding", lookup_value)) {
-		variant_legacy_encoding = lookup_value.GetValue<bool>();
+	Value binary_as_string_val;
+	if (context.TryGetCurrentSetting("binary_as_string", binary_as_string_val)) {
+		binary_as_string = binary_as_string_val.GetValue<bool>();
 	}
 }
 
@@ -837,7 +750,7 @@ ParquetReader::ParquetReader(ClientContext &context_p, OpenFileInfo file_p, Parq
                              shared_ptr<ParquetFileMetadataCache> metadata_p)
     : BaseFileReader(std::move(file_p)), fs(CachingFileSystem::Get(context_p)),
       allocator(BufferAllocator::Get(context_p)), parquet_options(std::move(parquet_options_p)) {
-	file_handle = fs.OpenFile(QueryContext(context_p), file, FileFlags::FILE_FLAGS_READ);
+	file_handle = fs.OpenFile(file, FileFlags::FILE_FLAGS_READ);
 	if (!file_handle->CanSeek()) {
 		throw NotImplementedException(
 		    "Reading parquet files from a FIFO stream is not supported and cannot be efficiently supported since "
@@ -900,13 +813,6 @@ shared_ptr<ParquetFileMetadataCache> ParquetReader::GetMetadataCacheEntry(Client
 ParquetUnionData::~ParquetUnionData() {
 }
 
-unique_ptr<BaseStatistics> ParquetUnionData::GetStatistics(ClientContext &context, const string &name) {
-	if (reader) {
-		return reader->Cast<ParquetReader>().GetStatistics(context, name);
-	}
-	return ParquetReader::ReadStatistics(*this, name);
-}
-
 ParquetReader::ParquetReader(ClientContext &context_p, ParquetOptions parquet_options_p,
                              shared_ptr<ParquetFileMetadataCache> metadata_p)
     : BaseFileReader(string()), fs(CachingFileSystem::Get(context_p)), allocator(BufferAllocator::Get(context_p)),
@@ -923,16 +829,24 @@ const FileMetaData *ParquetReader::GetFileMetadata() const {
 	return metadata->metadata.get();
 }
 
-static unique_ptr<BaseStatistics> ReadStatisticsInternal(const FileMetaData &file_meta_data,
-                                                         const ParquetColumnSchema &root_schema,
-                                                         const ParquetOptions &parquet_options,
-                                                         const idx_t &file_col_idx) {
+unique_ptr<BaseStatistics> ParquetReader::ReadStatistics(const string &name) {
+	idx_t file_col_idx;
+	for (file_col_idx = 0; file_col_idx < columns.size(); file_col_idx++) {
+		if (columns[file_col_idx].name == name) {
+			break;
+		}
+	}
+	if (file_col_idx == columns.size()) {
+		return nullptr;
+	}
+
 	unique_ptr<BaseStatistics> column_stats;
-	auto &column_schema = root_schema.children[file_col_idx];
+	auto file_meta_data = GetFileMetadata();
+	auto &column_schema = root_schema->children[file_col_idx];
 
-	for (idx_t row_group_idx = 0; row_group_idx < file_meta_data.row_groups.size(); row_group_idx++) {
-		auto &row_group = file_meta_data.row_groups[row_group_idx];
-		auto chunk_stats = column_schema.Stats(file_meta_data, parquet_options, row_group_idx, row_group.columns);
+	for (idx_t row_group_idx = 0; row_group_idx < file_meta_data->row_groups.size(); row_group_idx++) {
+		auto &row_group = file_meta_data->row_groups[row_group_idx];
+		auto chunk_stats = column_schema.Stats(*this, row_group_idx, row_group.columns);
 		if (!chunk_stats) {
 			return nullptr;
 		}
@@ -945,20 +859,6 @@ static unique_ptr<BaseStatistics> ReadStatisticsInternal(const FileMetaData &fil
 	return column_stats;
 }
 
-unique_ptr<BaseStatistics> ParquetReader::ReadStatistics(const string &name) {
-	idx_t file_col_idx;
-	for (file_col_idx = 0; file_col_idx < columns.size(); file_col_idx++) {
-		if (columns[file_col_idx].name == name) {
-			break;
-		}
-	}
-	if (file_col_idx == columns.size()) {
-		return nullptr;
-	}
-
-	return ReadStatisticsInternal(*GetFileMetadata(), *root_schema, parquet_options, file_col_idx);
-}
-
 unique_ptr<BaseStatistics> ParquetReader::ReadStatistics(ClientContext &context, ParquetOptions parquet_options,
                                                          shared_ptr<ParquetFileMetadataCache> metadata,
                                                          const string &name) {
@@ -966,23 +866,6 @@ unique_ptr<BaseStatistics> ParquetReader::ReadStatistics(ClientContext &context,
 	return reader.ReadStatistics(name);
 }
 
-unique_ptr<BaseStatistics> ParquetReader::ReadStatistics(const ParquetUnionData &union_data, const string &name) {
-	const auto &col_names = union_data.names;
-
-	idx_t file_col_idx;
-	for (file_col_idx = 0; file_col_idx < col_names.size(); file_col_idx++) {
-		if (col_names[file_col_idx] == name) {
-			break;
-		}
-	}
-	if (file_col_idx == col_names.size()) {
-		return nullptr;
-	}
-
-	return ReadStatisticsInternal(*union_data.metadata->metadata, *union_data.root_schema, union_data.options,
-	                              file_col_idx);
-}
-
 uint32_t ParquetReader::Read(duckdb_apache::thrift::TBase &object, TProtocol &iprot) {
 	if (parquet_options.encryption_config) {
 		return ParquetCrypto::Read(object, iprot, parquet_options.encryption_config->GetFooterKey(), *encryption_util);
@@ -1001,7 +884,7 @@ uint32_t ParquetReader::ReadData(duckdb_apache::thrift::protocol::TProtocol &ipr
 	}
 }
 
-static idx_t GetRowGroupOffset(ParquetReader &reader, idx_t group_idx) {
+idx_t GetRowGroupOffset(ParquetReader &reader, idx_t group_idx) {
 	idx_t row_group_offset = 0;
 	auto &row_groups = reader.GetFileMetadata()->row_groups;
 	for (idx_t i = 0; i < group_idx; i++) {
@@ -1018,8 +901,8 @@ const ParquetRowGroup &ParquetReader::GetGroup(ParquetReaderScanState &state) {
 }
 
 uint64_t ParquetReader::GetGroupCompressedSize(ParquetReaderScanState &state) {
-	const auto &group = GetGroup(state);
-	int64_t total_compressed_size = group.__isset.total_compressed_size ? group.total_compressed_size : 0;
+	auto &group = GetGroup(state);
+	auto total_compressed_size = group.total_compressed_size;
 
 	idx_t calc_compressed_size = 0;
 
@@ -1032,9 +915,7 @@ uint64_t ParquetReader::GetGroupCompressedSize(ParquetReaderScanState &state) {
 
 	if (total_compressed_size != 0 && calc_compressed_size != 0 &&
 	    (idx_t)total_compressed_size != calc_compressed_size) {
-		throw InvalidInputException(
-		    "Failed to read file \"%s\": mismatch between calculated compressed size and reported compressed size",
-		    GetFileName());
+		throw InvalidInputException("mismatch between calculated compressed size and reported compressed size");
 	}
 
 	return total_compressed_size ? total_compressed_size : calc_compressed_size;
@@ -1236,7 +1117,7 @@ void ParquetReader::InitializeScan(ClientContext &context, ParquetReaderScanStat
 			state.prefetch_mode = false;
 		}
 
-		state.file_handle = fs.OpenFile(QueryContext(context), file, flags);
+		state.file_handle = fs.OpenFile(file, flags);
 	}
 	state.adaptive_filter.reset();
 	state.scan_filters.clear();
@@ -1247,7 +1128,7 @@ void ParquetReader::InitializeScan(ClientContext &context, ParquetReaderScanStat
 		}
 	}
 
-	state.thrift_file_proto = CreateThriftFileProtocol(QueryContext(context), *state.file_handle, state.prefetch_mode);
+	state.thrift_file_proto = CreateThriftFileProtocol(*state.file_handle, state.prefetch_mode);
 	state.root_reader = CreateReader(context);
 	state.define_buf.resize(allocator, STANDARD_VECTOR_SIZE);
 	state.repeat_buf.resize(allocator, STANDARD_VECTOR_SIZE);
@@ -1313,12 +1194,6 @@ bool ParquetReader::ScanInternal(ClientContext &context, ParquetReaderScanState
 		}
 
 		auto &group = GetGroup(state);
-		if (state.op) {
-			DUCKDB_LOG(context, PhysicalOperatorLogType, *state.op, "ParquetReader",
-			           state.offset_in_group == (idx_t)group.num_rows ? "SkipRowGroup" : "ReadRowGroup",
-			           {{"file", file.path}, {"row_group_id", to_string(state.group_idx_list[state.current_group])}});
-		}
-
 		if (state.prefetch_mode && state.offset_in_group != (idx_t)group.num_rows) {
 			uint64_t total_row_group_span = GetGroupSpan(state);
 

package/src/duckdb/extension/parquet/parquet_statistics.cpp

@@ -8,12 +8,15 @@
 #include "reader/string_column_reader.hpp"
 #include "reader/struct_column_reader.hpp"
 #include "zstd/common/xxhash.hpp"
+
+#ifndef DUCKDB_AMALGAMATION
 #include "duckdb/common/types/blob.hpp"
 #include "duckdb/common/types/time.hpp"
 #include "duckdb/common/types/value.hpp"
 #include "duckdb/storage/statistics/struct_stats.hpp"
 #include "duckdb/planner/filter/constant_filter.hpp"
 #include "reader/uuid_column_reader.hpp"
+#endif
 
 namespace duckdb {
 
@@ -211,25 +214,6 @@ Value ParquetStatisticsUtils::ConvertValueInternal(const LogicalType &type, cons
 			return Value::TIME(dtime_t(val));
 		}
 	}
-	case LogicalTypeId::TIME_NS: {
-		int64_t val;
-		if (stats.size() == sizeof(int32_t)) {
-			val = Load<int32_t>(stats_data);
-		} else if (stats.size() == sizeof(int64_t)) {
-			val = Load<int64_t>(stats_data);
-		} else {
-			throw InvalidInputException("Incorrect stats size for type TIME");
-		}
-		switch (schema_ele.type_info) {
-		case ParquetExtraTypeInfo::UNIT_MS:
-			return Value::TIME_NS(ParquetMsIntToTimeNs(val));
-		case ParquetExtraTypeInfo::UNIT_NS:
-			return Value::TIME_NS(ParquetIntToTimeNs(val));
-		case ParquetExtraTypeInfo::UNIT_MICROS:
-		default:
-			return Value::TIME_NS(dtime_ns_t(val));
-		}
-	}
 	case LogicalTypeId::TIME_TZ: {
 		int64_t val;
 		if (stats.size() == sizeof(int32_t)) {
@@ -347,9 +331,6 @@ unique_ptr<BaseStatistics> ParquetStatisticsUtils::TransformColumnStatistics(con
 			row_group_stats->Set(StatsInfo::CAN_HAVE_NULL_AND_VALID_VALUES);
 		}
 		return row_group_stats;
-	} else if (schema.schema_type == ParquetColumnSchemaType::VARIANT) {
-		//! FIXME: there are situations where VARIANT columns can have stats
-		return nullptr;
 	}
 
 	// Otherwise, its a standard column with stats
@@ -402,6 +383,8 @@ unique_ptr<BaseStatistics> ParquetStatisticsUtils::TransformColumnStatistics(con
 		} else if (parquet_stats.__isset.min) {
 			StringColumnReader::VerifyString(parquet_stats.min.c_str(), parquet_stats.min.size(), true);
 			StringStats::Update(string_stats, parquet_stats.min);
+		} else {
+			return nullptr;
 		}
 		if (parquet_stats.__isset.max_value) {
 			StringColumnReader::VerifyString(parquet_stats.max_value.c_str(), parquet_stats.max_value.size(), true);
@@ -409,6 +392,8 @@ unique_ptr<BaseStatistics> ParquetStatisticsUtils::TransformColumnStatistics(con
 		} else if (parquet_stats.__isset.max) {
 			StringColumnReader::VerifyString(parquet_stats.max.c_str(), parquet_stats.max.size(), true);
 			StringStats::Update(string_stats, parquet_stats.max);
+		} else {
+			return nullptr;
 		}
 		StringStats::SetContainsUnicode(string_stats);
 		StringStats::ResetMaxStringLength(string_stats);
@@ -426,9 +411,6 @@ unique_ptr<BaseStatistics> ParquetStatisticsUtils::TransformColumnStatistics(con
 		if (parquet_stats.__isset.null_count && parquet_stats.null_count == 0) {
 			row_group_stats->Set(StatsInfo::CANNOT_HAVE_NULL_VALUES);
 		}
-		if (parquet_stats.__isset.null_count && parquet_stats.null_count == column_chunk.meta_data.num_values) {
-			row_group_stats->Set(StatsInfo::CANNOT_HAVE_VALID_VALUES);
-		}
 	}
 	return row_group_stats;
 }
@@ -461,7 +443,7 @@ static bool HasFilterConstants(const TableFilter &duckdb_filter) {
 }
 
 template <class T>
-static uint64_t ValueXH64FixedWidth(const Value &constant) {
+uint64_t ValueXH64FixedWidth(const Value &constant) {
 	T val = constant.GetValue<T>();
 	return duckdb_zstd::XXH64(&val, sizeof(val), 0);
 }

package/src/duckdb/extension/parquet/parquet_timestamp.cpp

@@ -1,9 +1,11 @@
 #include "parquet_timestamp.hpp"
 
 #include "duckdb.hpp"
+#ifndef DUCKDB_AMALGAMATION
 #include "duckdb/common/types/date.hpp"
 #include "duckdb/common/types/time.hpp"
 #include "duckdb/common/types/timestamp.hpp"
+#endif
 
 namespace duckdb {
 
@@ -113,7 +115,7 @@ static T ParquetWrapTime(const T &raw, const T day) {
 	return modulus + (modulus < 0) * day;
 }
 
-dtime_t ParquetMsIntToTime(const int32_t &raw_millis) {
+dtime_t ParquetIntToTimeMs(const int32_t &raw_millis) {
 	return Time::FromTimeMs(raw_millis);
 }
 
@@ -121,22 +123,10 @@ dtime_t ParquetIntToTime(const int64_t &raw_micros) {
 	return dtime_t(raw_micros);
 }
 
-dtime_t ParquetNsIntToTime(const int64_t &raw_nanos) {
+dtime_t ParquetIntToTimeNs(const int64_t &raw_nanos) {
 	return Time::FromTimeNs(raw_nanos);
 }
 
-dtime_ns_t ParquetMsIntToTimeNs(const int32_t &raw_millis) {
-	return dtime_ns_t(Interval::NANOS_PER_MSEC * raw_millis);
-}
-
-dtime_ns_t ParquetUsIntToTimeNs(const int64_t &raw_micros) {
-	return dtime_ns_t(raw_micros * Interval::NANOS_PER_MICRO);
-}
-
-dtime_ns_t ParquetIntToTimeNs(const int64_t &raw_nanos) {
-	return dtime_ns_t(raw_nanos);
-}
-
 dtime_tz_t ParquetIntToTimeMsTZ(const int32_t &raw_millis) {
 	const int32_t MSECS_PER_DAY = Interval::MSECS_PER_SEC * Interval::SECS_PER_DAY;
 	const auto millis = ParquetWrapTime(raw_millis, MSECS_PER_DAY);