duckdb 1.0.1-dev22.0 → 1.0.1-dev27.0

package/src/duckdb/extension/parquet/parquet_metadata.cpp

@@ -211,19 +211,19 @@ void ParquetMetaDataOperatorData::LoadRowGroupMetadata(ClientContext &context, c
 			current_chunk.SetValue(0, count, file_path);
 
 			// row_group_id, LogicalType::BIGINT
-			current_chunk.SetValue(1, count, Value::BIGINT(row_group_idx));
+			current_chunk.SetValue(1, count, Value::BIGINT(UnsafeNumericCast<int64_t>(row_group_idx)));
 
 			// row_group_num_rows, LogicalType::BIGINT
 			current_chunk.SetValue(2, count, Value::BIGINT(row_group.num_rows));
 
 			// row_group_num_columns, LogicalType::BIGINT
-			current_chunk.SetValue(3, count, Value::BIGINT(row_group.columns.size()));
+			current_chunk.SetValue(3, count, Value::BIGINT(UnsafeNumericCast<int64_t>(row_group.columns.size())));
 
 			// row_group_bytes, LogicalType::BIGINT
 			current_chunk.SetValue(4, count, Value::BIGINT(row_group.total_byte_size));
 
 			// column_id, LogicalType::BIGINT
-			current_chunk.SetValue(5, count, Value::BIGINT(col_idx));
+			current_chunk.SetValue(5, count, Value::BIGINT(UnsafeNumericCast<int64_t>(col_idx)));
 
 			// file_offset, LogicalType::BIGINT
 			current_chunk.SetValue(6, count, ParquetElementBigint(column.file_offset, row_group.__isset.file_offset));
@@ -545,7 +545,7 @@ void ParquetMetaDataOperatorData::LoadFileMetaData(ClientContext &context, const
 	//	num_rows
 	current_chunk.SetValue(2, 0, Value::BIGINT(meta_data->num_rows));
 	//	num_row_groups
-	current_chunk.SetValue(3, 0, Value::BIGINT(meta_data->row_groups.size()));
+	current_chunk.SetValue(3, 0, Value::BIGINT(UnsafeNumericCast<int64_t>(meta_data->row_groups.size())));
 	//	format_version
 	current_chunk.SetValue(4, 0, Value::BIGINT(meta_data->version));
 	//	encryption_algorithm

package/src/duckdb/extension/parquet/parquet_reader.cpp

@@ -5,27 +5,29 @@
 #include "cast_column_reader.hpp"
 #include "column_reader.hpp"
 #include "duckdb.hpp"
+#include "expression_column_reader.hpp"
+#include "geo_parquet.hpp"
 #include "list_column_reader.hpp"
 #include "parquet_crypto.hpp"
 #include "parquet_file_metadata_cache.hpp"
 #include "parquet_statistics.hpp"
 #include "parquet_timestamp.hpp"
+#include "mbedtls_wrapper.hpp"
 #include "row_number_column_reader.hpp"
 #include "string_column_reader.hpp"
 #include "struct_column_reader.hpp"
 #include "templated_column_reader.hpp"
 #include "thrift_tools.hpp"
+#include "duckdb/main/config.hpp"
+
 #ifndef DUCKDB_AMALGAMATION
+#include "duckdb/common/encryption_state.hpp"
 #include "duckdb/common/file_system.hpp"
-#include "duckdb/common/hive_partitioning.hpp"
-#include "duckdb/common/pair.hpp"
 #include "duckdb/common/helper.hpp"
+#include "duckdb/common/hive_partitioning.hpp"
 #include "duckdb/common/string_util.hpp"
-#include "duckdb/common/types/date.hpp"
-#include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/planner/filter/conjunction_filter.hpp"
 #include "duckdb/planner/filter/constant_filter.hpp"
-#include "duckdb/planner/filter/null_filter.hpp"
 #include "duckdb/planner/filter/struct_filter.hpp"
 #include "duckdb/planner/table_filter.hpp"
 #include "duckdb/storage/object_cache.hpp"
@@ -55,8 +57,9 @@ CreateThriftFileProtocol(Allocator &allocator, FileHandle &file_handle, bool pre
 }
 
 static shared_ptr<ParquetFileMetadataCache>
-LoadMetadata(Allocator &allocator, FileHandle &file_handle,
-             const shared_ptr<const ParquetEncryptionConfig> &encryption_config) {
+LoadMetadata(ClientContext &context, Allocator &allocator, FileHandle &file_handle,
+             const shared_ptr<const ParquetEncryptionConfig> &encryption_config,
+             const EncryptionUtil &encryption_util) {
 	auto current_time = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
 
 	auto file_proto = CreateThriftFileProtocol(allocator, file_handle, false);
@@ -108,12 +111,15 @@ LoadMetadata(Allocator &allocator, FileHandle &file_handle,
 			throw InvalidInputException("File '%s' is encrypted with AES_GCM_CTR_V1, but only AES_GCM_V1 is supported",
 			                            file_handle.path);
 		}
-		ParquetCrypto::Read(*metadata, *file_proto, encryption_config->GetFooterKey());
+		ParquetCrypto::Read(*metadata, *file_proto, encryption_config->GetFooterKey(), encryption_util);
 	} else {
 		metadata->read(file_proto.get());
 	}
 
-	return make_shared_ptr<ParquetFileMetadataCache>(std::move(metadata), current_time);
+	// Try to read the GeoParquet metadata (if present)
+	auto geo_metadata = GeoParquetFileMetadata::TryRead(*metadata, context);
+
+	return make_shared_ptr<ParquetFileMetadataCache>(std::move(metadata), current_time, std::move(geo_metadata));
 }
 
 LogicalType ParquetReader::DeriveLogicalType(const SchemaElement &s_ele, bool binary_as_string) {
@@ -129,6 +135,8 @@ LogicalType ParquetReader::DeriveLogicalType(const SchemaElement &s_ele, bool bi
 		} else if (s_ele.logicalType.__isset.TIMESTAMP) {
 			if (s_ele.logicalType.TIMESTAMP.isAdjustedToUTC) {
 				return LogicalType::TIMESTAMP_TZ;
+			} else if (s_ele.logicalType.TIMESTAMP.unit.__isset.NANOS) {
+				return LogicalType::TIMESTAMP_NS;
 			}
 			return LogicalType::TIMESTAMP;
 		} else if (s_ele.logicalType.__isset.TIME) {
@@ -242,7 +250,7 @@ LogicalType ParquetReader::DeriveLogicalType(const SchemaElement &s_ele, bool bi
 		case ConvertedType::INTERVAL:
 			return LogicalType::INTERVAL;
 		case ConvertedType::JSON:
-			return LogicalType::VARCHAR;
+			return LogicalType::JSON();
 		case ConvertedType::NULL_TYPE:
 			return LogicalTypeId::SQLNULL;
 		case ConvertedType::MAP:
@@ -284,8 +292,9 @@ LogicalType ParquetReader::DeriveLogicalType(const SchemaElement &s_ele) {
 	return DeriveLogicalType(s_ele, parquet_options.binary_as_string);
 }
 
-unique_ptr<ColumnReader> ParquetReader::CreateReaderRecursive(idx_t depth, idx_t max_define, idx_t max_repeat,
-                                                              idx_t &next_schema_idx, idx_t &next_file_idx) {
+unique_ptr<ColumnReader> ParquetReader::CreateReaderRecursive(ClientContext &context, idx_t depth, idx_t max_define,
+                                                              idx_t max_repeat, idx_t &next_schema_idx,
+                                                              idx_t &next_file_idx) {
 	auto file_meta_data = GetFileMetadata();
 	D_ASSERT(file_meta_data);
 	D_ASSERT(next_schema_idx < file_meta_data->schema.size());
@@ -302,6 +311,16 @@ unique_ptr<ColumnReader> ParquetReader::CreateReaderRecursive(idx_t depth, idx_t
 	if (repetition_type == FieldRepetitionType::REPEATED) {
 		max_repeat++;
 	}
+
+	// Check for geoparquet spatial types
+	if (depth == 1) {
+		// geoparquet types have to be at the root of the schema, and have to be present in the kv metadata
+		if (metadata->geo_metadata && metadata->geo_metadata->IsGeometryColumn(s_ele.name)) {
+			return metadata->geo_metadata->CreateColumnReader(*this, DeriveLogicalType(s_ele), s_ele, next_file_idx++,
+			                                                  max_define, max_repeat, context);
+		}
+	}
+
 	if (s_ele.__isset.num_children && s_ele.num_children > 0) { // inner node
 		child_list_t<LogicalType> child_types;
 		vector<unique_ptr<ColumnReader>> child_readers;
@@ -313,12 +332,27 @@ unique_ptr<ColumnReader> ParquetReader::CreateReaderRecursive(idx_t depth, idx_t
 			auto &child_ele = file_meta_data->schema[next_schema_idx];
 
 			auto child_reader =
-			    CreateReaderRecursive(depth + 1, max_define, max_repeat, next_schema_idx, next_file_idx);
+			    CreateReaderRecursive(context, depth + 1, max_define, max_repeat, next_schema_idx, next_file_idx);
 			child_types.push_back(make_pair(child_ele.name, child_reader->Type()));
 			child_readers.push_back(std::move(child_reader));
 
 			c_idx++;
 		}
+		// rename child type entries if there are case-insensitive duplicates by appending _1, _2 etc.
+		// behavior consistent with CSV reader fwiw
+		case_insensitive_map_t<idx_t> name_collision_count;
+		// get header names from CSV
+		for (auto &child_type : child_types) {
+			auto col_name = child_type.first;
+			// avoid duplicate header names
+			while (name_collision_count.find(col_name) != name_collision_count.end()) {
+				name_collision_count[col_name] += 1;
+				col_name = col_name + "_" + to_string(name_collision_count[col_name]);
+			}
+			child_type.first = col_name;
+			name_collision_count[col_name] = 0;
+		}
+
 		D_ASSERT(!child_types.empty());
 		unique_ptr<ColumnReader> result;
 		LogicalType result_type;
@@ -387,7 +421,7 @@ unique_ptr<ColumnReader> ParquetReader::CreateReaderRecursive(idx_t depth, idx_t
 }
 
 // TODO we don't need readers for columns we are not going to read ay
-unique_ptr<ColumnReader> ParquetReader::CreateReader() {
+unique_ptr<ColumnReader> ParquetReader::CreateReader(ClientContext &context) {
 	auto file_meta_data = GetFileMetadata();
 	idx_t next_schema_idx = 0;
 	idx_t next_file_idx = 0;
@@ -398,7 +432,7 @@ unique_ptr<ColumnReader> ParquetReader::CreateReader() {
 	if (file_meta_data->schema[0].num_children == 0) {
 		throw IOException("Parquet reader: root schema element has no children");
 	}
-	auto ret = CreateReaderRecursive(0, 0, 0, next_schema_idx, next_file_idx);
+	auto ret = CreateReaderRecursive(context, 0, 0, 0, next_schema_idx, next_file_idx);
 	if (ret->Type().id() != LogicalTypeId::STRUCT) {
 		throw InvalidInputException("Root element of Parquet file must be a struct");
 	}
@@ -425,7 +459,7 @@ unique_ptr<ColumnReader> ParquetReader::CreateReader() {
 	return ret;
 }
 
-void ParquetReader::InitializeSchema() {
+void ParquetReader::InitializeSchema(ClientContext &context) {
 	auto file_meta_data = GetFileMetadata();
 
 	if (file_meta_data->__isset.encryption_algorithm) {
@@ -438,7 +472,7 @@ void ParquetReader::InitializeSchema() {
 	if (file_meta_data->schema.size() < 2) {
 		throw FormatException("Need at least one non-root column in the file");
 	}
-	root_reader = CreateReader();
+	root_reader = CreateReader(context);
 	auto &root_type = root_reader->Type();
 	auto &child_types = StructType::GetChildTypes(root_type);
 	D_ASSERT(root_type.id() == LogicalTypeId::STRUCT);
@@ -484,7 +518,8 @@ ParquetColumnDefinition ParquetColumnDefinition::FromSchemaValue(ClientContext &
 	return result;
 }
 
-ParquetReader::ParquetReader(ClientContext &context_p, string file_name_p, ParquetOptions parquet_options_p)
+ParquetReader::ParquetReader(ClientContext &context_p, string file_name_p, ParquetOptions parquet_options_p,
+                             shared_ptr<ParquetFileMetadataCache> metadata_p)
     : fs(FileSystem::GetFileSystem(context_p)), allocator(BufferAllocator::Get(context_p)),
       parquet_options(std::move(parquet_options_p)) {
 	file_name = std::move(file_name_p);
@@ -494,27 +529,45 @@ ParquetReader::ParquetReader(ClientContext &context_p, string file_name_p, Parqu
 		    "Reading parquet files from a FIFO stream is not supported and cannot be efficiently supported since "
 		    "metadata is located at the end of the file. Write the stream to disk first and read from there instead.");
 	}
+
+	// set pointer to factory method for AES state
+	auto &config = DBConfig::GetConfig(context_p);
+	if (config.encryption_util && parquet_options.debug_use_openssl) {
+		encryption_util = config.encryption_util;
+	} else {
+		encryption_util = make_shared_ptr<duckdb_mbedtls::MbedTlsWrapper::AESGCMStateMBEDTLSFactory>();
+	}
+
 	// If object cached is disabled
 	// or if this file has cached metadata
 	// or if the cached version already expired
-	if (!ObjectCache::ObjectCacheEnabled(context_p)) {
-		metadata = LoadMetadata(allocator, *file_handle, parquet_options.encryption_config);
-	} else {
-		auto last_modify_time = fs.GetLastModifiedTime(*file_handle);
-		metadata = ObjectCache::GetObjectCache(context_p).Get<ParquetFileMetadataCache>(file_name);
-		if (!metadata || (last_modify_time + 10 >= metadata->read_time)) {
-			metadata = LoadMetadata(allocator, *file_handle, parquet_options.encryption_config);
-			ObjectCache::GetObjectCache(context_p).Put(file_name, metadata);
+	if (!metadata_p) {
+		if (!ObjectCache::ObjectCacheEnabled(context_p)) {
+			metadata =
+			    LoadMetadata(context_p, allocator, *file_handle, parquet_options.encryption_config, *encryption_util);
+		} else {
+			auto last_modify_time = fs.GetLastModifiedTime(*file_handle);
+			metadata = ObjectCache::GetObjectCache(context_p).Get<ParquetFileMetadataCache>(file_name);
+			if (!metadata || (last_modify_time + 10 >= metadata->read_time)) {
+				metadata = LoadMetadata(context_p, allocator, *file_handle, parquet_options.encryption_config,
+				                        *encryption_util);
+				ObjectCache::GetObjectCache(context_p).Put(file_name, metadata);
+			}
 		}
+	} else {
+		metadata = std::move(metadata_p);
 	}
-	InitializeSchema();
+	InitializeSchema(context_p);
+}
+
+ParquetUnionData::~ParquetUnionData() {
 }
 
 ParquetReader::ParquetReader(ClientContext &context_p, ParquetOptions parquet_options_p,
                              shared_ptr<ParquetFileMetadataCache> metadata_p)
     : fs(FileSystem::GetFileSystem(context_p)), allocator(BufferAllocator::Get(context_p)),
       metadata(std::move(metadata_p)), parquet_options(std::move(parquet_options_p)) {
-	InitializeSchema();
+	InitializeSchema(context_p);
 }
 
 ParquetReader::~ParquetReader() {
@@ -556,9 +609,16 @@ unique_ptr<BaseStatistics> ParquetReader::ReadStatistics(const string &name) {
 	return column_stats;
 }
 
+unique_ptr<BaseStatistics> ParquetReader::ReadStatistics(ClientContext &context, ParquetOptions parquet_options,
+                                                         shared_ptr<ParquetFileMetadataCache> metadata,
+                                                         const string &name) {
+	ParquetReader reader(context, std::move(parquet_options), std::move(metadata));
+	return reader.ReadStatistics(name);
+}
+
 uint32_t ParquetReader::Read(duckdb_apache::thrift::TBase &object, TProtocol &iprot) {
 	if (parquet_options.encryption_config) {
-		return ParquetCrypto::Read(object, iprot, parquet_options.encryption_config->GetFooterKey());
+		return ParquetCrypto::Read(object, iprot, parquet_options.encryption_config->GetFooterKey(), *encryption_util);
 	} else {
 		return object.read(&iprot);
 	}
@@ -567,7 +627,8 @@ uint32_t ParquetReader::Read(duckdb_apache::thrift::TBase &object, TProtocol &ip
 uint32_t ParquetReader::ReadData(duckdb_apache::thrift::protocol::TProtocol &iprot, const data_ptr_t buffer,
                                  const uint32_t buffer_size) {
 	if (parquet_options.encryption_config) {
-		return ParquetCrypto::ReadData(iprot, buffer, buffer_size, parquet_options.encryption_config->GetFooterKey());
+		return ParquetCrypto::ReadData(iprot, buffer, buffer_size, parquet_options.encryption_config->GetFooterKey(),
+		                               *encryption_util);
 	} else {
 		return iprot.getTransport()->read(buffer, buffer_size);
 	}
@@ -642,6 +703,20 @@ idx_t ParquetReader::GetGroupOffset(ParquetReaderScanState &state) {
 	return min_offset;
 }
 
+static FilterPropagateResult CheckParquetStringFilter(BaseStatistics &stats, const Statistics &pq_col_stats,
+                                                      TableFilter &filter) {
+	if (filter.filter_type == TableFilterType::CONSTANT_COMPARISON) {
+		auto &constant_filter = filter.Cast<ConstantFilter>();
+		auto &min_value = pq_col_stats.min_value;
+		auto &max_value = pq_col_stats.max_value;
+		return StringStats::CheckZonemap(const_data_ptr_cast(min_value.c_str()), min_value.size(),
+		                                 const_data_ptr_cast(max_value.c_str()), max_value.size(),
+		                                 constant_filter.comparison_type, StringValue::Get(constant_filter.constant));
+	} else {
+		return filter.CheckStatistics(stats);
+	}
+}
+
 void ParquetReader::PrepareRowGroupBuffer(ParquetReaderScanState &state, idx_t col_idx) {
 	auto &group = GetGroup(state);
 	auto column_id = reader_data.column_ids[col_idx];
@@ -656,7 +731,36 @@ void ParquetReader::PrepareRowGroupBuffer(ParquetReaderScanState &state, idx_t c
 		if (stats && filter_entry != reader_data.filters->filters.end()) {
 			bool skip_chunk = false;
 			auto &filter = *filter_entry->second;
-			auto prune_result = filter.CheckStatistics(*stats);
+
+			FilterPropagateResult prune_result;
+			if (column_reader->Type().id() == LogicalTypeId::VARCHAR &&
+			    group.columns[column_reader->FileIdx()].meta_data.statistics.__isset.min_value &&
+			    group.columns[column_reader->FileIdx()].meta_data.statistics.__isset.max_value) {
+				// our StringStats only store the first 8 bytes of strings (even if Parquet has longer string stats)
+				// however, when reading remote Parquet files, skipping row groups is really important
+				// here, we implement a special case to check the full length for string filters
+				if (filter.filter_type == TableFilterType::CONJUNCTION_AND) {
+					const auto &and_filter = filter.Cast<ConjunctionAndFilter>();
+					auto and_result = FilterPropagateResult::FILTER_ALWAYS_TRUE;
+					for (auto &child_filter : and_filter.child_filters) {
+						auto child_prune_result = CheckParquetStringFilter(
+						    *stats, group.columns[column_reader->FileIdx()].meta_data.statistics, *child_filter);
+						if (child_prune_result == FilterPropagateResult::FILTER_ALWAYS_FALSE) {
+							and_result = FilterPropagateResult::FILTER_ALWAYS_FALSE;
+							break;
+						} else if (child_prune_result != and_result) {
+							and_result = FilterPropagateResult::NO_PRUNING_POSSIBLE;
+						}
+					}
+					prune_result = and_result;
+				} else {
+					prune_result = CheckParquetStringFilter(
+					    *stats, group.columns[column_reader->FileIdx()].meta_data.statistics, filter);
+				}
+			} else {
+				prune_result = filter.CheckStatistics(*stats);
+			}
+
 			if (prune_result == FilterPropagateResult::FILTER_ALWAYS_FALSE) {
 				skip_chunk = true;
 			}
@@ -680,7 +784,8 @@ idx_t ParquetReader::NumRowGroups() {
 	return GetFileMetadata()->row_groups.size();
 }
 
-void ParquetReader::InitializeScan(ParquetReaderScanState &state, vector<idx_t> groups_to_read) {
+void ParquetReader::InitializeScan(ClientContext &context, ParquetReaderScanState &state,
+                                   vector<idx_t> groups_to_read) {
 	state.current_group = -1;
 	state.finished = false;
 	state.group_offset = 0;
@@ -700,7 +805,7 @@ void ParquetReader::InitializeScan(ParquetReaderScanState &state, vector<idx_t>
 	}
 
 	state.thrift_file_proto = CreateThriftFileProtocol(allocator, *state.file_handle, state.prefetch_mode);
-	state.root_reader = CreateReader();
+	state.root_reader = CreateReader(context);
 	state.define_buf.resize(allocator, STANDARD_VECTOR_SIZE);
 	state.repeat_buf.resize(allocator, STANDARD_VECTOR_SIZE);
 }
@@ -720,7 +825,9 @@ void FilterIsNull(Vector &v, parquet_filter_t &filter_mask, idx_t count) {
 		filter_mask.reset();
 	} else {
 		for (idx_t i = 0; i < count; i++) {
-			filter_mask[i] = filter_mask[i] && !mask.RowIsValid(i);
+			if (filter_mask.test(i)) {
+				filter_mask.set(i, !mask.RowIsValid(i));
+			}
 		}
 	}
 }
@@ -738,7 +845,9 @@ void FilterIsNotNull(Vector &v, parquet_filter_t &filter_mask, idx_t count) {
 	auto &mask = FlatVector::Validity(v);
 	if (!mask.AllValid()) {
 		for (idx_t i = 0; i < count; i++) {
-			filter_mask[i] = filter_mask[i] && mask.RowIsValid(i);
+			if (filter_mask.test(i)) {
+				filter_mask.set(i, mask.RowIsValid(i));
+			}
 		}
 	}
 }
@@ -763,13 +872,15 @@ void TemplatedFilterOperation(Vector &v, T constant, parquet_filter_t &filter_ma
 
 	if (!mask.AllValid()) {
 		for (idx_t i = 0; i < count; i++) {
-			if (mask.RowIsValid(i)) {
-				filter_mask[i] = filter_mask[i] && OP::Operation(v_ptr[i], constant);
+			if (filter_mask.test(i) && mask.RowIsValid(i)) {
+				filter_mask.set(i, OP::Operation(v_ptr[i], constant));
 			}
 		}
 	} else {
 		for (idx_t i = 0; i < count; i++) {
-			filter_mask[i] = filter_mask[i] && OP::Operation(v_ptr[i], constant);
+			if (filter_mask.test(i)) {
+				filter_mask.set(i, OP::Operation(v_ptr[i], constant));
+			}
 		}
 	}
 }
@@ -932,7 +1043,7 @@ bool ParquetReader::ScanInternal(ParquetReaderScanState &state, DataChunk &resul
 
 			uint64_t total_row_group_span = GetGroupSpan(state);
 
-			double scan_percentage = (double)(to_scan_compressed_bytes) / total_row_group_span;
+			double scan_percentage = (double)(to_scan_compressed_bytes) / static_cast<double>(total_row_group_span);
 
 			if (to_scan_compressed_bytes > total_row_group_span) {
 				throw InvalidInputException(
@@ -1049,7 +1160,7 @@ bool ParquetReader::ScanInternal(ParquetReaderScanState &state, DataChunk &resul
 
 		idx_t sel_size = 0;
 		for (idx_t i = 0; i < this_output_chunk_rows; i++) {
-			if (filter_mask[i]) {
+			if (filter_mask.test(i)) {
 				state.sel.set_index(sel_size++, i);
 			}
 		}

package/src/duckdb/extension/parquet/parquet_statistics.cpp

@@ -26,17 +26,17 @@ static unique_ptr<BaseStatistics> CreateNumericStats(const LogicalType &type,
 	// `max_value`. All are optional. such elegance.
 	Value min;
 	Value max;
-	if (parquet_stats.__isset.min) {
-		min = ParquetStatisticsUtils::ConvertValue(type, schema_ele, parquet_stats.min).DefaultCastAs(type);
-	} else if (parquet_stats.__isset.min_value) {
+	if (parquet_stats.__isset.min_value) {
 		min = ParquetStatisticsUtils::ConvertValue(type, schema_ele, parquet_stats.min_value).DefaultCastAs(type);
+	} else if (parquet_stats.__isset.min) {
+		min = ParquetStatisticsUtils::ConvertValue(type, schema_ele, parquet_stats.min).DefaultCastAs(type);
 	} else {
 		min = Value(type);
 	}
-	if (parquet_stats.__isset.max) {
-		max = ParquetStatisticsUtils::ConvertValue(type, schema_ele, parquet_stats.max).DefaultCastAs(type);
-	} else if (parquet_stats.__isset.max_value) {
+	if (parquet_stats.__isset.max_value) {
 		max = ParquetStatisticsUtils::ConvertValue(type, schema_ele, parquet_stats.max_value).DefaultCastAs(type);
+	} else if (parquet_stats.__isset.max) {
+		max = ParquetStatisticsUtils::ConvertValue(type, schema_ele, parquet_stats.max).DefaultCastAs(type);
 	} else {
 		max = Value(type);
 	}
@@ -188,7 +188,9 @@ Value ParquetStatisticsUtils::ConvertValue(const LogicalType &type,
 	}
 	case LogicalTypeId::TIME_TZ: {
 		int64_t val;
-		if (stats.size() == sizeof(int64_t)) {
+		if (stats.size() == sizeof(int32_t)) {
+			val = Load<int32_t>(stats_data);
+		} else if (stats.size() == sizeof(int64_t)) {
 			val = Load<int64_t>(stats_data);
 		} else {
 			throw InternalException("Incorrect stats size for type TIMETZ");
@@ -196,7 +198,7 @@ Value ParquetStatisticsUtils::ConvertValue(const LogicalType &type,
 		if (schema_ele.__isset.logicalType && schema_ele.logicalType.__isset.TIME) {
 			// logical type
 			if (schema_ele.logicalType.TIME.unit.__isset.MILLIS) {
-				return Value::TIMETZ(ParquetIntToTimeMsTZ(val));
+				return Value::TIMETZ(ParquetIntToTimeMsTZ(NumericCast<int32_t>(val)));
 			} else if (schema_ele.logicalType.TIME.unit.__isset.MICROS) {
 				return Value::TIMETZ(ParquetIntToTimeTZ(val));
 			} else if (schema_ele.logicalType.TIME.unit.__isset.NANOS) {
@@ -244,6 +246,38 @@ Value ParquetStatisticsUtils::ConvertValue(const LogicalType &type,
 			return Value::TIMESTAMP(timestamp_value);
 		}
 	}
+	case LogicalTypeId::TIMESTAMP_NS: {
+		timestamp_ns_t timestamp_value;
+		if (schema_ele.type == Type::INT96) {
+			if (stats.size() != sizeof(Int96)) {
+				throw InternalException("Incorrect stats size for type TIMESTAMP_NS");
+			}
+			timestamp_value = ImpalaTimestampToTimestampNS(Load<Int96>(stats_data));
+		} else {
+			D_ASSERT(schema_ele.type == Type::INT64);
+			if (stats.size() != sizeof(int64_t)) {
+				throw InternalException("Incorrect stats size for type TIMESTAMP_NS");
+			}
+			auto val = Load<int64_t>(stats_data);
+			if (schema_ele.__isset.logicalType && schema_ele.logicalType.__isset.TIMESTAMP) {
+				// logical type
+				if (schema_ele.logicalType.TIMESTAMP.unit.__isset.MILLIS) {
+					timestamp_value = ParquetTimestampMsToTimestampNs(val);
+				} else if (schema_ele.logicalType.TIMESTAMP.unit.__isset.NANOS) {
+					timestamp_value = ParquetTimestampNsToTimestampNs(val);
+				} else if (schema_ele.logicalType.TIMESTAMP.unit.__isset.MICROS) {
+					timestamp_value = ParquetTimestampUsToTimestampNs(val);
+				} else {
+					throw InternalException("Timestamp (NS) logicalType is set but unit is unknown");
+				}
+			} else if (schema_ele.converted_type == duckdb_parquet::format::ConvertedType::TIMESTAMP_MILLIS) {
+				timestamp_value = ParquetTimestampMsToTimestampNs(val);
+			} else {
+				timestamp_value = ParquetTimestampUsToTimestampNs(val);
+			}
+		}
+		return Value::TIMESTAMPNS(timestamp_value);
+	}
 	default:
 		throw InternalException("Unsupported type for stats %s", type.ToString());
 	}
@@ -315,21 +349,21 @@ unique_ptr<BaseStatistics> ParquetStatisticsUtils::TransformColumnStatistics(con
 		break;
 	case LogicalTypeId::VARCHAR: {
 		auto string_stats = StringStats::CreateEmpty(type);
-		if (parquet_stats.__isset.min) {
-			StringColumnReader::VerifyString(parquet_stats.min.c_str(), parquet_stats.min.size(), true);
-			StringStats::Update(string_stats, parquet_stats.min);
-		} else if (parquet_stats.__isset.min_value) {
+		if (parquet_stats.__isset.min_value) {
 			StringColumnReader::VerifyString(parquet_stats.min_value.c_str(), parquet_stats.min_value.size(), true);
 			StringStats::Update(string_stats, parquet_stats.min_value);
+		} else if (parquet_stats.__isset.min) {
+			StringColumnReader::VerifyString(parquet_stats.min.c_str(), parquet_stats.min.size(), true);
+			StringStats::Update(string_stats, parquet_stats.min);
 		} else {
 			return nullptr;
 		}
-		if (parquet_stats.__isset.max) {
-			StringColumnReader::VerifyString(parquet_stats.max.c_str(), parquet_stats.max.size(), true);
-			StringStats::Update(string_stats, parquet_stats.max);
-		} else if (parquet_stats.__isset.max_value) {
+		if (parquet_stats.__isset.max_value) {
 			StringColumnReader::VerifyString(parquet_stats.max_value.c_str(), parquet_stats.max_value.size(), true);
 			StringStats::Update(string_stats, parquet_stats.max_value);
+		} else if (parquet_stats.__isset.max) {
+			StringColumnReader::VerifyString(parquet_stats.max.c_str(), parquet_stats.max.size(), true);
+			StringStats::Update(string_stats, parquet_stats.max);
 		} else {
 			return nullptr;
 		}

package/src/duckdb/extension/parquet/parquet_timestamp.cpp

@@ -14,14 +14,31 @@ static constexpr int64_t JULIAN_TO_UNIX_EPOCH_DAYS = 2440588LL;
 static constexpr int64_t MILLISECONDS_PER_DAY = 86400000LL;
 static constexpr int64_t MICROSECONDS_PER_DAY = MILLISECONDS_PER_DAY * 1000LL;
 static constexpr int64_t NANOSECONDS_PER_MICRO = 1000LL;
+static constexpr int64_t NANOSECONDS_PER_DAY = MICROSECONDS_PER_DAY * 1000LL;
+
+static inline int64_t ImpalaTimestampToDays(const Int96 &impala_timestamp) {
+	return impala_timestamp.value[2] - JULIAN_TO_UNIX_EPOCH_DAYS;
+}
 
 static int64_t ImpalaTimestampToMicroseconds(const Int96 &impala_timestamp) {
-	int64_t days_since_epoch = impala_timestamp.value[2] - JULIAN_TO_UNIX_EPOCH_DAYS;
+	int64_t days_since_epoch = ImpalaTimestampToDays(impala_timestamp);
 	auto nanoseconds = Load<int64_t>(const_data_ptr_cast(impala_timestamp.value));
 	auto microseconds = nanoseconds / NANOSECONDS_PER_MICRO;
 	return days_since_epoch * MICROSECONDS_PER_DAY + microseconds;
 }
 
+static int64_t ImpalaTimestampToNanoseconds(const Int96 &impala_timestamp) {
+	int64_t days_since_epoch = ImpalaTimestampToDays(impala_timestamp);
+	auto nanoseconds = Load<int64_t>(const_data_ptr_cast(impala_timestamp.value));
+	return days_since_epoch * NANOSECONDS_PER_DAY + nanoseconds;
+}
+
+timestamp_ns_t ImpalaTimestampToTimestampNS(const Int96 &raw_ts) {
+	timestamp_ns_t result;
+	result.value = ImpalaTimestampToNanoseconds(raw_ts);
+	return result;
+}
+
 timestamp_t ImpalaTimestampToTimestamp(const Int96 &raw_ts) {
 	auto impala_us = ImpalaTimestampToMicroseconds(raw_ts);
 	return Timestamp::FromEpochMicroSeconds(impala_us);
@@ -52,6 +69,30 @@ timestamp_t ParquetTimestampMsToTimestamp(const int64_t &raw_ts) {
 	return Timestamp::FromEpochMs(raw_ts);
 }
 
+timestamp_ns_t ParquetTimestampMsToTimestampNs(const int64_t &raw_ms) {
+	timestamp_ns_t input;
+	input.value = raw_ms;
+	if (!Timestamp::IsFinite(input)) {
+		return input;
+	}
+	return Timestamp::TimestampNsFromEpochMillis(raw_ms);
+}
+
+timestamp_ns_t ParquetTimestampUsToTimestampNs(const int64_t &raw_us) {
+	timestamp_ns_t input;
+	input.value = raw_us;
+	if (!Timestamp::IsFinite(input)) {
+		return input;
+	}
+	return Timestamp::TimestampNsFromEpochMicros(raw_us);
+}
+
+timestamp_ns_t ParquetTimestampNsToTimestampNs(const int64_t &raw_ns) {
+	timestamp_ns_t result;
+	result.value = raw_ns;
+	return result;
+}
+
 timestamp_t ParquetTimestampNsToTimestamp(const int64_t &raw_ts) {
 	timestamp_t input(raw_ts);
 	if (!Timestamp::IsFinite(input)) {