duckdb 0.9.3-dev2.0 → 0.10.0

package/src/duckdb/extension/parquet/parquet_metadata.cpp

@@ -24,6 +24,8 @@ public:
 	}
 };
 
+enum class ParquetMetadataOperatorType { META_DATA, SCHEMA, KEY_VALUE_META_DATA, FILE_META_DATA };
+
 struct ParquetMetaDataOperatorData : public GlobalTableFunctionState {
 	explicit ParquetMetaDataOperatorData(ClientContext &context, const vector<LogicalType> &types)
 	    : collection(context, types) {
@@ -36,9 +38,13 @@ struct ParquetMetaDataOperatorData : public GlobalTableFunctionState {
 public:
 	static void BindMetaData(vector<LogicalType> &return_types, vector<string> &names);
 	static void BindSchema(vector<LogicalType> &return_types, vector<string> &names);
+	static void BindKeyValueMetaData(vector<LogicalType> &return_types, vector<string> &names);
+	static void BindFileMetaData(vector<LogicalType> &return_types, vector<string> &names);
 
-	void LoadFileMetaData(ClientContext &context, const vector<LogicalType> &return_types, const string &file_path);
+	void LoadRowGroupMetadata(ClientContext &context, const vector<LogicalType> &return_types, const string &file_path);
 	void LoadSchemaData(ClientContext &context, const vector<LogicalType> &return_types, const string &file_path);
+	void LoadKeyValueMetaData(ClientContext &context, const vector<LogicalType> &return_types, const string &file_path);
+	void LoadFileMetaData(ClientContext &context, const vector<LogicalType> &return_types, const string &file_path);
 };
 
 template <class T>
@@ -63,6 +69,13 @@ Value ParquetElementString(T &&value, bool is_set) {
 	return Value(ConvertParquetElementToString(value));
 }
 
+Value ParquetElementStringVal(const string &value, bool is_set) {
+	if (!is_set) {
+		return Value();
+	}
+	return Value(value);
+}
+
 template <class T>
 Value ParquetElementInteger(T &&value, bool is_iset) {
 	if (!is_iset) {
@@ -79,6 +92,9 @@ Value ParquetElementBigint(T &&value, bool is_iset) {
 	return Value::BIGINT(value);
 }
 
+//===--------------------------------------------------------------------===//
+// Row Group Meta Data
+//===--------------------------------------------------------------------===//
 void ParquetMetaDataOperatorData::BindMetaData(vector<LogicalType> &return_types, vector<string> &names) {
 	names.emplace_back("file_name");
 	return_types.emplace_back(LogicalType::VARCHAR);
@@ -148,6 +164,9 @@ void ParquetMetaDataOperatorData::BindMetaData(vector<LogicalType> &return_types
 
 	names.emplace_back("total_uncompressed_size");
 	return_types.emplace_back(LogicalType::BIGINT);
+
+	names.emplace_back("key_value_metadata");
+	return_types.emplace_back(LogicalType::MAP(LogicalType::BLOB, LogicalType::BLOB));
 }
 
 Value ConvertParquetStats(const LogicalType &type, const duckdb_parquet::format::SchemaElement &schema_ele,
@@ -158,8 +177,8 @@ Value ConvertParquetStats(const LogicalType &type, const duckdb_parquet::format:
 	return ParquetStatisticsUtils::ConvertValue(type, schema_ele, stats).DefaultCastAs(LogicalType::VARCHAR);
 }
 
-void ParquetMetaDataOperatorData::LoadFileMetaData(ClientContext &context, const vector<LogicalType> &return_types,
-                                                   const string &file_path) {
+void ParquetMetaDataOperatorData::LoadRowGroupMetadata(ClientContext &context, const vector<LogicalType> &return_types,
+                                                       const string &file_path) {
 	collection.Reset();
 	ParquetOptions parquet_options(context);
 	auto reader = make_uniq<ParquetReader>(context, file_path, parquet_options);
@@ -272,6 +291,16 @@ void ParquetMetaDataOperatorData::LoadFileMetaData(ClientContext &context, const
 			// total_uncompressed_size, LogicalType::BIGINT
 			current_chunk.SetValue(22, count, Value::BIGINT(col_meta.total_uncompressed_size));
 
+			// key_value_metadata, LogicalType::MAP(LogicalType::BLOB, LogicalType::BLOB)
+			vector<Value> map_keys, map_values;
+			for (auto &entry : col_meta.key_value_metadata) {
+				map_keys.push_back(Value::BLOB_RAW(entry.key));
+				map_values.push_back(Value::BLOB_RAW(entry.value));
+			}
+			current_chunk.SetValue(
+			    23, count,
+			    Value::MAP(LogicalType::BLOB, LogicalType::BLOB, std::move(map_keys), std::move(map_values)));
+
 			count++;
 			if (count >= STANDARD_VECTOR_SIZE) {
 				current_chunk.SetCardinality(count);
@@ -288,6 +317,9 @@ void ParquetMetaDataOperatorData::LoadFileMetaData(ClientContext &context, const
 	collection.InitializeScan(scan_state);
 }
 
+//===--------------------------------------------------------------------===//
+// Schema Data
+//===--------------------------------------------------------------------===//
 void ParquetMetaDataOperatorData::BindSchema(vector<LogicalType> &return_types, vector<string> &names) {
 	names.emplace_back("file_name");
 	return_types.emplace_back(LogicalType::VARCHAR);
@@ -429,13 +461,129 @@ void ParquetMetaDataOperatorData::LoadSchemaData(ClientContext &context, const v
 	collection.InitializeScan(scan_state);
 }
 
-template <bool SCHEMA>
+//===--------------------------------------------------------------------===//
+// KV Meta Data
+//===--------------------------------------------------------------------===//
+void ParquetMetaDataOperatorData::BindKeyValueMetaData(vector<LogicalType> &return_types, vector<string> &names) {
+	names.emplace_back("file_name");
+	return_types.emplace_back(LogicalType::VARCHAR);
+
+	names.emplace_back("key");
+	return_types.emplace_back(LogicalType::BLOB);
+
+	names.emplace_back("value");
+	return_types.emplace_back(LogicalType::BLOB);
+}
+
+void ParquetMetaDataOperatorData::LoadKeyValueMetaData(ClientContext &context, const vector<LogicalType> &return_types,
+                                                       const string &file_path) {
+	collection.Reset();
+	ParquetOptions parquet_options(context);
+	auto reader = make_uniq<ParquetReader>(context, file_path, parquet_options);
+	idx_t count = 0;
+	DataChunk current_chunk;
+	current_chunk.Initialize(context, return_types);
+	auto meta_data = reader->GetFileMetadata();
+
+	for (idx_t col_idx = 0; col_idx < meta_data->key_value_metadata.size(); col_idx++) {
+		auto &entry = meta_data->key_value_metadata[col_idx];
+
+		current_chunk.SetValue(0, count, Value(file_path));
+		current_chunk.SetValue(1, count, Value::BLOB_RAW(entry.key));
+		current_chunk.SetValue(2, count, Value::BLOB_RAW(entry.value));
+
+		count++;
+		if (count >= STANDARD_VECTOR_SIZE) {
+			current_chunk.SetCardinality(count);
+			collection.Append(current_chunk);
+
+			count = 0;
+			current_chunk.Reset();
+		}
+	}
+	current_chunk.SetCardinality(count);
+	collection.Append(current_chunk);
+	collection.InitializeScan(scan_state);
+}
+
+//===--------------------------------------------------------------------===//
+// File Meta Data
+//===--------------------------------------------------------------------===//
+void ParquetMetaDataOperatorData::BindFileMetaData(vector<LogicalType> &return_types, vector<string> &names) {
+	names.emplace_back("file_name");
+	return_types.emplace_back(LogicalType::VARCHAR);
+
+	names.emplace_back("created_by");
+	return_types.emplace_back(LogicalType::VARCHAR);
+
+	names.emplace_back("num_rows");
+	return_types.emplace_back(LogicalType::BIGINT);
+
+	names.emplace_back("num_row_groups");
+	return_types.emplace_back(LogicalType::BIGINT);
+
+	names.emplace_back("format_version");
+	return_types.emplace_back(LogicalType::BIGINT);
+
+	names.emplace_back("encryption_algorithm");
+	return_types.emplace_back(LogicalType::VARCHAR);
+
+	names.emplace_back("footer_signing_key_metadata");
+	return_types.emplace_back(LogicalType::VARCHAR);
+}
+
+void ParquetMetaDataOperatorData::LoadFileMetaData(ClientContext &context, const vector<LogicalType> &return_types,
+                                                   const string &file_path) {
+	collection.Reset();
+	ParquetOptions parquet_options(context);
+	auto reader = make_uniq<ParquetReader>(context, file_path, parquet_options);
+	DataChunk current_chunk;
+	current_chunk.Initialize(context, return_types);
+	auto meta_data = reader->GetFileMetadata();
+
+	//	file_name
+	current_chunk.SetValue(0, 0, Value(file_path));
+	//	created_by
+	current_chunk.SetValue(1, 0, ParquetElementStringVal(meta_data->created_by, meta_data->__isset.created_by));
+	//	num_rows
+	current_chunk.SetValue(2, 0, Value::BIGINT(meta_data->num_rows));
+	//	num_row_groups
+	current_chunk.SetValue(3, 0, Value::BIGINT(meta_data->row_groups.size()));
+	//	format_version
+	current_chunk.SetValue(4, 0, Value::BIGINT(meta_data->version));
+	//	encryption_algorithm
+	current_chunk.SetValue(
+	    5, 0, ParquetElementString(meta_data->encryption_algorithm, meta_data->__isset.encryption_algorithm));
+	//	footer_signing_key_metadata
+	current_chunk.SetValue(6, 0,
+	                       ParquetElementStringVal(meta_data->footer_signing_key_metadata,
+	                                               meta_data->__isset.footer_signing_key_metadata));
+	current_chunk.SetCardinality(1);
+	collection.Append(current_chunk);
+	collection.InitializeScan(scan_state);
+}
+
+//===--------------------------------------------------------------------===//
+// Bind
+//===--------------------------------------------------------------------===//
+template <ParquetMetadataOperatorType TYPE>
 unique_ptr<FunctionData> ParquetMetaDataBind(ClientContext &context, TableFunctionBindInput &input,
                                              vector<LogicalType> &return_types, vector<string> &names) {
-	if (SCHEMA) {
+	switch (TYPE) {
+	case ParquetMetadataOperatorType::SCHEMA:
 		ParquetMetaDataOperatorData::BindSchema(return_types, names);
-	} else {
+		break;
+	case ParquetMetadataOperatorType::META_DATA:
 		ParquetMetaDataOperatorData::BindMetaData(return_types, names);
+		break;
+	case ParquetMetadataOperatorType::KEY_VALUE_META_DATA:
+		ParquetMetaDataOperatorData::BindKeyValueMetaData(return_types, names);
+		break;
+	case ParquetMetadataOperatorType::FILE_META_DATA:
+		ParquetMetaDataOperatorData::BindFileMetaData(return_types, names);
+		break;
+	default:
+		throw InternalException("Unsupported ParquetMetadataOperatorType");
 	}
 
 	auto result = make_uniq<ParquetMetaDataBindData>();
@@ -444,22 +592,33 @@ unique_ptr<FunctionData> ParquetMetaDataBind(ClientContext &context, TableFuncti
 	return std::move(result);
 }
 
-template <bool SCHEMA>
+template <ParquetMetadataOperatorType TYPE>
 unique_ptr<GlobalTableFunctionState> ParquetMetaDataInit(ClientContext &context, TableFunctionInitInput &input) {
 	auto &bind_data = input.bind_data->Cast<ParquetMetaDataBindData>();
 	D_ASSERT(!bind_data.files.empty());
 
 	auto result = make_uniq<ParquetMetaDataOperatorData>(context, bind_data.return_types);
-	if (SCHEMA) {
+	switch (TYPE) {
+	case ParquetMetadataOperatorType::SCHEMA:
 		result->LoadSchemaData(context, bind_data.return_types, bind_data.files[0]);
-	} else {
+		break;
+	case ParquetMetadataOperatorType::META_DATA:
+		result->LoadRowGroupMetadata(context, bind_data.return_types, bind_data.files[0]);
+		break;
+	case ParquetMetadataOperatorType::KEY_VALUE_META_DATA:
+		result->LoadKeyValueMetaData(context, bind_data.return_types, bind_data.files[0]);
+		break;
+	case ParquetMetadataOperatorType::FILE_META_DATA:
 		result->LoadFileMetaData(context, bind_data.return_types, bind_data.files[0]);
+		break;
+	default:
+		throw InternalException("Unsupported ParquetMetadataOperatorType");
 	}
 	result->file_index = 0;
 	return std::move(result);
 }
 
-template <bool SCHEMA>
+template <ParquetMetadataOperatorType TYPE>
 void ParquetMetaDataImplementation(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) {
 	auto &data = data_p.global_state->Cast<ParquetMetaDataOperatorData>();
 	auto &bind_data = data_p.bind_data->Cast<ParquetMetaDataBindData>();
@@ -469,10 +628,21 @@ void ParquetMetaDataImplementation(ClientContext &context, TableFunctionInput &d
 			if (data.file_index + 1 < bind_data.files.size()) {
 				// load the metadata for the next file
 				data.file_index++;
-				if (SCHEMA) {
+				switch (TYPE) {
+				case ParquetMetadataOperatorType::SCHEMA:
 					data.LoadSchemaData(context, bind_data.return_types, bind_data.files[data.file_index]);
-				} else {
+					break;
+				case ParquetMetadataOperatorType::META_DATA:
+					data.LoadRowGroupMetadata(context, bind_data.return_types, bind_data.files[data.file_index]);
+					break;
+				case ParquetMetadataOperatorType::KEY_VALUE_META_DATA:
+					data.LoadKeyValueMetaData(context, bind_data.return_types, bind_data.files[data.file_index]);
+					break;
+				case ParquetMetadataOperatorType::FILE_META_DATA:
 					data.LoadFileMetaData(context, bind_data.return_types, bind_data.files[data.file_index]);
+					break;
+				default:
+					throw InternalException("Unsupported ParquetMetadataOperatorType");
 				}
 				continue;
 			} else {
@@ -487,13 +657,31 @@ void ParquetMetaDataImplementation(ClientContext &context, TableFunctionInput &d
 }
 
 ParquetMetaDataFunction::ParquetMetaDataFunction()
-    : TableFunction("parquet_metadata", {LogicalType::VARCHAR}, ParquetMetaDataImplementation<false>,
-                    ParquetMetaDataBind<false>, ParquetMetaDataInit<false>) {
+    : TableFunction("parquet_metadata", {LogicalType::VARCHAR},
+                    ParquetMetaDataImplementation<ParquetMetadataOperatorType::META_DATA>,
+                    ParquetMetaDataBind<ParquetMetadataOperatorType::META_DATA>,
+                    ParquetMetaDataInit<ParquetMetadataOperatorType::META_DATA>) {
 }
 
 ParquetSchemaFunction::ParquetSchemaFunction()
-    : TableFunction("parquet_schema", {LogicalType::VARCHAR}, ParquetMetaDataImplementation<true>,
-                    ParquetMetaDataBind<true>, ParquetMetaDataInit<true>) {
+    : TableFunction("parquet_schema", {LogicalType::VARCHAR},
+                    ParquetMetaDataImplementation<ParquetMetadataOperatorType::SCHEMA>,
+                    ParquetMetaDataBind<ParquetMetadataOperatorType::SCHEMA>,
+                    ParquetMetaDataInit<ParquetMetadataOperatorType::SCHEMA>) {
+}
+
+ParquetKeyValueMetadataFunction::ParquetKeyValueMetadataFunction()
+    : TableFunction("parquet_kv_metadata", {LogicalType::VARCHAR},
+                    ParquetMetaDataImplementation<ParquetMetadataOperatorType::KEY_VALUE_META_DATA>,
+                    ParquetMetaDataBind<ParquetMetadataOperatorType::KEY_VALUE_META_DATA>,
+                    ParquetMetaDataInit<ParquetMetadataOperatorType::KEY_VALUE_META_DATA>) {
+}
+
+ParquetFileMetadataFunction::ParquetFileMetadataFunction()
+    : TableFunction("parquet_file_metadata", {LogicalType::VARCHAR},
+                    ParquetMetaDataImplementation<ParquetMetadataOperatorType::FILE_META_DATA>,
+                    ParquetMetaDataBind<ParquetMetadataOperatorType::FILE_META_DATA>,
+                    ParquetMetaDataInit<ParquetMetadataOperatorType::FILE_META_DATA>) {
 }
 
 } // namespace duckdb

package/src/duckdb/extension/parquet/parquet_reader.cpp

@@ -1,47 +1,46 @@
 #include "parquet_reader.hpp"
-#include "parquet_timestamp.hpp"
-#include "parquet_statistics.hpp"
-#include "column_reader.hpp"
 
 #include "boolean_column_reader.hpp"
-#include "row_number_column_reader.hpp"
-#include "cast_column_reader.hpp"
 #include "callback_column_reader.hpp"
+#include "cast_column_reader.hpp"
+#include "column_reader.hpp"
+#include "duckdb.hpp"
 #include "list_column_reader.hpp"
+#include "parquet_crypto.hpp"
+#include "parquet_file_metadata_cache.hpp"
+#include "parquet_statistics.hpp"
+#include "parquet_timestamp.hpp"
+#include "row_number_column_reader.hpp"
 #include "string_column_reader.hpp"
 #include "struct_column_reader.hpp"
 #include "templated_column_reader.hpp"
-
 #include "thrift_tools.hpp"
-
-#include "parquet_file_metadata_cache.hpp"
-
-#include "duckdb.hpp"
 #ifndef DUCKDB_AMALGAMATION
-#include "duckdb/planner/table_filter.hpp"
-#include "duckdb/planner/filter/constant_filter.hpp"
-#include "duckdb/planner/filter/null_filter.hpp"
-#include "duckdb/planner/filter/conjunction_filter.hpp"
 #include "duckdb/common/file_system.hpp"
+#include "duckdb/common/hive_partitioning.hpp"
+#include "duckdb/common/pair.hpp"
 #include "duckdb/common/string_util.hpp"
 #include "duckdb/common/types/date.hpp"
-#include "duckdb/common/pair.hpp"
-#include "duckdb/common/hive_partitioning.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
-
+#include "duckdb/planner/filter/conjunction_filter.hpp"
+#include "duckdb/planner/filter/constant_filter.hpp"
+#include "duckdb/planner/filter/null_filter.hpp"
+#include "duckdb/planner/filter/struct_filter.hpp"
+#include "duckdb/planner/table_filter.hpp"
 #include "duckdb/storage/object_cache.hpp"
 #endif
 
-#include <sstream>
 #include <cassert>
 #include <chrono>
 #include <cstring>
+#include <sstream>
 
 namespace duckdb {
 
 using duckdb_parquet::format::ColumnChunk;
 using duckdb_parquet::format::ConvertedType;
 using duckdb_parquet::format::FieldRepetitionType;
+using duckdb_parquet::format::FileCryptoMetaData;
 using duckdb_parquet::format::FileMetaData;
 using ParquetRowGroup = duckdb_parquet::format::RowGroup;
 using duckdb_parquet::format::SchemaElement;
@@ -49,16 +48,18 @@ using duckdb_parquet::format::Statistics;
 using duckdb_parquet::format::Type;
 
 static unique_ptr<duckdb_apache::thrift::protocol::TProtocol>
-CreateThriftProtocol(Allocator &allocator, FileHandle &file_handle, bool prefetch_mode) {
+CreateThriftFileProtocol(Allocator &allocator, FileHandle &file_handle, bool prefetch_mode) {
 	auto transport = make_shared<ThriftFileTransport>(allocator, file_handle, prefetch_mode);
 	return make_uniq<duckdb_apache::thrift::protocol::TCompactProtocolT<ThriftFileTransport>>(std::move(transport));
 }
 
-static shared_ptr<ParquetFileMetadataCache> LoadMetadata(Allocator &allocator, FileHandle &file_handle) {
+static shared_ptr<ParquetFileMetadataCache>
+LoadMetadata(Allocator &allocator, FileHandle &file_handle,
+             const shared_ptr<const ParquetEncryptionConfig> &encryption_config) {
 	auto current_time = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
 
-	auto proto = CreateThriftProtocol(allocator, file_handle, false);
-	auto &transport = reinterpret_cast<ThriftFileTransport &>(*proto->getTransport());
+	auto file_proto = CreateThriftFileProtocol(allocator, file_handle, false);
+	auto &transport = reinterpret_cast<ThriftFileTransport &>(*file_proto->getTransport());
 	auto file_size = transport.GetSize();
 	if (file_size < 12) {
 		throw InvalidInputException("File '%s' too small to be a Parquet file", file_handle.path);
@@ -69,25 +70,48 @@ static shared_ptr<ParquetFileMetadataCache> LoadMetadata(Allocator &allocator, F
 	buf.zero();
 
 	transport.SetLocation(file_size - 8);
-	transport.read((uint8_t *)buf.ptr, 8);
-
-	if (memcmp(buf.ptr + 4, "PAR1", 4) != 0) {
-		if (memcmp(buf.ptr + 4, "PARE", 4) == 0) {
-			throw InvalidInputException("Encrypted Parquet files are not supported for file '%s'", file_handle.path);
+	transport.read(buf.ptr, 8);
+
+	bool footer_encrypted;
+	if (memcmp(buf.ptr + 4, "PAR1", 4) == 0) {
+		footer_encrypted = false;
+		if (encryption_config) {
+			throw InvalidInputException("File '%s' is not encrypted, but 'encryption_config' was set",
+			                            file_handle.path);
 		}
+	} else if (memcmp(buf.ptr + 4, "PARE", 4) == 0) {
+		footer_encrypted = true;
+		if (!encryption_config) {
+			throw InvalidInputException("File '%s' is encrypted, but 'encryption_config' was not set",
+			                            file_handle.path);
+		}
+	} else {
 		throw InvalidInputException("No magic bytes found at end of file '%s'", file_handle.path);
 	}
+
 	// read four-byte footer length from just before the end magic bytes
 	auto footer_len = *reinterpret_cast<uint32_t *>(buf.ptr);
 	if (footer_len == 0 || file_size < 12 + footer_len) {
 		throw InvalidInputException("Footer length error in file '%s'", file_handle.path);
 	}
+
 	auto metadata_pos = file_size - (footer_len + 8);
 	transport.SetLocation(metadata_pos);
 	transport.Prefetch(metadata_pos, footer_len);
 
 	auto metadata = make_uniq<FileMetaData>();
-	metadata->read(proto.get());
+	if (footer_encrypted) {
+		auto crypto_metadata = make_uniq<FileCryptoMetaData>();
+		crypto_metadata->read(file_proto.get());
+		if (crypto_metadata->encryption_algorithm.__isset.AES_GCM_CTR_V1) {
+			throw InvalidInputException("File '%s' is encrypted with AES_GCM_CTR_V1, but only AES_GCM_V1 is supported",
+			                            file_handle.path);
+		}
+		ParquetCrypto::Read(*metadata, *file_proto, encryption_config->GetFooterKey());
+	} else {
+		metadata->read(file_proto.get());
+	}
+
 	return make_shared<ParquetFileMetadataCache>(std::move(metadata), current_time);
 }
 
@@ -180,6 +204,9 @@ LogicalType ParquetReader::DeriveLogicalType(const SchemaElement &s_ele, bool bi
 			if (!s_ele.__isset.precision || !s_ele.__isset.scale) {
 				throw IOException("DECIMAL requires a length and scale specifier!");
 			}
+			if (s_ele.precision > DecimalType::MaxWidth()) {
+				return LogicalType::DOUBLE;
+			}
 			switch (s_ele.type) {
 			case Type::BYTE_ARRAY:
 			case Type::FIXED_LEN_BYTE_ARRAY:
@@ -385,8 +412,11 @@ unique_ptr<ColumnReader> ParquetReader::CreateReader() {
 		root_struct_reader.child_readers[column_idx] = std::move(cast_reader);
 	}
 	if (parquet_options.file_row_number) {
-		root_struct_reader.child_readers.push_back(
-		    make_uniq<RowNumberColumnReader>(*this, LogicalType::BIGINT, SchemaElement(), next_file_idx, 0, 0));
+		file_row_number_idx = root_struct_reader.child_readers.size();
+
+		generated_column_schema.push_back(SchemaElement());
+		root_struct_reader.child_readers.push_back(make_uniq<RowNumberColumnReader>(
+		    *this, LogicalType::BIGINT, generated_column_schema.back(), next_file_idx, 0, 0));
 	}
 
 	return ret;
@@ -396,7 +426,10 @@ void ParquetReader::InitializeSchema() {
 	auto file_meta_data = GetFileMetadata();
 
 	if (file_meta_data->__isset.encryption_algorithm) {
-		throw FormatException("Encrypted Parquet files are not supported");
+		if (file_meta_data->encryption_algorithm.__isset.AES_GCM_CTR_V1) {
+			throw InvalidInputException("File '%s' is encrypted with AES_GCM_CTR_V1, but only AES_GCM_V1 is supported",
+			                            file_name);
+		}
 	}
 	// check if we like this schema
 	if (file_meta_data->schema.size() < 2) {
@@ -429,6 +462,25 @@ ParquetOptions::ParquetOptions(ClientContext &context) {
 	}
 }
 
+ParquetColumnDefinition ParquetColumnDefinition::FromSchemaValue(ClientContext &context, const Value &column_value) {
+	ParquetColumnDefinition result;
+	result.field_id = IntegerValue::Get(StructValue::GetChildren(column_value)[0]);
+
+	const auto &column_def = StructValue::GetChildren(column_value)[1];
+	D_ASSERT(column_def.type().id() == LogicalTypeId::STRUCT);
+
+	const auto children = StructValue::GetChildren(column_def);
+	result.name = StringValue::Get(children[0]);
+	result.type = TransformStringToLogicalType(StringValue::Get(children[1]));
+	string error_message;
+	if (!children[2].TryCastAs(context, result.type, result.default_value, &error_message)) {
+		throw BinderException("Unable to cast Parquet schema default_value \"%s\" to %s", children[2].ToString(),
+		                      result.type.ToString());
+	}
+
+	return result;
+}
+
 ParquetReader::ParquetReader(ClientContext &context_p, string file_name_p, ParquetOptions parquet_options_p)
     : fs(FileSystem::GetFileSystem(context_p)), allocator(BufferAllocator::Get(context_p)),
       parquet_options(std::move(parquet_options_p)) {
@@ -443,12 +495,12 @@ ParquetReader::ParquetReader(ClientContext &context_p, string file_name_p, Parqu
 	// or if this file has cached metadata
 	// or if the cached version already expired
 	if (!ObjectCache::ObjectCacheEnabled(context_p)) {
-		metadata = LoadMetadata(allocator, *file_handle);
+		metadata = LoadMetadata(allocator, *file_handle, parquet_options.encryption_config);
 	} else {
 		auto last_modify_time = fs.GetLastModifiedTime(*file_handle);
 		metadata = ObjectCache::GetObjectCache(context_p).Get<ParquetFileMetadataCache>(file_name);
 		if (!metadata || (last_modify_time + 10 >= metadata->read_time)) {
-			metadata = LoadMetadata(allocator, *file_handle);
+			metadata = LoadMetadata(allocator, *file_handle, parquet_options.encryption_config);
 			ObjectCache::GetObjectCache(context_p).Put(file_name, metadata);
 		}
 	}
@@ -501,6 +553,23 @@ unique_ptr<BaseStatistics> ParquetReader::ReadStatistics(const string &name) {
 	return column_stats;
 }
 
+uint32_t ParquetReader::Read(duckdb_apache::thrift::TBase &object, TProtocol &iprot) {
+	if (parquet_options.encryption_config) {
+		return ParquetCrypto::Read(object, iprot, parquet_options.encryption_config->GetFooterKey());
+	} else {
+		return object.read(&iprot);
+	}
+}
+
+uint32_t ParquetReader::ReadData(duckdb_apache::thrift::protocol::TProtocol &iprot, const data_ptr_t buffer,
+                                 const uint32_t buffer_size) {
+	if (parquet_options.encryption_config) {
+		return ParquetCrypto::ReadData(iprot, buffer, buffer_size, parquet_options.encryption_config->GetFooterKey());
+	} else {
+		return iprot.getTransport()->read(buffer, buffer_size);
+	}
+}
+
 const ParquetRowGroup &ParquetReader::GetGroup(ParquetReaderScanState &state) {
 	auto file_meta_data = GetFileMetadata();
 	D_ASSERT(state.current_group >= 0 && (idx_t)state.current_group < state.group_idx_list.size());
@@ -627,7 +696,7 @@ void ParquetReader::InitializeScan(ParquetReaderScanState &state, vector<idx_t>
 		state.file_handle = fs.OpenFile(file_handle->path, flags);
 	}
 
-	state.thrift_file_proto = CreateThriftProtocol(allocator, *state.file_handle, state.prefetch_mode);
+	state.thrift_file_proto = CreateThriftFileProtocol(allocator, *state.file_handle, state.prefetch_mode);
 	state.root_reader = CreateReader();
 	state.define_buf.resize(allocator, STANDARD_VECTOR_SIZE);
 	state.repeat_buf.resize(allocator, STANDARD_VECTOR_SIZE);
@@ -806,6 +875,11 @@ static void ApplyFilter(Vector &v, TableFilter &filter, parquet_filter_t &filter
 	case TableFilterType::IS_NULL:
 		FilterIsNull(v, filter_mask, count);
 		break;
+	case TableFilterType::STRUCT_EXTRACT: {
+		auto &struct_filter = filter.Cast<StructFilter>();
+		auto &child = StructVector::GetEntries(v)[struct_filter.child_idx];
+		ApplyFilter(*child, *struct_filter.child_filter, filter_mask, count);
+	} break;
 	default:
 		D_ASSERT(0);
 		break;