duckdb 1.1.3 → 1.1.4-dev11.0

package/src/duckdb/extension/parquet/parquet_extension.cpp

@@ -5,6 +5,7 @@
 #include "cast_column_reader.hpp"
 #include "duckdb.hpp"
 #include "duckdb/parser/expression/positional_reference_expression.hpp"
+#include "duckdb/parser/expression/constant_expression.hpp"
 #include "duckdb/parser/query_node/select_node.hpp"
 #include "duckdb/parser/tableref/subqueryref.hpp"
 #include "duckdb/planner/operator/logical_projection.hpp"
@@ -60,6 +61,7 @@ struct ParquetReadBindData : public TableFunctionData {
 	atomic<idx_t> chunk_count;
 	vector<string> names;
 	vector<LogicalType> types;
+	vector<MultiFileReaderColumnDefinition> columns;
 	//! Table column names - set when using COPY tbl FROM file.parquet
 	vector<string> table_columns;
 
@@ -165,7 +167,7 @@ struct ParquetReadGlobalState : public GlobalTableFunctionState {
 	idx_t max_threads;
 	vector<idx_t> projection_ids;
 	vector<LogicalType> scanned_types;
-	vector<column_t> column_ids;
+	vector<ColumnIndex> column_indexes;
 	optional_ptr<TableFilterSet> filters;
 
 	idx_t MaxThreads() const override {
@@ -180,27 +182,27 @@ struct ParquetReadGlobalState : public GlobalTableFunctionState {
 struct ParquetWriteBindData : public TableFunctionData {
 	vector<LogicalType> sql_types;
 	vector<string> column_names;
-	duckdb_parquet::format::CompressionCodec::type codec = duckdb_parquet::format::CompressionCodec::SNAPPY;
+	duckdb_parquet::CompressionCodec::type codec = duckdb_parquet::CompressionCodec::SNAPPY;
 	vector<pair<string, string>> kv_metadata;
-	idx_t row_group_size = Storage::ROW_GROUP_SIZE;
-
-	//! If row_group_size_bytes is not set, we default to row_group_size * BYTES_PER_ROW
-	static constexpr const idx_t BYTES_PER_ROW = 1024;
-	idx_t row_group_size_bytes;
+	idx_t row_group_size = DEFAULT_ROW_GROUP_SIZE;
+	idx_t row_group_size_bytes = NumericLimits<idx_t>::Maximum();
 
 	//! How/Whether to encrypt the data
 	shared_ptr<ParquetEncryptionConfig> encryption_config;
 	bool debug_use_openssl = true;
 
-	//! Dictionary compression is applied only if the compression ratio exceeds this threshold
-	double dictionary_compression_ratio_threshold = 1.0;
+	//! After how many distinct values should we abandon dictionary compression and bloom filters?
+	idx_t dictionary_size_limit = row_group_size / 100;
+
+	//! What false positive rate are we willing to accept for bloom filters
+	double bloom_filter_false_positive_ratio = 0.01;
 
 	//! After how many row groups to rotate to a new file
 	optional_idx row_groups_per_file;
 
 	ChildFieldIDs field_ids;
 	//! The compression level, higher value is more
-	optional_idx compression_level;
+	int64_t compression_level = ZStdFileSystem::DefaultCompressionLevel();
 };
 
 struct ParquetWriteGlobalState : public GlobalFunctionData {
@@ -209,7 +211,7 @@ struct ParquetWriteGlobalState : public GlobalFunctionData {
 
 struct ParquetWriteLocalState : public LocalFunctionData {
 	explicit ParquetWriteLocalState(ClientContext &context, const vector<LogicalType> &types)
-	    : buffer(context, types, ColumnDataAllocatorType::HYBRID) {
+	    : buffer(BufferAllocator::Get(context), types) {
 		buffer.InitializeAppend(append_state);
 	}
 
@@ -263,15 +265,47 @@ static MultiFileReaderBindData BindSchema(ClientContext &context, vector<Logical
 
 	vector<string> schema_col_names;
 	vector<LogicalType> schema_col_types;
+	MultiFileReaderBindData bind_data;
 	schema_col_names.reserve(options.schema.size());
 	schema_col_types.reserve(options.schema.size());
-	for (const auto &column : options.schema) {
+	bool match_by_field_id;
+	if (!options.schema.empty()) {
+		auto &column = options.schema[0];
+		if (column.identifier.type().id() == LogicalTypeId::INTEGER) {
+			match_by_field_id = true;
+		} else {
+			match_by_field_id = false;
+		}
+	} else {
+		match_by_field_id = false;
+	}
+
+	for (idx_t i = 0; i < options.schema.size(); i++) {
+		const auto &column = options.schema[i];
 		schema_col_names.push_back(column.name);
 		schema_col_types.push_back(column.type);
+
+		auto res = MultiFileReaderColumnDefinition(column.name, column.type);
+		res.identifier = column.identifier;
+#ifdef DEBUG
+		if (match_by_field_id) {
+			D_ASSERT(res.identifier.type().id() == LogicalTypeId::INTEGER);
+		} else {
+			D_ASSERT(res.identifier.type().id() == LogicalTypeId::VARCHAR);
+		}
+#endif
+
+		res.default_expression = make_uniq<ConstantExpression>(column.default_value);
+		bind_data.schema.emplace_back(std::move(res));
+	}
+
+	if (match_by_field_id) {
+		bind_data.mapping = MultiFileReaderColumnMappingMode::BY_FIELD_ID;
+	} else {
+		bind_data.mapping = MultiFileReaderColumnMappingMode::BY_NAME;
 	}
 
 	// perform the binding on the obtained set of names + types
-	MultiFileReaderBindData bind_data;
 	result.multi_file_reader->BindOptions(options.file_options, *result.file_list, schema_col_types, schema_col_names,
 	                                      bind_data);
 
@@ -285,7 +319,7 @@ static MultiFileReaderBindData BindSchema(ClientContext &context, vector<Logical
 }
 
 static void InitializeParquetReader(ParquetReader &reader, const ParquetReadBindData &bind_data,
-                                    const vector<column_t> &global_column_ids,
+                                    const vector<ColumnIndex> &global_column_ids,
                                     optional_ptr<TableFilterSet> table_filters, ClientContext &context,
                                     optional_idx file_idx, optional_ptr<MultiFileReaderGlobalState> reader_state) {
 	auto &parquet_options = bind_data.parquet_options;
@@ -295,79 +329,13 @@ static void InitializeParquetReader(ParquetReader &reader, const ParquetReadBind
 	// Mark the file in the file list we are scanning here
 	reader_data.file_list_idx = file_idx;
 
-	if (bind_data.parquet_options.schema.empty()) {
-		bind_data.multi_file_reader->InitializeReader(
-		    reader, parquet_options.file_options, bind_data.reader_bind, bind_data.types, bind_data.names,
-		    global_column_ids, table_filters, bind_data.file_list->GetFirstFile(), context, reader_state);
-		return;
-	}
-
-	// a fixed schema was supplied, initialize the MultiFileReader settings here so we can read using the schema
-
-	// this deals with hive partitioning and filename=true
-	bind_data.multi_file_reader->FinalizeBind(parquet_options.file_options, bind_data.reader_bind, reader.GetFileName(),
-	                                          reader.GetNames(), bind_data.types, bind_data.names, global_column_ids,
-	                                          reader_data, context, reader_state);
-
-	// create a mapping from field id to column index in file
-	unordered_map<uint32_t, idx_t> field_id_to_column_index;
-	auto &column_readers = reader.root_reader->Cast<StructColumnReader>().child_readers;
-	for (idx_t column_index = 0; column_index < column_readers.size(); column_index++) {
-		auto &column_schema = column_readers[column_index]->Schema();
-		if (column_schema.__isset.field_id) {
-			field_id_to_column_index[column_schema.field_id] = column_index;
-		}
-	}
-
-	// loop through the schema definition
-	for (idx_t i = 0; i < global_column_ids.size(); i++) {
-		auto global_column_index = global_column_ids[i];
-
-		// check if this is a constant column
-		bool constant = false;
-		for (auto &entry : reader_data.constant_map) {
-			if (entry.column_id == i) {
-				constant = true;
-				break;
-			}
-		}
-		if (constant) {
-			// this column is constant for this file
-			continue;
-		}
-
-		// Handle any generate columns that are not in the schema (currently only file_row_number)
-		if (global_column_index >= parquet_options.schema.size()) {
-			if (bind_data.reader_bind.file_row_number_idx == global_column_index) {
-				reader_data.column_mapping.push_back(i);
-				reader_data.column_ids.push_back(reader.file_row_number_idx);
-			}
-			continue;
-		}
-
-		const auto &column_definition = parquet_options.schema[global_column_index];
-		auto it = field_id_to_column_index.find(column_definition.field_id);
-		if (it == field_id_to_column_index.end()) {
-			// field id not present in file, use default value
-			reader_data.constant_map.emplace_back(i, column_definition.default_value);
-			continue;
-		}
-
-		const auto &local_column_index = it->second;
-		auto &column_reader = column_readers[local_column_index];
-		if (column_reader->Type() != column_definition.type) {
-			// differing types, wrap in a cast column reader
-			reader_data.cast_map[local_column_index] = column_definition.type;
-		}
-
-		reader_data.column_mapping.push_back(i);
-		reader_data.column_ids.push_back(local_column_index);
-	}
-	reader_data.empty_columns = reader_data.column_ids.empty();
-
-	// Finally, initialize the filters
-	bind_data.multi_file_reader->CreateFilterMap(bind_data.types, table_filters, reader_data, reader_state);
-	reader_data.filters = table_filters;
+	// 'reader_bind.schema' could be set explicitly by:
+	// 1. The MultiFileReader::Bind call
+	// 2. The 'schema' parquet option
+	auto &global_columns = bind_data.reader_bind.schema.empty() ? bind_data.columns : bind_data.reader_bind.schema;
+	bind_data.multi_file_reader->InitializeReader(reader, parquet_options.file_options, bind_data.reader_bind,
+	                                              global_columns, global_column_ids, table_filters,
+	                                              bind_data.file_list->GetFirstFile(), context, reader_state);
 }
 
 static bool GetBooleanArgument(const pair<string, vector<Value>> &option) {
@@ -383,6 +351,11 @@ static bool GetBooleanArgument(const pair<string, vector<Value>> &option) {
 	return BooleanValue::Get(boolean_value);
 }
 
+TablePartitionInfo ParquetGetPartitionInfo(ClientContext &context, TableFunctionPartitionInput &input) {
+	auto &parquet_bind = input.bind_data->Cast<ParquetReadBindData>();
+	return parquet_bind.multi_file_reader->GetPartitionInfo(context, parquet_bind.reader_bind, input);
+}
+
 class ParquetScanFunction {
 public:
 	static TableFunctionSet GetFunctionSet() {
@@ -396,12 +369,9 @@ public:
 		table_function.named_parameters["debug_use_openssl"] = LogicalType::BOOLEAN;
 		table_function.named_parameters["compression"] = LogicalType::VARCHAR;
 		table_function.named_parameters["explicit_cardinality"] = LogicalType::UBIGINT;
-		table_function.named_parameters["schema"] =
-		    LogicalType::MAP(LogicalType::INTEGER, LogicalType::STRUCT({{{"name", LogicalType::VARCHAR},
-		                                                                 {"type", LogicalType::VARCHAR},
-		                                                                 {"default_value", LogicalType::VARCHAR}}}));
+		table_function.named_parameters["schema"] = LogicalTypeId::ANY;
 		table_function.named_parameters["encryption_config"] = LogicalTypeId::ANY;
-		table_function.get_batch_index = ParquetScanGetBatchIndex;
+		table_function.get_partition_data = ParquetScanGetPartitionData;
 		table_function.serialize = ParquetScanSerialize;
 		table_function.deserialize = ParquetScanDeserialize;
 		table_function.get_bind_info = ParquetGetBindInfo;
@@ -409,6 +379,7 @@ public:
 		table_function.filter_pushdown = true;
 		table_function.filter_prune = true;
 		table_function.pushdown_complex_filter = ParquetComplexFilterPushdown;
+		table_function.get_partition_info = ParquetGetPartitionInfo;
 
 		MultiFileReader::AddParameters(table_function);
 
@@ -461,66 +432,23 @@ public:
 		}
 
 		// NOTE: we do not want to parse the Parquet metadata for the sole purpose of getting column statistics
-
-		auto &config = DBConfig::GetConfig(context);
-
-		if (bind_data.file_list->GetExpandResult() != FileExpandResult::MULTIPLE_FILES) {
-			if (bind_data.initial_reader) {
-				// most common path, scanning single parquet file
-				return bind_data.initial_reader->ReadStatistics(bind_data.names[column_index]);
-			} else if (!config.options.object_cache_enable) {
-				// our initial reader was reset
-				return nullptr;
-			}
-		} else if (config.options.object_cache_enable) {
-			// multiple files, object cache enabled: merge statistics
-			unique_ptr<BaseStatistics> overall_stats;
-
-			auto &cache = ObjectCache::GetObjectCache(context);
-			// for more than one file, we could be lucky and metadata for *every* file is in the object cache (if
-			// enabled at all)
-			FileSystem &fs = FileSystem::GetFileSystem(context);
-
-			for (const auto &file_name : bind_data.file_list->Files()) {
-				auto metadata = cache.Get<ParquetFileMetadataCache>(file_name);
-				if (!metadata) {
-					// missing metadata entry in cache, no usable stats
-					return nullptr;
-				}
-				if (!fs.IsRemoteFile(file_name)) {
-					auto handle = fs.OpenFile(file_name, FileFlags::FILE_FLAGS_READ);
-					// we need to check if the metadata cache entries are current
-					if (fs.GetLastModifiedTime(*handle) >= metadata->read_time) {
-						// missing or invalid metadata entry in cache, no usable stats overall
-						return nullptr;
-					}
-				} else {
-					// for remote files we just avoid reading stats entirely
-					return nullptr;
-				}
-				// get and merge stats for file
-				auto file_stats = ParquetReader::ReadStatistics(context, bind_data.parquet_options, metadata,
-				                                                bind_data.names[column_index]);
-				if (!file_stats) {
-					return nullptr;
-				}
-				if (overall_stats) {
-					overall_stats->Merge(*file_stats);
-				} else {
-					overall_stats = std::move(file_stats);
-				}
-			}
-			// success!
-			return overall_stats;
+		if (bind_data.file_list->GetExpandResult() == FileExpandResult::MULTIPLE_FILES) {
+			// multiple files, no luck!
+			return nullptr;
+		}
+		if (!bind_data.initial_reader) {
+			// no reader
+			return nullptr;
 		}
+		// scanning single parquet file and we have the metadata read already
+		return bind_data.initial_reader->ReadStatistics(bind_data.names[column_index]);
 
-		// multiple files and no object cache, no luck!
 		return nullptr;
 	}
 
 	static unique_ptr<FunctionData> ParquetScanBindInternal(ClientContext &context,
 	                                                        unique_ptr<MultiFileReader> multi_file_reader,
-	                                                        unique_ptr<MultiFileList> file_list,
+	                                                        shared_ptr<MultiFileList> file_list,
 	                                                        vector<LogicalType> &return_types, vector<string> &names,
 	                                                        ParquetOptions parquet_options) {
 		auto result = make_uniq<ParquetReadBindData>();
@@ -546,16 +474,21 @@ public:
 			result->reader_bind = result->multi_file_reader->BindReader<ParquetReader>(
 			    context, result->types, result->names, *result->file_list, *result, parquet_options);
 		}
+
+		// Set the explicit cardinality if requested
 		if (parquet_options.explicit_cardinality) {
 			auto file_count = result->file_list->GetTotalFileCount();
 			result->explicit_cardinality = parquet_options.explicit_cardinality;
 			result->initial_file_cardinality = result->explicit_cardinality / (file_count ? file_count : 1);
 		}
+
 		if (return_types.empty()) {
 			// no expected types - just copy the types
 			return_types = result->types;
 			names = result->names;
 		} else {
+			// We're deserializing from a previously successful bind call
+			// verify that the amount of columns still matches
 			if (return_types.size() != result->types.size()) {
 				auto file_string = bound_on_first_file ? result->file_list->GetFirstFile()
 				                                       : StringUtil::Join(result->file_list->GetPaths(), ",");
@@ -586,9 +519,56 @@ public:
 			result->table_columns = names;
 		}
 		result->parquet_options = parquet_options;
+		result->columns = MultiFileReaderColumnDefinition::ColumnsFromNamesAndTypes(result->names, result->types);
 		return std::move(result);
 	}
 
+	static void VerifyParquetSchemaParameter(const Value &schema) {
+		LogicalType::MAP(LogicalType::BLOB, LogicalType::STRUCT({{{"name", LogicalType::VARCHAR},
+		                                                          {"type", LogicalType::VARCHAR},
+		                                                          {"default_value", LogicalType::VARCHAR}}}));
+		auto &map_type = schema.type();
+		if (map_type.id() != LogicalTypeId::MAP) {
+			throw InvalidInputException("'schema' expects a value of type MAP, not %s",
+			                            LogicalTypeIdToString(map_type.id()));
+		}
+		auto &key_type = MapType::KeyType(map_type);
+		auto &value_type = MapType::ValueType(map_type);
+
+		if (value_type.id() != LogicalTypeId::STRUCT) {
+			throw InvalidInputException("'schema' expects a STRUCT as the value type of the map");
+		}
+		auto &children = StructType::GetChildTypes(value_type);
+		if (children.size() < 3) {
+			throw InvalidInputException(
+			    "'schema' expects the STRUCT to have 3 children, 'name', 'type' and 'default_value");
+		}
+		if (!StringUtil::CIEquals(children[0].first, "name")) {
+			throw InvalidInputException("'schema' expects the first field of the struct to be called 'name'");
+		}
+		if (children[0].second.id() != LogicalTypeId::VARCHAR) {
+			throw InvalidInputException("'schema' expects the 'name' field to be of type VARCHAR, not %s",
+			                            LogicalTypeIdToString(children[0].second.id()));
+		}
+		if (!StringUtil::CIEquals(children[1].first, "type")) {
+			throw InvalidInputException("'schema' expects the second field of the struct to be called 'type'");
+		}
+		if (children[1].second.id() != LogicalTypeId::VARCHAR) {
+			throw InvalidInputException("'schema' expects the 'type' field to be of type VARCHAR, not %s",
+			                            LogicalTypeIdToString(children[1].second.id()));
+		}
+		if (!StringUtil::CIEquals(children[2].first, "default_value")) {
+			throw InvalidInputException("'schema' expects the third field of the struct to be called 'default_value'");
+		}
+		//! NOTE: default_value can be any type
+
+		if (key_type.id() != LogicalTypeId::INTEGER && key_type.id() != LogicalTypeId::VARCHAR) {
+			throw InvalidInputException(
+			    "'schema' expects the value type of the map to be either INTEGER or VARCHAR, not %s",
+			    LogicalTypeIdToString(key_type.id()));
+		}
+	}
+
 	static unique_ptr<FunctionData> ParquetScanBind(ClientContext &context, TableFunctionBindInput &input,
 	                                                vector<LogicalType> &return_types, vector<string> &names) {
 		auto multi_file_reader = MultiFileReader::Create(input.table_function);
@@ -611,6 +591,7 @@ public:
 			} else if (loption == "schema") {
 				// Argument is a map that defines the schema
 				const auto &schema_value = kv.second;
+				VerifyParquetSchemaParameter(schema_value);
 				const auto column_values = ListValue::GetChildren(schema_value);
 				if (column_values.empty()) {
 					throw BinderException("Parquet schema cannot be empty");
@@ -697,9 +678,11 @@ public:
 		auto &file_list = result->file_list;
 		file_list.InitializeScan(result->file_list_scan);
 
+		auto &global_columns = bind_data.reader_bind.schema.empty() ? bind_data.columns : bind_data.reader_bind.schema;
 		result->multi_file_reader_state = bind_data.multi_file_reader->InitializeGlobalState(
-		    context, bind_data.parquet_options.file_options, bind_data.reader_bind, file_list, bind_data.types,
-		    bind_data.names, input.column_ids);
+		    context, bind_data.parquet_options.file_options, bind_data.reader_bind, file_list, global_columns,
+		    input.column_indexes);
+
 		if (file_list.IsEmpty()) {
 			result->readers = {};
 		} else if (!bind_data.union_readers.empty()) {
@@ -737,12 +720,12 @@ public:
 				if (file_name != reader_data->reader->file_name) {
 					throw InternalException("Mismatch in filename order and reader order in parquet scan");
 				}
-				InitializeParquetReader(*reader_data->reader, bind_data, input.column_ids, input.filters, context,
+				InitializeParquetReader(*reader_data->reader, bind_data, input.column_indexes, input.filters, context,
 				                        file_idx, result->multi_file_reader_state);
 			}
 		}
 
-		result->column_ids = input.column_ids;
+		result->column_indexes = input.column_indexes;
 		result->filters = input.filters.get();
 		result->row_group_index = 0;
 		result->file_index = 0;
@@ -755,16 +738,16 @@ public:
 			if (!input.projection_ids.empty()) {
 				result->projection_ids = input.projection_ids;
 			} else {
-				result->projection_ids.resize(input.column_ids.size());
+				result->projection_ids.resize(input.column_indexes.size());
 				iota(begin(result->projection_ids), end(result->projection_ids), 0);
 			}
 
 			const auto table_types = bind_data.types;
-			for (const auto &col_idx : input.column_ids) {
-				if (IsRowIdColumnId(col_idx)) {
+			for (const auto &col_idx : input.column_indexes) {
+				if (col_idx.IsRowIdColumn()) {
 					result->scanned_types.emplace_back(LogicalType::ROW_TYPE);
 				} else {
-					result->scanned_types.push_back(table_types[col_idx]);
+					result->scanned_types.push_back(table_types[col_idx.GetPrimaryIndex()]);
 				}
 			}
 		}
@@ -778,11 +761,16 @@ public:
 		return std::move(result);
 	}
 
-	static idx_t ParquetScanGetBatchIndex(ClientContext &context, const FunctionData *bind_data_p,
-	                                      LocalTableFunctionState *local_state,
-	                                      GlobalTableFunctionState *global_state) {
-		auto &data = local_state->Cast<ParquetReadLocalState>();
-		return data.batch_index;
+	static OperatorPartitionData ParquetScanGetPartitionData(ClientContext &context,
+	                                                         TableFunctionGetPartitionInput &input) {
+		auto &bind_data = input.bind_data->CastNoConst<ParquetReadBindData>();
+		auto &data = input.local_state->Cast<ParquetReadLocalState>();
+		auto &gstate = input.global_state->Cast<ParquetReadGlobalState>();
+		OperatorPartitionData partition_data(data.batch_index);
+		bind_data.multi_file_reader->GetPartitionData(context, bind_data.reader_bind, data.reader->reader_data,
+		                                              gstate.multi_file_reader_state, input.partition_info,
+		                                              partition_data);
+		return partition_data;
 	}
 
 	static void ParquetScanSerialize(Serializer &serializer, const optional_ptr<FunctionData> bind_data_p,
@@ -829,15 +817,18 @@ public:
 		auto &gstate = data_p.global_state->Cast<ParquetReadGlobalState>();
 		auto &bind_data = data_p.bind_data->CastNoConst<ParquetReadBindData>();
 
+		bool rowgroup_finished;
 		do {
 			if (gstate.CanRemoveColumns()) {
 				data.all_columns.Reset();
 				data.reader->Scan(data.scan_state, data.all_columns);
+				rowgroup_finished = data.all_columns.size() == 0;
 				bind_data.multi_file_reader->FinalizeChunk(context, bind_data.reader_bind, data.reader->reader_data,
 				                                           data.all_columns, gstate.multi_file_reader_state);
 				output.ReferenceColumns(data.all_columns, gstate.projection_ids);
 			} else {
 				data.reader->Scan(data.scan_state, output);
+				rowgroup_finished = output.size() == 0;
 				bind_data.multi_file_reader->FinalizeChunk(context, bind_data.reader_bind, data.reader->reader_data,
 				                                           output, gstate.multi_file_reader_state);
 			}
@@ -846,7 +837,7 @@ public:
 			if (output.size() > 0) {
 				return;
 			}
-			if (!ParquetParallelStateNext(context, bind_data, data, gstate)) {
+			if (rowgroup_finished && !ParquetParallelStateNext(context, bind_data, data, gstate)) {
 				return;
 			}
 		} while (true);
@@ -1014,7 +1005,7 @@ public:
 						reader =
 						    make_shared_ptr<ParquetReader>(context, current_reader_data.file_to_be_opened, pq_options);
 					}
-					InitializeParquetReader(*reader, bind_data, parallel_state.column_ids, parallel_state.filters,
+					InitializeParquetReader(*reader, bind_data, parallel_state.column_indexes, parallel_state.filters,
 					                        context, i, parallel_state.multi_file_reader_state);
 				} catch (...) {
 					parallel_lock.lock();
@@ -1189,6 +1180,7 @@ unique_ptr<FunctionData> ParquetWriteBind(ClientContext &context, CopyFunctionBi
                                           const vector<string> &names, const vector<LogicalType> &sql_types) {
 	D_ASSERT(names.size() == sql_types.size());
 	bool row_group_size_bytes_set = false;
+	bool compression_level_set = false;
 	auto bind_data = make_uniq<ParquetWriteBindData>();
 	for (auto &option : input.info.options) {
 		const auto loption = StringUtil::Lower(option.first);
@@ -1211,19 +1203,19 @@ unique_ptr<FunctionData> ParquetWriteBind(ClientContext &context, CopyFunctionBi
 		} else if (loption == "compression" || loption == "codec") {
 			const auto roption = StringUtil::Lower(option.second[0].ToString());
 			if (roption == "uncompressed") {
-				bind_data->codec = duckdb_parquet::format::CompressionCodec::UNCOMPRESSED;
+				bind_data->codec = duckdb_parquet::CompressionCodec::UNCOMPRESSED;
 			} else if (roption == "snappy") {
-				bind_data->codec = duckdb_parquet::format::CompressionCodec::SNAPPY;
+				bind_data->codec = duckdb_parquet::CompressionCodec::SNAPPY;
 			} else if (roption == "gzip") {
-				bind_data->codec = duckdb_parquet::format::CompressionCodec::GZIP;
+				bind_data->codec = duckdb_parquet::CompressionCodec::GZIP;
 			} else if (roption == "zstd") {
-				bind_data->codec = duckdb_parquet::format::CompressionCodec::ZSTD;
+				bind_data->codec = duckdb_parquet::CompressionCodec::ZSTD;
 			} else if (roption == "brotli") {
-				bind_data->codec = duckdb_parquet::format::CompressionCodec::BROTLI;
+				bind_data->codec = duckdb_parquet::CompressionCodec::BROTLI;
 			} else if (roption == "lz4" || roption == "lz4_raw") {
 				/* LZ4 is technically another compression scheme, but deprecated and arrow also uses them
 				 * interchangeably */
-				bind_data->codec = duckdb_parquet::format::CompressionCodec::LZ4_RAW;
+				bind_data->codec = duckdb_parquet::CompressionCodec::LZ4_RAW;
 			} else {
 				throw BinderException("Expected %s argument to be either [uncompressed, brotli, gzip, snappy, or zstd]",
 				                      loption);
@@ -1266,14 +1258,19 @@ unique_ptr<FunctionData> ParquetWriteBind(ClientContext &context, CopyFunctionBi
 		} else if (loption == "encryption_config") {
 			bind_data->encryption_config = ParquetEncryptionConfig::Create(context, option.second[0]);
 		} else if (loption == "dictionary_compression_ratio_threshold") {
+			// deprecated, ignore setting
+		} else if (loption == "dictionary_size_limit") {
+			auto val = option.second[0].GetValue<int64_t>();
+			if (val < 0) {
+				throw BinderException("dictionary_size_limit must be greater than 0 or 0 to disable");
+			}
+			bind_data->dictionary_size_limit = val;
+		} else if (loption == "bloom_filter_false_positive_ratio") {
 			auto val = option.second[0].GetValue<double>();
-			if (val == -1) {
-				val = NumericLimits<double>::Maximum();
-			} else if (val < 0) {
-				throw BinderException("dictionary_compression_ratio_threshold must be greater than 0, or -1 to disable "
-				                      "dictionary compression");
+			if (val <= 0) {
+				throw BinderException("bloom_filter_false_positive_ratio must be greater than 0");
 			}
-			bind_data->dictionary_compression_ratio_threshold = val;
+			bind_data->bloom_filter_false_positive_ratio = val;
 		} else if (loption == "debug_use_openssl") {
 			auto val = StringUtil::Lower(option.second[0].GetValue<std::string>());
 			if (val == "false") {
@@ -1284,7 +1281,14 @@ unique_ptr<FunctionData> ParquetWriteBind(ClientContext &context, CopyFunctionBi
 				throw BinderException("Expected debug_use_openssl to be a BOOLEAN");
 			}
 		} else if (loption == "compression_level") {
-			bind_data->compression_level = option.second[0].GetValue<uint64_t>();
+			const auto val = option.second[0].GetValue<int64_t>();
+			if (val < ZStdFileSystem::MinimumCompressionLevel() || val > ZStdFileSystem::MaximumCompressionLevel()) {
+				throw BinderException("Compression level must be between %lld and %lld",
+				                      ZStdFileSystem::MinimumCompressionLevel(),
+				                      ZStdFileSystem::MaximumCompressionLevel());
+			}
+			bind_data->compression_level = val;
+			compression_level_set = true;
 		} else {
 			throw NotImplementedException("Unrecognized option for PARQUET: %s", option.first.c_str());
 		}
@@ -1294,9 +1298,10 @@ unique_ptr<FunctionData> ParquetWriteBind(ClientContext &context, CopyFunctionBi
 			throw BinderException("ROW_GROUP_SIZE_BYTES does not work while preserving insertion order. Use \"SET "
 			                      "preserve_insertion_order=false;\" to disable preserving insertion order.");
 		}
-	} else {
-		// We always set a max row group size bytes so we don't use too much memory
-		bind_data->row_group_size_bytes = bind_data->row_group_size * ParquetWriteBindData::BYTES_PER_ROW;
+	}
+
+	if (compression_level_set && bind_data->codec != CompressionCodec::ZSTD) {
+		throw BinderException("Compression level is only supported for the ZSTD compression codec");
 	}
 
 	bind_data->sql_types = sql_types;
@@ -1310,11 +1315,11 @@ unique_ptr<GlobalFunctionData> ParquetWriteInitializeGlobal(ClientContext &conte
 	auto &parquet_bind = bind_data.Cast<ParquetWriteBindData>();
 
 	auto &fs = FileSystem::GetFileSystem(context);
-	global_state->writer =
-	    make_uniq<ParquetWriter>(context, fs, file_path, parquet_bind.sql_types, parquet_bind.column_names,
-	                             parquet_bind.codec, parquet_bind.field_ids.Copy(), parquet_bind.kv_metadata,
-	                             parquet_bind.encryption_config, parquet_bind.dictionary_compression_ratio_threshold,
-	                             parquet_bind.compression_level, parquet_bind.debug_use_openssl);
+	global_state->writer = make_uniq<ParquetWriter>(
+	    context, fs, file_path, parquet_bind.sql_types, parquet_bind.column_names, parquet_bind.codec,
+	    parquet_bind.field_ids.Copy(), parquet_bind.kv_metadata, parquet_bind.encryption_config,
+	    parquet_bind.dictionary_size_limit, parquet_bind.bloom_filter_false_positive_ratio,
+	    parquet_bind.compression_level, parquet_bind.debug_use_openssl);
 	return std::move(global_state);
 }
 
@@ -1359,8 +1364,7 @@ unique_ptr<LocalFunctionData> ParquetWriteInitializeLocal(ExecutionContext &cont
 
 // FIXME: Have these be generated instead
 template <>
-const char *EnumUtil::ToChars<duckdb_parquet::format::CompressionCodec::type>(
-    duckdb_parquet::format::CompressionCodec::type value) {
+const char *EnumUtil::ToChars<duckdb_parquet::CompressionCodec::type>(duckdb_parquet::CompressionCodec::type value) {
 	switch (value) {
 	case CompressionCodec::UNCOMPRESSED:
 		return "UNCOMPRESSED";
@@ -1392,8 +1396,7 @@ const char *EnumUtil::ToChars<duckdb_parquet::format::CompressionCodec::type>(
 }
 
 template <>
-duckdb_parquet::format::CompressionCodec::type
-EnumUtil::FromString<duckdb_parquet::format::CompressionCodec::type>(const char *value) {
+duckdb_parquet::CompressionCodec::type EnumUtil::FromString<duckdb_parquet::CompressionCodec::type>(const char *value) {
 	if (StringUtil::Equals(value, "UNCOMPRESSED")) {
 		return CompressionCodec::UNCOMPRESSED;
 	}
@@ -1421,6 +1424,23 @@ EnumUtil::FromString<duckdb_parquet::format::CompressionCodec::type>(const char
 	throw NotImplementedException(StringUtil::Format("Enum value: '%s' not implemented", value));
 }
 
+static optional_idx SerializeCompressionLevel(const int64_t compression_level) {
+	return compression_level < 0 ? NumericLimits<idx_t>::Maximum() - NumericCast<idx_t>(AbsValue(compression_level))
+	                             : NumericCast<idx_t>(compression_level);
+}
+
+static int64_t DeserializeCompressionLevel(const optional_idx compression_level) {
+	// Was originally an optional_idx, now int64_t, so we still serialize as such
+	if (!compression_level.IsValid()) {
+		return ZStdFileSystem::DefaultCompressionLevel();
+	}
+	if (compression_level.GetIndex() > NumericCast<idx_t>(ZStdFileSystem::MaximumCompressionLevel())) {
+		// restore the negative compression level
+		return -NumericCast<int64_t>(NumericLimits<idx_t>::Maximum() - compression_level.GetIndex());
+	}
+	return NumericCast<int64_t>(compression_level.GetIndex());
+}
+
 static void ParquetCopySerialize(Serializer &serializer, const FunctionData &bind_data_p,
                                  const CopyFunction &function) {
 	auto &bind_data = bind_data_p.Cast<ParquetWriteBindData>();
@@ -1433,30 +1453,42 @@ static void ParquetCopySerialize(Serializer &serializer, const FunctionData &bin
 	serializer.WriteProperty(106, "field_ids", bind_data.field_ids);
 	serializer.WritePropertyWithDefault<shared_ptr<ParquetEncryptionConfig>>(107, "encryption_config",
 	                                                                         bind_data.encryption_config, nullptr);
-	serializer.WriteProperty(108, "dictionary_compression_ratio_threshold",
-	                         bind_data.dictionary_compression_ratio_threshold);
-	serializer.WritePropertyWithDefault<optional_idx>(109, "compression_level", bind_data.compression_level);
+
+	// 108 was dictionary_compression_ratio_threshold, but was deleted
+	const auto compression_level = SerializeCompressionLevel(bind_data.compression_level);
+	D_ASSERT(DeserializeCompressionLevel(compression_level) == bind_data.compression_level);
+	serializer.WritePropertyWithDefault<optional_idx>(109, "compression_level", compression_level);
 	serializer.WriteProperty(110, "row_groups_per_file", bind_data.row_groups_per_file);
 	serializer.WriteProperty(111, "debug_use_openssl", bind_data.debug_use_openssl);
+	serializer.WriteProperty(112, "dictionary_size_limit", bind_data.dictionary_size_limit);
+	serializer.WriteProperty(113, "bloom_filter_false_positive_ratio", bind_data.bloom_filter_false_positive_ratio);
 }
 
 static unique_ptr<FunctionData> ParquetCopyDeserialize(Deserializer &deserializer, CopyFunction &function) {
 	auto data = make_uniq<ParquetWriteBindData>();
 	data->sql_types = deserializer.ReadProperty<vector<LogicalType>>(100, "sql_types");
 	data->column_names = deserializer.ReadProperty<vector<string>>(101, "column_names");
-	data->codec = deserializer.ReadProperty<duckdb_parquet::format::CompressionCodec::type>(102, "codec");
+	data->codec = deserializer.ReadProperty<duckdb_parquet::CompressionCodec::type>(102, "codec");
 	data->row_group_size = deserializer.ReadProperty<idx_t>(103, "row_group_size");
 	data->row_group_size_bytes = deserializer.ReadProperty<idx_t>(104, "row_group_size_bytes");
 	data->kv_metadata = deserializer.ReadProperty<vector<pair<string, string>>>(105, "kv_metadata");
 	data->field_ids = deserializer.ReadProperty<ChildFieldIDs>(106, "field_ids");
 	deserializer.ReadPropertyWithExplicitDefault<shared_ptr<ParquetEncryptionConfig>>(107, "encryption_config",
 	                                                                                  data->encryption_config, nullptr);
-	deserializer.ReadPropertyWithExplicitDefault<double>(108, "dictionary_compression_ratio_threshold",
-	                                                     data->dictionary_compression_ratio_threshold, 1.0);
-	deserializer.ReadPropertyWithDefault<optional_idx>(109, "compression_level", data->compression_level);
+	deserializer.ReadDeletedProperty<double>(108, "dictionary_compression_ratio_threshold");
+
+	optional_idx compression_level;
+	deserializer.ReadPropertyWithDefault<optional_idx>(109, "compression_level", compression_level);
+	data->compression_level = DeserializeCompressionLevel(compression_level);
+	D_ASSERT(SerializeCompressionLevel(data->compression_level) == compression_level);
 	data->row_groups_per_file =
 	    deserializer.ReadPropertyWithExplicitDefault<optional_idx>(110, "row_groups_per_file", optional_idx::Invalid());
 	data->debug_use_openssl = deserializer.ReadPropertyWithExplicitDefault<bool>(111, "debug_use_openssl", true);
+	data->dictionary_size_limit =
+	    deserializer.ReadPropertyWithExplicitDefault<idx_t>(112, "dictionary_size_limit", data->row_group_size / 10);
+	data->bloom_filter_false_positive_ratio =
+	    deserializer.ReadPropertyWithExplicitDefault<double>(113, "bloom_filter_false_positive_ratio", 0.01);
+
 	return std::move(data);
 }
 // LCOV_EXCL_STOP
@@ -1577,7 +1609,7 @@ static vector<unique_ptr<Expression>> ParquetWriteSelect(CopyToSelectInput &inpu
 	for (auto &expr : input.select_list) {
 
 		const auto &type = expr->return_type;
-		const auto &name = expr->alias;
+		const auto &name = expr->GetAlias();
 
 		// Spatial types need to be encoded into WKB when writing GeoParquet.
 		// But dont perform this conversion if this is a EXPORT DATABASE statement
@@ -1588,7 +1620,7 @@ static vector<unique_ptr<Expression>> ParquetWriteSelect(CopyToSelectInput &inpu
 			wkb_blob_type.SetAlias("WKB_BLOB");
 
 			auto cast_expr = BoundCastExpression::AddCastToType(context, std::move(expr), wkb_blob_type, false);
-			cast_expr->alias = name;
+			cast_expr->SetAlias(name);
 			result.push_back(std::move(cast_expr));
 			any_change = true;
 		}
@@ -1600,7 +1632,7 @@ static vector<unique_ptr<Expression>> ParquetWriteSelect(CopyToSelectInput &inpu
 
 			// Cast the column to the new type
 			auto cast_expr = BoundCastExpression::AddCastToType(context, std::move(expr), new_type, false);
-			cast_expr->alias = name;
+			cast_expr->SetAlias(name);
 			result.push_back(std::move(cast_expr));
 			any_change = true;
 		}
@@ -1613,7 +1645,7 @@ static vector<unique_ptr<Expression>> ParquetWriteSelect(CopyToSelectInput &inpu
 			});
 
 			auto cast_expr = BoundCastExpression::AddCastToType(context, std::move(expr), new_type, false);
-			cast_expr->alias = name;
+			cast_expr->SetAlias(name);
 			result.push_back(std::move(cast_expr));
 			any_change = true;
 		}
@@ -1658,6 +1690,10 @@ void ParquetExtension::Load(DuckDB &db) {
 	ParquetFileMetadataFunction file_meta_fun;
 	ExtensionUtil::RegisterFunction(db_instance, MultiFileReader::CreateFunctionSet(file_meta_fun));
 
+	// parquet_bloom_probe
+	ParquetBloomProbeFunction bloom_probe_fun;
+	ExtensionUtil::RegisterFunction(db_instance, MultiFileReader::CreateFunctionSet(bloom_probe_fun));
+
 	CopyFunction function("parquet");
 	function.copy_to_select = ParquetWriteSelect;
 	function.copy_to_bind = ParquetWriteBind;
@@ -1689,7 +1725,14 @@ void ParquetExtension::Load(DuckDB &db) {
 	config.replacement_scans.emplace_back(ParquetScanReplacement);
 	config.AddExtensionOption("binary_as_string", "In Parquet files, interpret binary data as a string.",
 	                          LogicalType::BOOLEAN);
-
+	config.AddExtensionOption("disable_parquet_prefetching", "Disable the prefetching mechanism in Parquet",
+	                          LogicalType::BOOLEAN, Value(false));
+	config.AddExtensionOption("prefetch_all_parquet_files",
+	                          "Use the prefetching mechanism for all types of parquet files", LogicalType::BOOLEAN,
+	                          Value(false));
+	config.AddExtensionOption("parquet_metadata_cache",
+	                          "Cache Parquet metadata - useful when reading the same files multiple times",
+	                          LogicalType::BOOLEAN, Value(false));
 	config.AddExtensionOption(
 	    "enable_geoparquet_conversion",
 	    "Attempt to decode/encode geometry data in/as GeoParquet files if the spatial extension is present.",