duckdb 0.8.2-dev161.0 → 0.8.2-dev1764.0

package/src/duckdb/extension/parquet/parquet_extension.cpp

@@ -1,7 +1,8 @@
 #define DUCKDB_EXTENSION_MAIN
 
-#include "duckdb.hpp"
 #include "parquet_extension.hpp"
+
+#include "duckdb.hpp"
 #include "parquet_metadata.hpp"
 #include "parquet_reader.hpp"
 #include "parquet_writer.hpp"
@@ -14,15 +15,18 @@
 #include <vector>
 #ifndef DUCKDB_AMALGAMATION
 #include "duckdb/catalog/catalog.hpp"
+#include "duckdb/catalog/catalog_entry/table_function_catalog_entry.hpp"
 #include "duckdb/common/constants.hpp"
 #include "duckdb/common/enums/file_compression_type.hpp"
 #include "duckdb/common/field_writer.hpp"
 #include "duckdb/common/file_system.hpp"
+#include "duckdb/common/multi_file_reader.hpp"
 #include "duckdb/common/types/chunk_collection.hpp"
 #include "duckdb/function/copy_function.hpp"
 #include "duckdb/function/table_function.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/main/config.hpp"
+#include "duckdb/main/extension_util.hpp"
 #include "duckdb/parser/expression/constant_expression.hpp"
 #include "duckdb/parser/expression/function_expression.hpp"
 #include "duckdb/parser/parsed_data/create_copy_function_info.hpp"
@@ -30,10 +34,7 @@
 #include "duckdb/parser/tableref/table_function_ref.hpp"
 #include "duckdb/planner/operator/logical_get.hpp"
 #include "duckdb/storage/statistics/base_statistics.hpp"
-#include "duckdb/catalog/catalog_entry/table_function_catalog_entry.hpp"
-#include "duckdb/common/multi_file_reader.hpp"
 #include "duckdb/storage/table/row_group.hpp"
-#include "duckdb/main/extension_util.hpp"
 #endif
 
 namespace duckdb {
@@ -115,6 +116,7 @@ struct ParquetWriteBindData : public TableFunctionData {
 	vector<string> column_names;
 	duckdb_parquet::format::CompressionCodec::type codec = duckdb_parquet::format::CompressionCodec::SNAPPY;
 	idx_t row_group_size = RowGroup::ROW_GROUP_SIZE;
+	ChildFieldIDs field_ids;
 };
 
 struct ParquetWriteGlobalState : public GlobalFunctionData {
@@ -123,10 +125,12 @@ struct ParquetWriteGlobalState : public GlobalFunctionData {
 
 struct ParquetWriteLocalState : public LocalFunctionData {
 	explicit ParquetWriteLocalState(ClientContext &context, const vector<LogicalType> &types)
-	    : buffer(Allocator::Get(context), types) {
+	    : buffer(context, types, ColumnDataAllocatorType::HYBRID) {
+		buffer.InitializeAppend(append_state);
 	}
 
 	ColumnDataCollection buffer;
+	ColumnDataAppendState append_state;
 };
 
 void ParquetOptions::Serialize(FieldWriter &writer) const {
@@ -148,10 +152,12 @@ BindInfo ParquetGetBatchInfo(const FunctionData *bind_data) {
 	for (auto &path : parquet_bind.files) {
 		file_path.emplace_back(path);
 	}
+	// LCOV_EXCL_START
 	bind_info.InsertOption("file_path", Value::LIST(LogicalType::VARCHAR, file_path));
 	bind_info.InsertOption("binary_as_string", Value::BOOLEAN(parquet_bind.parquet_options.binary_as_string));
 	bind_info.InsertOption("file_row_number", Value::BOOLEAN(parquet_bind.parquet_options.file_row_number));
 	parquet_bind.parquet_options.file_options.AddBatchInfo(bind_info);
+	// LCOV_EXCL_STOP
 	return bind_info;
 }
 
@@ -171,7 +177,6 @@ public:
 		table_function.serialize = ParquetScanSerialize;
 		table_function.deserialize = ParquetScanDeserialize;
 		table_function.get_batch_info = ParquetGetBatchInfo;
-
 		table_function.projection_pushdown = true;
 		table_function.filter_pushdown = true;
 		table_function.filter_prune = true;
@@ -293,7 +298,7 @@ public:
 		ParquetOptions parquet_options(context);
 		for (auto &kv : input.named_parameters) {
 			auto loption = StringUtil::Lower(kv.first);
-			if (MultiFileReader::ParseOption(kv.first, kv.second, parquet_options.file_options)) {
+			if (MultiFileReader::ParseOption(kv.first, kv.second, parquet_options.file_options, context)) {
 				continue;
 			}
 			if (loption == "binary_as_string") {
@@ -302,9 +307,7 @@ public:
 				parquet_options.file_row_number = BooleanValue::Get(kv.second);
 			}
 		}
-		if (parquet_options.file_options.auto_detect_hive_partitioning) {
-			parquet_options.file_options.hive_partitioning = MultiFileReaderOptions::AutoDetectHivePartitioning(files);
-		}
+		parquet_options.file_options.AutoDetectHivePartitioning(files, context);
 		return ParquetScanBindInternal(context, std::move(files), return_types, names, parquet_options);
 	}
 
@@ -371,7 +374,7 @@ public:
 			}
 			MultiFileReader::InitializeReader(*reader, bind_data.parquet_options.file_options, bind_data.reader_bind,
 			                                  bind_data.types, bind_data.names, input.column_ids, input.filters,
-			                                  bind_data.files[0]);
+			                                  bind_data.files[0], context);
 		}
 
 		result->column_ids = input.column_ids;
@@ -518,6 +521,7 @@ public:
 	static void ParquetComplexFilterPushdown(ClientContext &context, LogicalGet &get, FunctionData *bind_data_p,
 	                                         vector<unique_ptr<Expression>> &filters) {
 		auto &data = bind_data_p->Cast<ParquetReadBindData>();
+
 		auto reset_reader = MultiFileReader::ComplexFilterPushdown(context, data.files,
 		                                                           data.parquet_options.file_options, get, filters);
 		if (reset_reader) {
@@ -564,9 +568,10 @@ public:
 				shared_ptr<ParquetReader> reader;
 				try {
 					reader = make_shared<ParquetReader>(context, file, pq_options);
-					MultiFileReader::InitializeReader(
-					    *reader, bind_data.parquet_options.file_options, bind_data.reader_bind, bind_data.types,
-					    bind_data.names, parallel_state.column_ids, parallel_state.filters, bind_data.files.front());
+					MultiFileReader::InitializeReader(*reader, bind_data.parquet_options.file_options,
+					                                  bind_data.reader_bind, bind_data.types, bind_data.names,
+					                                  parallel_state.column_ids, parallel_state.filters,
+					                                  bind_data.files.front(), context);
 				} catch (...) {
 					parallel_lock.lock();
 					parallel_state.error_opening_file = true;
@@ -585,8 +590,157 @@ public:
 	}
 };
 
+static case_insensitive_map_t<LogicalType> GetChildNameToTypeMap(const LogicalType &type) {
+	case_insensitive_map_t<LogicalType> name_to_type_map;
+	switch (type.id()) {
+	case LogicalTypeId::LIST:
+		name_to_type_map.emplace("element", ListType::GetChildType(type));
+		break;
+	case LogicalTypeId::MAP:
+		name_to_type_map.emplace("key", MapType::KeyType(type));
+		name_to_type_map.emplace("value", MapType::ValueType(type));
+		break;
+	case LogicalTypeId::STRUCT:
+		for (auto &child_type : StructType::GetChildTypes(type)) {
+			if (child_type.first == FieldID::DUCKDB_FIELD_ID) {
+				throw BinderException("Cannot have column named \"%s\" with FIELD_IDS", FieldID::DUCKDB_FIELD_ID);
+			}
+			name_to_type_map.emplace(child_type);
+		}
+		break;
+	default: // LCOV_EXCL_START
+		throw InternalException("Unexpected type in GetChildNameToTypeMap");
+	} // LCOV_EXCL_STOP
+	return name_to_type_map;
+}
+
+static void GetChildNamesAndTypes(const LogicalType &type, vector<string> &child_names,
+                                  vector<LogicalType> &child_types) {
+	switch (type.id()) {
+	case LogicalTypeId::LIST:
+		child_names.emplace_back("element");
+		child_types.emplace_back(ListType::GetChildType(type));
+		break;
+	case LogicalTypeId::MAP:
+		child_names.emplace_back("key");
+		child_names.emplace_back("value");
+		child_types.emplace_back(MapType::KeyType(type));
+		child_types.emplace_back(MapType::ValueType(type));
+		break;
+	case LogicalTypeId::STRUCT:
+		for (auto &child_type : StructType::GetChildTypes(type)) {
+			child_names.emplace_back(child_type.first);
+			child_types.emplace_back(child_type.second);
+		}
+		break;
+	default: // LCOV_EXCL_START
+		throw InternalException("Unexpected type in GetChildNamesAndTypes");
+	} // LCOV_EXCL_STOP
+}
+
+static void GenerateFieldIDs(ChildFieldIDs &field_ids, idx_t &field_id, const vector<string> &names,
+                             const vector<LogicalType> &sql_types) {
+	D_ASSERT(names.size() == sql_types.size());
+	for (idx_t col_idx = 0; col_idx < names.size(); col_idx++) {
+		const auto &col_name = names[col_idx];
+		auto inserted = field_ids.ids->insert(make_pair(col_name, FieldID(field_id++)));
+		D_ASSERT(inserted.second);
+
+		const auto &col_type = sql_types[col_idx];
+		if (col_type.id() != LogicalTypeId::LIST && col_type.id() != LogicalTypeId::MAP &&
+		    col_type.id() != LogicalTypeId::STRUCT) {
+			continue;
+		}
+
+		// Cannot use GetChildNameToTypeMap here because we lose order, and we want to generate depth-first
+		vector<string> child_names;
+		vector<LogicalType> child_types;
+		GetChildNamesAndTypes(col_type, child_names, child_types);
+
+		GenerateFieldIDs(inserted.first->second.child_field_ids, field_id, child_names, child_types);
+	}
+}
+
+static void GetFieldIDs(const Value &field_ids_value, ChildFieldIDs &field_ids,
+                        unordered_set<uint32_t> &unique_field_ids,
+                        const case_insensitive_map_t<LogicalType> &name_to_type_map) {
+	const auto &struct_type = field_ids_value.type();
+	if (struct_type.id() != LogicalTypeId::STRUCT) {
+		throw BinderException(
+		    "Expected FIELD_IDS to be a STRUCT, e.g., {col1: 42, col2: {%s: 43, nested_col: 44}, col3: 44}",
+		    FieldID::DUCKDB_FIELD_ID);
+	}
+	const auto &struct_children = StructValue::GetChildren(field_ids_value);
+	D_ASSERT(StructType::GetChildTypes(struct_type).size() == struct_children.size());
+	for (idx_t i = 0; i < struct_children.size(); i++) {
+		const auto &col_name = StringUtil::Lower(StructType::GetChildName(struct_type, i));
+		if (col_name == FieldID::DUCKDB_FIELD_ID) {
+			continue;
+		}
+
+		auto it = name_to_type_map.find(col_name);
+		if (it == name_to_type_map.end()) {
+			string names;
+			for (const auto &name : name_to_type_map) {
+				if (!names.empty()) {
+					names += ", ";
+				}
+				names += name.first;
+			}
+			throw BinderException("Column name \"%s\" specified in FIELD_IDS not found. Available column names: [%s]",
+			                      col_name, names);
+		}
+		D_ASSERT(field_ids.ids->find(col_name) == field_ids.ids->end()); // Caught by STRUCT - deduplicates keys
+
+		const auto &child_value = struct_children[i];
+		const auto &child_type = child_value.type();
+		optional_ptr<const Value> field_id_value;
+		optional_ptr<const Value> child_field_ids_value;
+
+		if (child_type.id() == LogicalTypeId::STRUCT) {
+			const auto &nested_children = StructValue::GetChildren(child_value);
+			D_ASSERT(StructType::GetChildTypes(child_type).size() == nested_children.size());
+			for (idx_t nested_i = 0; nested_i < nested_children.size(); nested_i++) {
+				const auto &field_id_or_nested_col = StructType::GetChildName(child_type, nested_i);
+				if (field_id_or_nested_col == FieldID::DUCKDB_FIELD_ID) {
+					field_id_value = &nested_children[nested_i];
+				} else {
+					child_field_ids_value = &child_value;
+				}
+			}
+		} else {
+			field_id_value = &child_value;
+		}
+
+		FieldID field_id;
+		if (field_id_value) {
+			Value field_id_integer_value = field_id_value->DefaultCastAs(LogicalType::INTEGER);
+			const uint32_t field_id_int = IntegerValue::Get(field_id_integer_value);
+			if (!unique_field_ids.insert(field_id_int).second) {
+				throw BinderException("Duplicate field_id %s found in FIELD_IDS", field_id_integer_value.ToString());
+			}
+			field_id = FieldID(field_id_int);
+		}
+		auto inserted = field_ids.ids->insert(make_pair(col_name, std::move(field_id)));
+		D_ASSERT(inserted.second);
+
+		if (child_field_ids_value) {
+			const auto &col_type = it->second;
+			if (col_type.id() != LogicalTypeId::LIST && col_type.id() != LogicalTypeId::MAP &&
+			    col_type.id() != LogicalTypeId::STRUCT) {
+				throw BinderException("Column \"%s\" with type \"%s\" cannot have a nested FIELD_IDS specification",
+				                      col_name, LogicalTypeIdToString(col_type.id()));
+			}
+
+			GetFieldIDs(*child_field_ids_value, inserted.first->second.child_field_ids, unique_field_ids,
+			            GetChildNameToTypeMap(col_type));
+		}
+	}
+}
+
 unique_ptr<FunctionData> ParquetWriteBind(ClientContext &context, CopyInfo &info, vector<string> &names,
                                           vector<LogicalType> &sql_types) {
+	D_ASSERT(names.size() == sql_types.size());
 	auto bind_data = make_uniq<ParquetWriteBindData>();
 	for (auto &option : info.options) {
 		auto loption = StringUtil::Lower(option.first);
@@ -609,7 +763,27 @@ unique_ptr<FunctionData> ParquetWriteBind(ClientContext &context, CopyInfo &info
 					continue;
 				}
 			}
-			throw ParserException("Expected %s argument to be either [uncompressed, snappy, gzip or zstd]", loption);
+			throw BinderException("Expected %s argument to be either [uncompressed, snappy, gzip or zstd]", loption);
+		} else if (loption == "field_ids") {
+			if (option.second.size() != 1) {
+				throw BinderException("FIELD_IDS requires exactly one argument");
+			}
+			if (option.second[0].type().id() == LogicalTypeId::VARCHAR &&
+			    StringUtil::Lower(StringValue::Get(option.second[0])) == "auto") {
+				idx_t field_id = 0;
+				GenerateFieldIDs(bind_data->field_ids, field_id, names, sql_types);
+			} else {
+				unordered_set<uint32_t> unique_field_ids;
+				case_insensitive_map_t<LogicalType> name_to_type_map;
+				for (idx_t col_idx = 0; col_idx < names.size(); col_idx++) {
+					if (names[col_idx] == FieldID::DUCKDB_FIELD_ID) {
+						throw BinderException("Cannot have a column named \"%s\" when writing FIELD_IDS",
+						                      FieldID::DUCKDB_FIELD_ID);
+					}
+					name_to_type_map.emplace(names[col_idx], sql_types[col_idx]);
+				}
+				GetFieldIDs(option.second[0], bind_data->field_ids, unique_field_ids, name_to_type_map);
+			}
 		} else {
 			throw NotImplementedException("Unrecognized option for PARQUET: %s", option.first.c_str());
 		}
@@ -625,8 +799,8 @@ unique_ptr<GlobalFunctionData> ParquetWriteInitializeGlobal(ClientContext &conte
 	auto &parquet_bind = bind_data.Cast<ParquetWriteBindData>();
 
 	auto &fs = FileSystem::GetFileSystem(context);
-	global_state->writer =
-	    make_uniq<ParquetWriter>(fs, file_path, parquet_bind.sql_types, parquet_bind.column_names, parquet_bind.codec);
+	global_state->writer = make_uniq<ParquetWriter>(fs, file_path, parquet_bind.sql_types, parquet_bind.column_names,
+	                                                parquet_bind.codec, parquet_bind.field_ids.Copy());
 	return std::move(global_state);
 }
 
@@ -637,12 +811,12 @@ void ParquetWriteSink(ExecutionContext &context, FunctionData &bind_data_p, Glob
 	auto &local_state = lstate.Cast<ParquetWriteLocalState>();
 
 	// append data to the local (buffered) chunk collection
-	local_state.buffer.Append(input);
+	local_state.buffer.Append(local_state.append_state, input);
 	if (local_state.buffer.Count() > bind_data.row_group_size) {
 		// if the chunk collection exceeds a certain size we flush it to the parquet file
+		local_state.append_state.current_chunk_state.handles.clear();
 		global_state.writer->Flush(local_state.buffer);
-		// and reset the buffer
-		local_state.buffer.Reset();
+		local_state.buffer.InitializeAppend(local_state.append_state);
 	}
 }
 

package/src/duckdb/extension/parquet/parquet_reader.cpp

@@ -48,7 +48,7 @@ using duckdb_parquet::format::SchemaElement;
 using duckdb_parquet::format::Statistics;
 using duckdb_parquet::format::Type;
 
-static duckdb::unique_ptr<duckdb_apache::thrift::protocol::TProtocol>
+static unique_ptr<duckdb_apache::thrift::protocol::TProtocol>
 CreateThriftProtocol(Allocator &allocator, FileHandle &file_handle, bool prefetch_mode) {
 	auto transport = make_shared<ThriftFileTransport>(allocator, file_handle, prefetch_mode);
 	return make_uniq<duckdb_apache::thrift::protocol::TCompactProtocolT<ThriftFileTransport>>(std::move(transport));
@@ -76,7 +76,7 @@ static shared_ptr<ParquetFileMetadataCache> LoadMetadata(Allocator &allocator, F
 	}
 	// read four-byte footer length from just before the end magic bytes
 	auto footer_len = *reinterpret_cast<uint32_t *>(buf.ptr);
-	if (footer_len <= 0 || file_size < 12 + footer_len) {
+	if (footer_len == 0 || file_size < 12 + footer_len) {
 		throw InvalidInputException("Footer length error in file '%s'", file_handle.path);
 	}
 	auto metadata_pos = file_size - (footer_len + 8);
@@ -271,7 +271,7 @@ unique_ptr<ColumnReader> ParquetReader::CreateReaderRecursive(idx_t depth, idx_t
 	}
 	if (s_ele.__isset.num_children && s_ele.num_children > 0) { // inner node
 		child_list_t<LogicalType> child_types;
-		vector<duckdb::unique_ptr<ColumnReader>> child_readers;
+		vector<unique_ptr<ColumnReader>> child_readers;
 
 		idx_t c_idx = 0;
 		while (c_idx < (idx_t)s_ele.num_children) {
@@ -287,7 +287,7 @@ unique_ptr<ColumnReader> ParquetReader::CreateReaderRecursive(idx_t depth, idx_t
 			c_idx++;
 		}
 		D_ASSERT(!child_types.empty());
-		duckdb::unique_ptr<ColumnReader> result;
+		unique_ptr<ColumnReader> result;
 		LogicalType result_type;
 
 		bool is_repeated = repetition_type == FieldRepetitionType::REPEATED;
@@ -429,7 +429,7 @@ ParquetOptions::ParquetOptions(ClientContext &context) {
 
 ParquetReader::ParquetReader(ClientContext &context_p, string file_name_p, ParquetOptions parquet_options_p)
     : fs(FileSystem::GetFileSystem(context_p)), allocator(BufferAllocator::Get(context_p)),
-      parquet_options(parquet_options_p) {
+      parquet_options(std::move(parquet_options_p)) {
 	file_name = std::move(file_name_p);
 	file_handle = fs.OpenFile(file_name, FileFlags::FILE_FLAGS_READ);
 	if (!file_handle->CanSeek()) {
@@ -457,7 +457,7 @@ ParquetReader::ParquetReader(ClientContext &context_p, string file_name_p, Parqu
 ParquetReader::ParquetReader(ClientContext &context_p, ParquetOptions parquet_options_p,
                              shared_ptr<ParquetFileMetadataCache> metadata_p)
     : fs(FileSystem::GetFileSystem(context_p)), allocator(BufferAllocator::Get(context_p)),
-      metadata(std::move(metadata_p)), parquet_options(parquet_options_p) {
+      metadata(std::move(metadata_p)), parquet_options(std::move(parquet_options_p)) {
 	InitializeSchema();
 }
 

package/src/duckdb/extension/parquet/parquet_statistics.cpp

@@ -1,12 +1,13 @@
 #include "parquet_statistics.hpp"
+
+#include "duckdb.hpp"
 #include "parquet_decimal_utils.hpp"
 #include "parquet_timestamp.hpp"
 #include "string_column_reader.hpp"
-#include "duckdb.hpp"
 #ifndef DUCKDB_AMALGAMATION
 #include "duckdb/common/types/blob.hpp"
-#include "duckdb/common/types/value.hpp"
 #include "duckdb/common/types/time.hpp"
+#include "duckdb/common/types/value.hpp"
 #endif
 
 namespace duckdb {
@@ -14,9 +15,9 @@ namespace duckdb {
 using duckdb_parquet::format::ConvertedType;
 using duckdb_parquet::format::Type;
 
-static duckdb::unique_ptr<BaseStatistics> CreateNumericStats(const LogicalType &type,
-                                                             const duckdb_parquet::format::SchemaElement &schema_ele,
-                                                             const duckdb_parquet::format::Statistics &parquet_stats) {
+static unique_ptr<BaseStatistics> CreateNumericStats(const LogicalType &type,
+                                                     const duckdb_parquet::format::SchemaElement &schema_ele,
+                                                     const duckdb_parquet::format::Statistics &parquet_stats) {
 	auto stats = NumericStats::CreateUnknown(type);
 
 	// for reasons unknown to science, Parquet defines *both* `min` and `min_value` as well as `max` and
@@ -226,7 +227,7 @@ unique_ptr<BaseStatistics> ParquetStatisticsUtils::TransformColumnStatistics(con
 		return nullptr;
 	}
 	auto &parquet_stats = column_chunk.meta_data.statistics;
-	duckdb::unique_ptr<BaseStatistics> row_group_stats;
+	unique_ptr<BaseStatistics> row_group_stats;
 
 	switch (type.id()) {
 	case LogicalTypeId::UTINYINT:

package/src/duckdb/extension/parquet/parquet_writer.cpp

@@ -29,6 +29,30 @@ using duckdb_parquet::format::PageType;
 using ParquetRowGroup = duckdb_parquet::format::RowGroup;
 using duckdb_parquet::format::Type;
 
+ChildFieldIDs::ChildFieldIDs() {
+	ids = make_uniq<case_insensitive_map_t<FieldID>>();
+}
+
+ChildFieldIDs ChildFieldIDs::Copy() const {
+	ChildFieldIDs result;
+	for (const auto &id : *ids) {
+		result.ids->emplace(id.first, id.second.Copy());
+	}
+	return result;
+}
+
+FieldID::FieldID() : set(false) {
+}
+
+FieldID::FieldID(int32_t field_id_p) : set(true), field_id(field_id_p) {
+}
+
+FieldID FieldID::Copy() const {
+	auto result = set ? FieldID(field_id) : FieldID();
+	result.child_field_ids = child_field_ids.Copy();
+	return result;
+}
+
 class MyTransport : public TTransport {
 public:
 	explicit MyTransport(Serializer &serializer) : serializer(serializer) {
@@ -226,8 +250,9 @@ void VerifyUniqueNames(const vector<string> &names) {
 }
 
 ParquetWriter::ParquetWriter(FileSystem &fs, string file_name_p, vector<LogicalType> types_p, vector<string> names_p,
-                             CompressionCodec::type codec)
-    : file_name(std::move(file_name_p)), sql_types(std::move(types_p)), column_names(std::move(names_p)), codec(codec) {
+                             CompressionCodec::type codec, ChildFieldIDs field_ids_p)
+    : file_name(std::move(file_name_p)), sql_types(std::move(types_p)), column_names(std::move(names_p)), codec(codec),
+      field_ids(std::move(field_ids_p)) {
 	// initialize the file writer
 	writer = make_uniq<BufferedFileWriter>(fs, file_name.c_str(),
 	                                       FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE_NEW);
@@ -257,11 +282,18 @@ ParquetWriter::ParquetWriter(FileSystem &fs, string file_name_p, vector<LogicalT
 	vector<string> schema_path;
 	for (idx_t i = 0; i < sql_types.size(); i++) {
 		column_writers.push_back(ColumnWriter::CreateWriterRecursive(file_meta_data.schema, *this, sql_types[i],
-		                                                             unique_names[i], schema_path));
+		                                                             unique_names[i], schema_path, &field_ids));
 	}
 }
 
 void ParquetWriter::PrepareRowGroup(ColumnDataCollection &buffer, PreparedRowGroup &result) {
+	// We write 8 columns at a time so that iterating over ColumnDataCollection is more efficient
+	static constexpr idx_t COLUMNS_PER_PASS = 8;
+
+	// We want these to be in-memory/hybrid so we don't have to copy over strings to the dictionary
+	D_ASSERT(buffer.GetAllocatorType() == ColumnDataAllocatorType::IN_MEMORY_ALLOCATOR ||
+	         buffer.GetAllocatorType() == ColumnDataAllocatorType::HYBRID);
+
 	// set up a new row group for this chunk collection
 	auto &row_group = result.row_group;
 	row_group.num_rows = buffer.Count();
@@ -270,24 +302,52 @@ void ParquetWriter::PrepareRowGroup(ColumnDataCollection &buffer, PreparedRowGro
 	auto &states = result.states;
 	// iterate over each of the columns of the chunk collection and write them
 	D_ASSERT(buffer.ColumnCount() == column_writers.size());
-	for (idx_t col_idx = 0; col_idx < buffer.ColumnCount(); col_idx++) {
-		const auto &col_writer = column_writers[col_idx];
-		auto write_state = col_writer->InitializeWriteState(row_group, buffer.GetAllocator());
-		if (col_writer->HasAnalyze()) {
-			for (auto &chunk : buffer.Chunks()) {
-				col_writer->Analyze(*write_state, nullptr, chunk.data[col_idx], chunk.size());
+	for (idx_t col_idx = 0; col_idx < buffer.ColumnCount(); col_idx += COLUMNS_PER_PASS) {
+		const auto next = MinValue<idx_t>(buffer.ColumnCount() - col_idx, COLUMNS_PER_PASS);
+		vector<column_t> column_ids;
+		vector<reference<ColumnWriter>> col_writers;
+		vector<unique_ptr<ColumnWriterState>> write_states;
+		for (idx_t i = 0; i < next; i++) {
+			column_ids.emplace_back(col_idx + i);
+			col_writers.emplace_back(*column_writers[column_ids.back()]);
+			write_states.emplace_back(col_writers.back().get().InitializeWriteState(row_group));
+		}
+
+		for (auto &chunk : buffer.Chunks({column_ids})) {
+			for (idx_t i = 0; i < next; i++) {
+				if (col_writers[i].get().HasAnalyze()) {
+					col_writers[i].get().Analyze(*write_states[i], nullptr, chunk.data[i], chunk.size());
+				}
+			}
+		}
+
+		for (idx_t i = 0; i < next; i++) {
+			if (col_writers[i].get().HasAnalyze()) {
+				col_writers[i].get().FinalizeAnalyze(*write_states[i]);
+			}
+		}
+
+		for (auto &chunk : buffer.Chunks({column_ids})) {
+			for (idx_t i = 0; i < next; i++) {
+				col_writers[i].get().Prepare(*write_states[i], nullptr, chunk.data[i], chunk.size());
 			}
-			col_writer->FinalizeAnalyze(*write_state);
 		}
-		for (auto &chunk : buffer.Chunks()) {
-			col_writer->Prepare(*write_state, nullptr, chunk.data[col_idx], chunk.size());
+
+		for (idx_t i = 0; i < next; i++) {
+			col_writers[i].get().BeginWrite(*write_states[i]);
 		}
-		col_writer->BeginWrite(*write_state);
-		for (auto &chunk : buffer.Chunks()) {
-			col_writer->Write(*write_state, chunk.data[col_idx], chunk.size());
+
+		for (auto &chunk : buffer.Chunks({column_ids})) {
+			for (idx_t i = 0; i < next; i++) {
+				col_writers[i].get().Write(*write_states[i], chunk.data[i], chunk.size());
+			}
+		}
+
+		for (auto &write_state : write_states) {
+			states.push_back(std::move(write_state));
 		}
-		states.push_back(std::move(write_state));
 	}
+	result.heaps = buffer.GetHeapReferences();
 }
 
 void ParquetWriter::FlushRowGroup(PreparedRowGroup &prepared) {
@@ -307,6 +367,8 @@ void ParquetWriter::FlushRowGroup(PreparedRowGroup &prepared) {
 	// append the row group to the file meta data
 	file_meta_data.row_groups.push_back(row_group);
 	file_meta_data.num_rows += row_group.num_rows;
+
+	prepared.heaps.clear();
 }
 
 void ParquetWriter::Flush(ColumnDataCollection &buffer) {
@@ -316,6 +378,7 @@ void ParquetWriter::Flush(ColumnDataCollection &buffer) {
 
 	PreparedRowGroup prepared_row_group;
 	PrepareRowGroup(buffer, prepared_row_group);
+	buffer.Reset();
 
 	FlushRowGroup(prepared_row_group);
 }

package/src/duckdb/extension/parquet/zstd_file_system.cpp

@@ -155,7 +155,7 @@ void ZstdStreamWrapper::Close() {
 
 class ZStdFile : public CompressedFile {
 public:
-	ZStdFile(duckdb::unique_ptr<FileHandle> child_handle_p, const string &path, bool write)
+	ZStdFile(unique_ptr<FileHandle> child_handle_p, const string &path, bool write)
 	    : CompressedFile(zstd_fs, std::move(child_handle_p), path) {
 		Initialize(write);
 	}
@@ -163,7 +163,7 @@ public:
 	ZStdFileSystem zstd_fs;
 };
 
-unique_ptr<FileHandle> ZStdFileSystem::OpenCompressedFile(duckdb::unique_ptr<FileHandle> handle, bool write) {
+unique_ptr<FileHandle> ZStdFileSystem::OpenCompressedFile(unique_ptr<FileHandle> handle, bool write) {
 	auto path = handle->path;
 	return make_uniq<ZStdFile>(std::move(handle), path, write);
 }

package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp

@@ -298,7 +298,7 @@ unique_ptr<CatalogEntry> DuckTableEntry::AddColumn(ClientContext &context, AddCo
 	auto binder = Binder::CreateBinder(context);
 	auto bound_create_info = binder->BindCreateTableInfo(std::move(create_info));
 	auto new_storage =
-	    make_shared<DataTable>(context, *storage, info.new_column, bound_create_info->bound_defaults.back().get());
+	    make_shared<DataTable>(context, *storage, info.new_column, *bound_create_info->bound_defaults.back());
 	return make_uniq<DuckTableEntry>(catalog, schema, *bound_create_info, new_storage);
 }
 

package/src/duckdb/src/catalog/catalog_search_path.cpp

@@ -1,10 +1,10 @@
 #include "duckdb/catalog/catalog_search_path.hpp"
 
+#include "duckdb/catalog/catalog.hpp"
 #include "duckdb/common/constants.hpp"
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/string_util.hpp"
 #include "duckdb/main/client_context.hpp"
-#include "duckdb/catalog/catalog.hpp"
 #include "duckdb/main/database_manager.hpp"
 
 namespace duckdb {
@@ -249,13 +249,14 @@ void CatalogSearchPath::SetPaths(vector<CatalogSearchEntry> new_paths) {
 bool CatalogSearchPath::SchemaInSearchPath(ClientContext &context, const string &catalog_name,
                                            const string &schema_name) {
 	for (auto &path : paths) {
-		if (path.schema != schema_name) {
+		if (!StringUtil::CIEquals(path.schema, schema_name)) {
 			continue;
 		}
-		if (path.catalog == catalog_name) {
+		if (StringUtil::CIEquals(path.catalog, catalog_name)) {
 			return true;
 		}
-		if (IsInvalidCatalog(path.catalog) && catalog_name == DatabaseManager::GetDefaultDatabase(context)) {
+		if (IsInvalidCatalog(path.catalog) &&
+		    StringUtil::CIEquals(catalog_name, DatabaseManager::GetDefaultDatabase(context))) {
 			return true;
 		}
 	}

package/src/duckdb/src/catalog/default/default_functions.cpp

@@ -99,6 +99,19 @@ static DefaultMacro internal_macros[] = {
 	{DEFAULT_SCHEMA, "count_if", {"l", nullptr}, "sum(if(l, 1, 0))"},
 	{DEFAULT_SCHEMA, "split_part", {"string", "delimiter", "position", nullptr}, "coalesce(string_split(string, delimiter)[position],'')"},
 
+    // FIXME implement as actual function if we encounter a lot of performance issues. Complexity now: n * m, with hashing possibly n + m
+    {DEFAULT_SCHEMA, "list_intersect", {"l1", "l2", nullptr}, "list_filter(l1, (x) -> list_contains(l2, x))"},
+    {DEFAULT_SCHEMA, "array_intersect", {"l1", "l2", nullptr}, "list_intersect(l1, l2)"},
+
+    {DEFAULT_SCHEMA, "list_has_any", {"l1", "l2", nullptr}, "CASE WHEN l1 IS NULL THEN NULL WHEN l2 IS NULL THEN NULL WHEN len(list_intersect(l1, l2)) > 0 THEN true ELSE false END"},
+    {DEFAULT_SCHEMA, "array_has_any", {"l1", "l2", nullptr}, "list_has_any(l1, l2)" },
+    {DEFAULT_SCHEMA, "&&", {"l1", "l2", nullptr}, "list_has_any(l1, l2)" }, // "&&" is the operator for "list_has_any
+
+    {DEFAULT_SCHEMA, "list_has_all", {"l1", "l2", nullptr}, "CASE WHEN l1 IS NULL THEN NULL WHEN l2 IS NULL THEN NULL WHEN len(list_intersect(l2, l1)) = len(list_filter(l2, x -> x IS NOT NULL)) THEN true ELSE false END"},
+    {DEFAULT_SCHEMA, "array_has_all", {"l1", "l2", nullptr}, "list_has_all(l1, l2)" },
+    {DEFAULT_SCHEMA, "@>", {"l1", "l2", nullptr}, "list_has_all(l1, l2)" }, // "@>" is the operator for "list_has_all
+    {DEFAULT_SCHEMA, "<@", {"l1", "l2", nullptr}, "list_has_all(l2, l1)" }, // "<@" is the operator for "list_has_all
+
 	// algebraic list aggregates
 	{DEFAULT_SCHEMA, "list_avg", {"l", nullptr}, "list_aggr(l, 'avg')"},
 	{DEFAULT_SCHEMA, "list_var_samp", {"l", nullptr}, "list_aggr(l, 'var_samp')"},
@@ -205,6 +218,9 @@ unique_ptr<CatalogEntry> DefaultFunctionGenerator::CreateDefaultEntry(ClientCont
 vector<string> DefaultFunctionGenerator::GetDefaultEntries() {
 	vector<string> result;
 	for (idx_t index = 0; internal_macros[index].name != nullptr; index++) {
+		if (StringUtil::Lower(internal_macros[index].name) != internal_macros[index].name) {
+			throw InternalException("Default macro name %s should be lowercase", internal_macros[index].name);
+		}
 		if (internal_macros[index].schema == schema.name) {
 			result.emplace_back(internal_macros[index].name);
 		}