duckdb 1.1.4-dev11.0 → 1.1.4-dev14.0

package/src/duckdb/src/execution/operator/persistent/physical_delete.cpp

@@ -47,7 +47,10 @@ class DeleteLocalState : public LocalSinkState {
 public:
 	DeleteLocalState(ClientContext &context, TableCatalogEntry &table,
 	                 const vector<unique_ptr<BoundConstraint>> &bound_constraints) {
-		delete_chunk.Initialize(Allocator::Get(context), table.GetTypes());
+		const auto &types = table.GetTypes();
+		auto initialize = vector<bool>(types.size(), false);
+		delete_chunk.Initialize(Allocator::Get(context), types, initialize);
+
 		auto &storage = table.GetStorage();
 		delete_state = storage.InitializeDelete(table, context, bound_constraints);
 	}
@@ -64,34 +67,79 @@ SinkResultType PhysicalDelete::Sink(ExecutionContext &context, DataChunk &chunk,
 	auto &transaction = DuckTransaction::Get(context.client, table.db);
 	auto &row_ids = chunk.data[row_id_index];
 
-	vector<StorageIndex> column_ids;
-	for (idx_t i = 0; i < table.ColumnCount(); i++) {
-		column_ids.emplace_back(i);
-	};
-	auto fetch_state = ColumnFetchState();
-
 	lock_guard<mutex> delete_guard(g_state.delete_lock);
 	if (!return_chunk && !g_state.has_unique_indexes) {
 		g_state.deleted_count += table.Delete(*l_state.delete_state, context.client, row_ids, chunk.size());
 		return SinkResultType::NEED_MORE_INPUT;
 	}
 
-	// Fetch the to-be-deleted chunk.
+	auto types = table.GetTypes();
+	auto to_be_fetched = vector<bool>(types.size(), return_chunk);
+	vector<StorageIndex> column_ids;
+	vector<LogicalType> column_types;
+	if (return_chunk) {
+		// Fetch all columns.
+		column_types = types;
+		for (idx_t i = 0; i < table.ColumnCount(); i++) {
+			column_ids.emplace_back(i);
+		}
+
+	} else {
+		// Fetch only the required columns for updating the delete indexes.
+		auto &local_storage = LocalStorage::Get(context.client, table.db);
+		auto storage = local_storage.GetStorage(table);
+		unordered_set<column_t> indexed_column_id_set;
+		storage->delete_indexes.Scan([&](Index &index) {
+			if (!index.IsBound() || !index.IsUnique()) {
+				return false;
+			}
+			auto &set = index.GetColumnIdSet();
+			indexed_column_id_set.insert(set.begin(), set.end());
+			return false;
+		});
+		for (auto &col : indexed_column_id_set) {
+			column_ids.emplace_back(col);
+		}
+		sort(column_ids.begin(), column_ids.end());
+		for (auto &col : column_ids) {
+			auto i = col.GetPrimaryIndex();
+			to_be_fetched[i] = true;
+			column_types.push_back(types[i]);
+		}
+	}
+
 	l_state.delete_chunk.Reset();
 	row_ids.Flatten(chunk.size());
-	table.Fetch(transaction, l_state.delete_chunk, column_ids, row_ids, chunk.size(), fetch_state);
+
+	// Fetch the to-be-deleted chunk.
+	DataChunk fetch_chunk;
+	fetch_chunk.Initialize(Allocator::Get(context.client), column_types, chunk.size());
+	auto fetch_state = ColumnFetchState();
+	table.Fetch(transaction, fetch_chunk, column_ids, row_ids, chunk.size(), fetch_state);
+
+	// Reference the necessary columns of the fetch_chunk.
+	idx_t fetch_idx = 0;
+	for (idx_t i = 0; i < table.ColumnCount(); i++) {
+		if (to_be_fetched[i]) {
+			l_state.delete_chunk.data[i].Reference(fetch_chunk.data[fetch_idx++]);
+			continue;
+		}
+		l_state.delete_chunk.data[i].Reference(Value(types[i]));
+	}
+	l_state.delete_chunk.SetCardinality(fetch_chunk);
 
 	// Append the deleted row IDs to the delete indexes.
 	// If we only delete local row IDs, then the delete_chunk is empty.
 	if (g_state.has_unique_indexes && l_state.delete_chunk.size() != 0) {
 		auto &local_storage = LocalStorage::Get(context.client, table.db);
 		auto storage = local_storage.GetStorage(table);
+		IndexAppendInfo index_append_info(IndexAppendMode::IGNORE_DUPLICATES, nullptr);
 		storage->delete_indexes.Scan([&](Index &index) {
 			if (!index.IsBound() || !index.IsUnique()) {
 				return false;
 			}
 			auto &bound_index = index.Cast<BoundIndex>();
-			auto error = bound_index.Append(l_state.delete_chunk, row_ids);
+			auto error = bound_index.Append(l_state.delete_chunk, row_ids, index_append_info);
 			if (error.HasError()) {
 				throw InternalException("failed to update delete ART in physical delete: ", error.Message());
 			}

package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp

@@ -229,6 +229,7 @@ static void CreateUpdateChunk(ExecutionContext &context, DataChunk &chunk, Table
 	auto &do_update_condition = op.do_update_condition;
 	auto &set_types = op.set_types;
 	auto &set_expressions = op.set_expressions;
+
 	// Check the optional condition for the DO UPDATE clause, to filter which rows will be updated
 	if (do_update_condition) {
 		DataChunk do_update_filter_result;
@@ -252,19 +253,28 @@ static void CreateUpdateChunk(ExecutionContext &context, DataChunk &chunk, Table
 			chunk.SetCardinality(selection.Count());
 			// Also apply this Slice to the to-update row_ids
 			row_ids.Slice(selection.Selection(), selection.Count());
+			row_ids.Flatten(selection.Count());
 		}
 	}
 
-	// Execute the SET expressions
-	update_chunk.Initialize(context.client, set_types);
+	if (chunk.size() == 0) {
+		auto initialize = vector<bool>(set_types.size(), false);
+		update_chunk.Initialize(context.client, set_types, initialize, chunk.size());
+		update_chunk.SetCardinality(chunk);
+		return;
+	}
+
+	// Execute the SET expressions.
+	update_chunk.Initialize(context.client, set_types, chunk.size());
 	ExpressionExecutor executor(context.client, set_expressions);
 	executor.Execute(chunk, update_chunk);
 	update_chunk.SetCardinality(chunk);
 }
 
 template <bool GLOBAL>
-static idx_t PerformOnConflictAction(InsertLocalState &lstate, ExecutionContext &context, DataChunk &chunk,
-                                     TableCatalogEntry &table, Vector &row_ids, const PhysicalInsert &op) {
+static idx_t PerformOnConflictAction(InsertLocalState &lstate, InsertGlobalState &gstate, ExecutionContext &context,
+                                     DataChunk &chunk, TableCatalogEntry &table, Vector &row_ids,
+                                     const PhysicalInsert &op) {
 	// Early-out, if we do nothing on conflicting rows.
 	if (op.action_type == OnConflictAction::NOTHING) {
 		return 0;
@@ -275,15 +285,8 @@ static idx_t PerformOnConflictAction(InsertLocalState &lstate, ExecutionContext
 	CreateUpdateChunk(context, chunk, table, row_ids, update_chunk, op);
 	auto &data_table = table.GetStorage();
 
-	// Perform the UPDATE on the (global) storage.
-	if (!op.update_is_del_and_insert) {
-		if (GLOBAL) {
-			auto update_state = data_table.InitializeUpdate(table, context.client, op.bound_constraints);
-			data_table.Update(*update_state, context.client, row_ids, set_columns, update_chunk);
-			return update_chunk.size();
-		}
-		auto &local_storage = LocalStorage::Get(context.client, data_table.db);
-		local_storage.Update(data_table, row_ids, set_columns, update_chunk);
+	if (update_chunk.size() == 0) {
+		// Nothing to do
 		return update_chunk.size();
 	}
 
@@ -297,6 +300,27 @@ static idx_t PerformOnConflictAction(InsertLocalState &lstate, ExecutionContext
 		append_chunk.data[set_columns[i].index].Reference(update_chunk.data[i]);
 	}
 
+	// Perform the UPDATE on the (global) storage.
+	if (!op.update_is_del_and_insert) {
+		if (!op.parallel && op.return_chunk) {
+			gstate.return_collection.Append(append_chunk);
+		}
+
+		if (GLOBAL) {
+			auto update_state = data_table.InitializeUpdate(table, context.client, op.bound_constraints);
+			data_table.Update(*update_state, context.client, row_ids, set_columns, update_chunk);
+			return update_chunk.size();
+		}
+		auto &local_storage = LocalStorage::Get(context.client, data_table.db);
+		if (gstate.initialized) {
+			// Flush the data first, it might be referenced by the Update
+			data_table.FinalizeLocalAppend(gstate.append_state);
+			gstate.initialized = false;
+		}
+		local_storage.Update(data_table, row_ids, set_columns, update_chunk);
+		return update_chunk.size();
+	}
+
 	if (GLOBAL) {
 		auto &delete_state = lstate.GetDeleteState(data_table, table, context.client);
 		data_table.Delete(delete_state, context.client, row_ids, update_chunk.size());
@@ -305,6 +329,9 @@ static idx_t PerformOnConflictAction(InsertLocalState &lstate, ExecutionContext
 		local_storage.Delete(data_table, row_ids, update_chunk.size());
 	}
 
+	if (!op.parallel && op.return_chunk) {
+		gstate.return_collection.Append(append_chunk);
+	}
 	data_table.LocalAppend(table, context.client, append_chunk, op.bound_constraints, row_ids, append_chunk);
 	return update_chunk.size();
 }
@@ -357,8 +384,8 @@ static void CheckDistinctnessInternal(ValidityMask &valid, vector<reference<Vect
 	}
 }
 
-void PrepareSortKeys(DataChunk &input, unordered_map<column_t, unique_ptr<Vector>> &sort_keys,
-                     const unordered_set<column_t> &column_ids) {
+static void PrepareSortKeys(DataChunk &input, unordered_map<column_t, unique_ptr<Vector>> &sort_keys,
+                            const unordered_set<column_t> &column_ids) {
 	OrderModifiers order_modifiers(OrderType::ASCENDING, OrderByNullType::NULLS_LAST);
 	for (auto &it : column_ids) {
 		auto &sort_key = sort_keys[it];
@@ -440,7 +467,7 @@ static void VerifyOnConflictCondition(ExecutionContext &context, DataChunk &comb
 
 template <bool GLOBAL>
 static idx_t HandleInsertConflicts(TableCatalogEntry &table, ExecutionContext &context, InsertLocalState &lstate,
-                                   DataChunk &tuples, const PhysicalInsert &op) {
+                                   InsertGlobalState &gstate, DataChunk &tuples, const PhysicalInsert &op) {
 	auto &types_to_fetch = op.types_to_fetch;
 	auto &on_conflict_condition = op.on_conflict_condition;
 	auto &conflict_target = op.conflict_target;
@@ -510,7 +537,7 @@ static idx_t HandleInsertConflicts(TableCatalogEntry &table, ExecutionContext &c
 		RegisterUpdatedRows(lstate, row_ids, combined_chunk.size());
 	}
 
-	affected_tuples += PerformOnConflictAction<GLOBAL>(lstate, context, combined_chunk, table, row_ids, op);
+	affected_tuples += PerformOnConflictAction<GLOBAL>(lstate, gstate, context, combined_chunk, table, row_ids, op);
 
 	// Remove the conflicting tuples from the insert chunk
 	SelectionVector sel_vec(tuples.size());
@@ -590,6 +617,11 @@ idx_t PhysicalInsert::OnConflictHandling(TableCatalogEntry &table, ExecutionCont
 			}
 		}
 		if (action_type == OnConflictAction::UPDATE) {
+			if (do_update_condition) {
+				//! See https://github.com/duckdblabs/duckdb-internal/issues/4090 for context
+				throw NotImplementedException("Inner conflicts detected with a conditional DO UPDATE on-conflict "
+				                              "action, not fully implemented yet");
+			}
 			ManagedSelection last_occurrences(last_occurrences_of_conflict.size());
 			for (auto &idx : last_occurrences_of_conflict) {
 				last_occurrences.Append(idx);
@@ -607,9 +639,9 @@ idx_t PhysicalInsert::OnConflictHandling(TableCatalogEntry &table, ExecutionCont
 	// Check whether any conflicts arise, and if they all meet the conflict_target + condition
 	// If that's not the case - We throw the first error
 	idx_t updated_tuples = 0;
-	updated_tuples += HandleInsertConflicts<true>(table, context, lstate, lstate.insert_chunk, *this);
+	updated_tuples += HandleInsertConflicts<true>(table, context, lstate, gstate, lstate.insert_chunk, *this);
 	// Also check the transaction-local storage+ART so we can detect conflicts within this transaction
-	updated_tuples += HandleInsertConflicts<false>(table, context, lstate, lstate.insert_chunk, *this);
+	updated_tuples += HandleInsertConflicts<false>(table, context, lstate, gstate, lstate.insert_chunk, *this);
 
 	return updated_tuples;
 }
@@ -628,31 +660,22 @@ SinkResultType PhysicalInsert::Sink(ExecutionContext &context, DataChunk &chunk,
 			gstate.initialized = true;
 		}
 
-		if (action_type != OnConflictAction::NOTHING && return_chunk) {
-			// If the action is UPDATE or REPLACE, we will always create either an APPEND or an INSERT
-			// for NOTHING we don't create either an APPEND or an INSERT for the tuple
-			// so it should not be added to the RETURNING chunk
-			gstate.return_collection.Append(lstate.insert_chunk);
-		}
 		idx_t updated_tuples = OnConflictHandling(table, context, gstate, lstate);
-		if (action_type == OnConflictAction::NOTHING && return_chunk) {
-			// Because we didn't add to the RETURNING chunk yet
-			// we add the tuples that did not get filtered out now
-			gstate.return_collection.Append(lstate.insert_chunk);
-		}
+
 		gstate.insert_count += lstate.insert_chunk.size();
 		gstate.insert_count += updated_tuples;
+		if (!parallel && return_chunk) {
+			gstate.return_collection.Append(lstate.insert_chunk);
+		}
 		storage.LocalAppend(gstate.append_state, context.client, lstate.insert_chunk, true);
 		if (action_type == OnConflictAction::UPDATE && lstate.update_chunk.size() != 0) {
-			// Flush the append so we can target the data we just appended with the update
-			storage.FinalizeLocalAppend(gstate.append_state);
-			gstate.initialized = false;
-			(void)HandleInsertConflicts<true>(table, context, lstate, lstate.update_chunk, *this);
-			(void)HandleInsertConflicts<false>(table, context, lstate, lstate.update_chunk, *this);
+			(void)HandleInsertConflicts<true>(table, context, lstate, gstate, lstate.update_chunk, *this);
+			(void)HandleInsertConflicts<false>(table, context, lstate, gstate, lstate.update_chunk, *this);
 			// All of the tuples should have been turned into an update, leaving the chunk empty afterwards
 			D_ASSERT(lstate.update_chunk.size() == 0);
 		}
 	} else {
+		//! FIXME: can't we enable this by using a BatchedDataCollection ?
 		D_ASSERT(!return_chunk);
 		// parallel append
 		if (!lstate.local_collection) {

package/src/duckdb/src/execution/operator/schema/physical_create_art_index.cpp

@@ -88,7 +88,8 @@ SinkResultType PhysicalCreateARTIndex::SinkUnsorted(OperatorSinkInput &input) co
 	// Insert each key and its corresponding row ID.
 	for (idx_t i = 0; i < row_count; i++) {
 		auto status = art.tree.GetGateStatus();
-		auto conflict_type = art.Insert(art.tree, l_state.keys[i], 0, l_state.row_ids[i], status, nullptr);
+		auto conflict_type =
+		    art.Insert(art.tree, l_state.keys[i], 0, l_state.row_ids[i], status, nullptr, IndexAppendMode::DEFAULT);
 		D_ASSERT(conflict_type != ARTConflictType::TRANSACTION);
 		if (conflict_type == ARTConflictType::CONSTRAINT) {
 			throw ConstraintException("Data contains duplicates on indexed column(s)");

package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp

@@ -97,6 +97,7 @@ public:
 	void SetRadixBits(const idx_t &radix_bits_p);
 	bool SetRadixBitsToExternal();
 	idx_t GetRadixBits() const;
+	idx_t GetExternalRadixBits() const;
 
 private:
 	void SetRadixBitsInternal(idx_t radix_bits_p, bool external);
@@ -210,13 +211,13 @@ RadixHTGlobalSinkState::RadixHTGlobalSinkState(ClientContext &context_p, const R
 	auto tuples_per_block = block_alloc_size / radix_ht.GetLayout().GetRowWidth();
 	idx_t ht_count =
 	    LossyNumericCast<idx_t>(static_cast<double>(config.sink_capacity) / GroupedAggregateHashTable::LOAD_FACTOR);
-	auto num_partitions = RadixPartitioning::NumberOfPartitions(config.GetRadixBits());
+	auto num_partitions = RadixPartitioning::NumberOfPartitions(config.GetExternalRadixBits());
 	auto count_per_partition = ht_count / num_partitions;
 	auto blocks_per_partition = (count_per_partition + tuples_per_block) / tuples_per_block + 1;
 	if (!radix_ht.GetLayout().AllConstant()) {
 		blocks_per_partition += 2;
 	}
-	auto ht_size = blocks_per_partition * block_alloc_size + config.sink_capacity * sizeof(ht_entry_t);
+	auto ht_size = num_partitions * blocks_per_partition * block_alloc_size + config.sink_capacity * sizeof(ht_entry_t);
 
 	// This really is the minimum reservation that we can do
 	auto num_threads = NumericCast<idx_t>(TaskScheduler::GetScheduler(context).NumberOfThreads());
@@ -280,13 +281,17 @@ idx_t RadixHTConfig::GetRadixBits() const {
 	return sink_radix_bits;
 }
 
+idx_t RadixHTConfig::GetExternalRadixBits() const {
+	return MAXIMUM_FINAL_SINK_RADIX_BITS;
+}
+
 void RadixHTConfig::SetRadixBitsInternal(const idx_t radix_bits_p, bool external) {
-	if (sink_radix_bits >= radix_bits_p || sink.any_combined) {
+	if (sink_radix_bits > radix_bits_p || sink.any_combined) {
 		return;
 	}
 
 	auto guard = sink.Lock();
-	if (sink_radix_bits >= radix_bits_p || sink.any_combined) {
+	if (sink_radix_bits > radix_bits_p || sink.any_combined) {
 		return;
 	}
 

package/src/duckdb/src/execution/sample/reservoir_sample.cpp

@@ -50,7 +50,10 @@ ReservoirSample::ReservoirSample(idx_t sample_count, unique_ptr<ReservoirChunk>
 	if (reservoir_chunk) {
 		this->reservoir_chunk = std::move(reservoir_chunk);
 		sel_size = this->reservoir_chunk->chunk.size();
-		sel = SelectionVector(0, sel_size);
+		sel = SelectionVector(FIXED_SAMPLE_SIZE);
+		for (idx_t i = 0; i < sel_size; i++) {
+			sel.set_index(i, i);
+		}
 		ExpandSerializedSample();
 	}
 	stats_sample = true;
@@ -225,10 +228,6 @@ vector<uint32_t> ReservoirSample::GetRandomizedVector(uint32_t range, uint32_t s
 	for (uint32_t i = 0; i < range; i++) {
 		ret.push_back(i);
 	}
-	if (size == FIXED_SAMPLE_SIZE) {
-		std::shuffle(ret.begin(), ret.end(), base_reservoir_sample->random);
-		return ret;
-	}
 	for (uint32_t i = 0; i < size; i++) {
 		uint32_t random_shuffle = base_reservoir_sample->random.NextRandomInteger32(i, range);
 		if (random_shuffle == i) {
@@ -305,6 +304,7 @@ void ReservoirSample::SimpleMerge(ReservoirSample &other) {
 	auto offset = reservoir_chunk->chunk.size();
 	for (idx_t i = keep_from_this; i < size_after_merge; i++) {
 		if (i >= GetActiveSampleCount()) {
+			D_ASSERT(sel_size >= GetActiveSampleCount());
 			sel.set_index(GetActiveSampleCount(), offset);
 			sel_size += 1;
 		} else {
@@ -551,7 +551,7 @@ void ReservoirSample::ExpandSerializedSample() {
 }
 
 idx_t ReservoirSample::GetReservoirChunkCapacity() const {
-	return sample_count + (FIXED_SAMPLE_SIZE_MULTIPLIER * FIXED_SAMPLE_SIZE);
+	return sample_count + (FIXED_SAMPLE_SIZE_MULTIPLIER * MinValue<idx_t>(sample_count, FIXED_SAMPLE_SIZE));
 }
 
 idx_t ReservoirSample::FillReservoir(DataChunk &chunk) {
@@ -749,6 +749,7 @@ void ReservoirSample::AddToReservoir(DataChunk &chunk) {
 
 	if (chunk_sel.size == 0) {
 		// not adding any samples
+		base_reservoir_sample->num_entries_seen_total += chunk.size();
 		return;
 	}
 	idx_t size = chunk_sel.size;

package/src/duckdb/src/function/compression_config.cpp

@@ -65,6 +65,10 @@ static optional_ptr<CompressionFunction> LoadCompressionFunction(CompressionFunc
 
 static void TryLoadCompression(DBConfig &config, vector<reference<CompressionFunction>> &result, CompressionType type,
                                const PhysicalType physical_type) {
+	if (config.options.disabled_compression_methods.find(type) != config.options.disabled_compression_methods.end()) {
+		// explicitly disabled
+		return;
+	}
 	auto function = config.GetCompressionFunction(type, physical_type);
 	if (!function) {
 		return;

package/src/duckdb/src/function/function_binder.cpp

@@ -457,7 +457,7 @@ unique_ptr<Expression> FunctionBinder::BindScalarFunction(ScalarFunction bound_f
 	                                                      std::move(children), std::move(bind_info), is_operator);
 	if (result_func->function.bind_expression) {
 		// if a bind_expression callback is registered - call it and emit the resulting expression
-		FunctionBindExpressionInput input(context, result_func->bind_info.get(), *result_func);
+		FunctionBindExpressionInput input(context, result_func->bind_info.get(), result_func->children);
 		result = result_func->function.bind_expression(input);
 	}
 	if (!result) {

package/src/duckdb/src/function/scalar/system/write_log.cpp

@@ -114,11 +114,11 @@ static void WriteLogValues(T &LogSource, LogLevel level, const string_t *data, c
                            const string &type) {
 	if (!type.empty()) {
 		for (idx_t i = 0; i < size; i++) {
-			Logger::Log(type.c_str(), LogSource, level, data[sel->get_index(i)]);
+			DUCKDB_LOG(LogSource, type.c_str(), level, data[sel->get_index(i)]);
 		}
 	} else {
 		for (idx_t i = 0; i < size; i++) {
-			Logger::Log(LogSource, level, data[sel->get_index(i)]);
+			DUCKDB_LOG(LogSource, type.c_str(), level, data[sel->get_index(i)]);
 		}
 	}
 }

package/src/duckdb/src/function/table/arrow/arrow_duck_schema.cpp

@@ -56,7 +56,7 @@ void ArrowType::ThrowIfInvalid() const {
 	}
 }
 
-unique_ptr<ArrowType> ArrowType::GetTypeFromFormat(DBConfig &config, ArrowSchema &schema, string &format) {
+unique_ptr<ArrowType> ArrowType::GetTypeFromFormat(string &format) {
 	if (format == "n") {
 		return make_uniq<ArrowType>(LogicalType::SQLNULL);
 	} else if (format == "b") {
@@ -179,6 +179,14 @@ unique_ptr<ArrowType> ArrowType::GetTypeFromFormat(DBConfig &config, ArrowSchema
 		}
 		return make_uniq<ArrowType>(LogicalType::TIMESTAMP_TZ, std::move(type_info));
 	}
+	return nullptr;
+}
+
+unique_ptr<ArrowType> ArrowType::GetTypeFromFormat(DBConfig &config, ArrowSchema &schema, string &format) {
+	auto type = GetTypeFromFormat(format);
+	if (type) {
+		return type;
+	}
 	if (format == "+l") {
 		return CreateListType(config, *schema.children[0], ArrowVariableSizeType::NORMAL, false);
 	} else if (format == "+L") {
@@ -361,8 +369,13 @@ unique_ptr<ArrowType> ArrowType::GetTypeFromSchema(DBConfig &config, ArrowSchema
 	auto arrow_type = GetTypeFromFormat(config, schema, format);
 	if (schema_metadata.HasExtension()) {
 		auto extension_info = schema_metadata.GetExtensionInfo(string(format));
-		arrow_type->extension_data = config.GetArrowExtension(extension_info).GetTypeExtension();
+		if (config.HasArrowExtension(extension_info)) {
+			auto extension = config.GetArrowExtension(extension_info);
+			arrow_type = extension.GetType(schema, schema_metadata);
+			arrow_type->extension_data = extension.GetTypeExtension();
+		}
 	}
+
 	return arrow_type;
 }
 

package/src/duckdb/src/function/table/arrow_conversion.cpp

@@ -118,7 +118,8 @@ static void ColumnArrowToDuckDBRunEndEncoded(Vector &vector, const ArrowArray &a
 
 static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArrayScanState &array_state, idx_t size,
                                 const ArrowType &arrow_type, int64_t nested_offset = -1,
-                                ValidityMask *parent_mask = nullptr, uint64_t parent_offset = 0);
+                                ValidityMask *parent_mask = nullptr, uint64_t parent_offset = 0,
+                                bool ignore_extensions = false);
 
 static void ColumnArrowToDuckDBDictionary(Vector &vector, ArrowArray &array, ArrowArrayScanState &array_state,
                                           idx_t size, const ArrowType &arrow_type, int64_t nested_offset = -1,
@@ -765,17 +766,15 @@ static void ColumnArrowToDuckDBRunEndEncoded(Vector &vector, const ArrowArray &a
 
 static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArrayScanState &array_state, idx_t size,
                                 const ArrowType &arrow_type, int64_t nested_offset, ValidityMask *parent_mask,
-                                uint64_t parent_offset) {
+                                uint64_t parent_offset, bool ignore_extensions) {
 	auto &scan_state = array_state.state;
 	D_ASSERT(!array.dictionary);
-	if (arrow_type.HasExtension()) {
+	if (!ignore_extensions && arrow_type.HasExtension()) {
 		if (arrow_type.extension_data->arrow_to_duckdb) {
-			// We allocate with the internal type, and cast to the end result
+			// Convert the storage and then call the cast function
 			Vector input_data(arrow_type.extension_data->GetInternalType());
-			// FIXME do we need this?
-			auto input_arrow_type = ArrowType(arrow_type.extension_data->GetInternalType());
-			ColumnArrowToDuckDB(input_data, array, array_state, size, input_arrow_type, nested_offset, parent_mask,
-			                    parent_offset);
+			ColumnArrowToDuckDB(input_data, array, array_state, size, arrow_type, nested_offset, parent_mask,
+			                    parent_offset, /*ignore_extensions*/ true);
 			arrow_type.extension_data->arrow_to_duckdb(array_state.context, input_data, vector, size);
 			return;
 		}
@@ -1105,7 +1104,7 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArraySca
 				break;
 			case ArrowArrayPhysicalType::DEFAULT:
 				ColumnArrowToDuckDB(child_entry, child_array, child_state, size, child_type, nested_offset,
-				                    &struct_validity_mask, NumericCast<uint64_t>(array.offset));
+				                    &struct_validity_mask, NumericCast<uint64_t>(array.offset), false);
 				break;
 			default:
 				throw NotImplementedException("ArrowArrayPhysicalType not recognized");
@@ -1138,7 +1137,8 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArraySca
 				ColumnArrowToDuckDBRunEndEncoded(child, child_array, child_state, size, child_type);
 				break;
 			case ArrowArrayPhysicalType::DEFAULT:
-				ColumnArrowToDuckDB(child, child_array, child_state, size, child_type, nested_offset, &validity_mask);
+				ColumnArrowToDuckDB(child, child_array, child_state, size, child_type, nested_offset, &validity_mask,
+				                    false);
 				break;
 			default:
 				throw NotImplementedException("ArrowArrayPhysicalType not recognized");

package/src/duckdb/src/function/table/copy_csv.cpp

@@ -97,7 +97,7 @@ void BaseCSVData::Finalize() {
 			const char escape = options.dialect_options.state_machine_options.escape.GetValue();
 			// Allow nullstr to be escape character + some non-special character, e.g., "\N" (MySQL default).
 			// In this case, only unquoted occurrences of the nullstr will be recognized as null values.
-			if (options.dialect_options.state_machine_options.rfc_4180 == false && null_str.size() == 2 &&
+			if (options.dialect_options.state_machine_options.strict_mode == false && null_str.size() == 2 &&
 			    null_str[0] == escape && null_str[1] != '\0') {
 				continue;
 			}
@@ -371,7 +371,7 @@ static void WriteQuotedString(WriteStream &writer, WriteCSVData &csv_data, const
 struct LocalWriteCSVData : public LocalFunctionData {
 public:
 	LocalWriteCSVData(ClientContext &context, vector<unique_ptr<Expression>> &expressions)
-	    : executor(context, expressions) {
+	    : executor(context, expressions), stream(Allocator::Get(context)) {
 	}
 
 public:
@@ -451,7 +451,7 @@ static unique_ptr<GlobalFunctionData> WriteCSVInitializeGlobal(ClientContext &co
 	}
 
 	if (!(options.dialect_options.header.IsSetByUser() && !options.dialect_options.header.GetValue())) {
-		MemoryStream stream;
+		MemoryStream stream(Allocator::Get(context));
 		// write the header line to the file
 		for (idx_t i = 0; i < csv_data.options.name_list.size(); i++) {
 			if (i != 0) {
@@ -554,7 +554,7 @@ void WriteCSVFinalize(ClientContext &context, FunctionData &bind_data, GlobalFun
 	auto &csv_data = bind_data.Cast<WriteCSVData>();
 	auto &options = csv_data.options;
 
-	MemoryStream stream;
+	MemoryStream stream(Allocator::Get(context));
 	if (!options.suffix.empty()) {
 		stream.WriteData(const_data_ptr_cast(options.suffix.c_str()), options.suffix.size());
 	} else if (global_state.written_anything) {
@@ -582,6 +582,9 @@ CopyFunctionExecutionMode WriteCSVExecutionMode(bool preserve_insertion_order, b
 // Prepare Batch
 //===--------------------------------------------------------------------===//
 struct WriteCSVBatchData : public PreparedBatchData {
+	explicit WriteCSVBatchData(Allocator &allocator) : stream(allocator) {
+	}
+
 	//! The thread-local buffer to write data into
 	MemoryStream stream;
 };
@@ -603,7 +606,7 @@ unique_ptr<PreparedBatchData> WriteCSVPrepareBatch(ClientContext &context, Funct
 
 	// write CSV chunks to the batch data
 	bool written_anything = false;
-	auto batch = make_uniq<WriteCSVBatchData>();
+	auto batch = make_uniq<WriteCSVBatchData>(Allocator::Get(context));
 	for (auto &chunk : collection->Chunks()) {
 		WriteCSVChunkInternal(context, bind_data, cast_chunk, batch->stream, chunk, written_anything, executor);
 	}

package/src/duckdb/src/function/table/read_csv.cpp

@@ -62,6 +62,8 @@ void SchemaDiscovery(ClientContext &context, ReadCSVData &result, CSVReaderOptio
 	options.file_path = file_paths[current_file];
 
 	result.buffer_manager = make_shared_ptr<CSVBufferManager>(context, options, options.file_path, 0, false);
+	idx_t only_header_or_empty_files = 0;
+
 	{
 		CSVSniffer sniffer(options, result.buffer_manager, CSVStateMachineCache::Get(context));
 		auto sniffer_result = sniffer.SniffCSV();
@@ -71,14 +73,17 @@ void SchemaDiscovery(ClientContext &context, ReadCSVData &result, CSVReaderOptio
 		schemas.emplace_back(sniffer_result.names, sniffer_result.return_types, file_paths[0], rows_read,
 		                     result.buffer_manager->GetBuffer(0)->actual_size == 0);
 		total_number_of_rows += sniffer.LinesSniffed();
+		current_file++;
+		if (sniffer.EmptyOrOnlyHeader()) {
+			only_header_or_empty_files++;
+		}
 	}
 
 	// We do a copy of the options to not pollute the options of the first file.
 	constexpr idx_t max_files_to_sniff = 10;
 	idx_t files_to_sniff = file_paths.size() > max_files_to_sniff ? max_files_to_sniff : file_paths.size();
-	while (total_number_of_rows < required_number_of_lines && current_file + 1 < files_to_sniff) {
+	while (total_number_of_rows < required_number_of_lines && current_file < files_to_sniff) {
 		auto option_copy = option_og;
-		current_file++;
 		option_copy.file_path = file_paths[current_file];
 		auto buffer_manager =
 		    make_shared_ptr<CSVBufferManager>(context, option_copy, option_copy.file_path, current_file, false);
@@ -94,6 +99,10 @@ void SchemaDiscovery(ClientContext &context, ReadCSVData &result, CSVReaderOptio
 			schemas.emplace_back(sniffer_result.names, sniffer_result.return_types, option_copy.file_path, rows_read);
 		}
 		total_number_of_rows += sniffer.LinesSniffed();
+		if (sniffer.EmptyOrOnlyHeader()) {
+			only_header_or_empty_files++;
+		}
+		current_file++;
 	}
 
 	// We might now have multiple schemas, we need to go through them to define the one true schema
@@ -115,6 +124,13 @@ void SchemaDiscovery(ClientContext &context, ReadCSVData &result, CSVReaderOptio
 		names = best_schema.GetNames();
 		return_types = best_schema.GetTypes();
 	}
+	if (only_header_or_empty_files == current_file && !options.columns_set) {
+		for (auto &type : return_types) {
+			D_ASSERT(type.id() == LogicalTypeId::BOOLEAN);
+			// we default to varchar if all files are empty or only have a header after all the sniffing
+			type = LogicalType::VARCHAR;
+		}
+	}
 	result.csv_types = return_types;
 	result.csv_names = names;
 }
@@ -334,7 +350,7 @@ void ReadCSVTableFunction::ReadCSVAddNamedParameters(TableFunction &table_functi
 	table_function.named_parameters["column_names"] = LogicalType::LIST(LogicalType::VARCHAR);
 	table_function.named_parameters["comment"] = LogicalType::VARCHAR;
 	table_function.named_parameters["encoding"] = LogicalType::VARCHAR;
-	table_function.named_parameters["rfc_4180"] = LogicalType::BOOLEAN;
+	table_function.named_parameters["strict_mode"] = LogicalType::BOOLEAN;
 
 	MultiFileReader::AddParameters(table_function);
 }
@@ -358,7 +374,8 @@ void CSVComplexFilterPushdown(ClientContext &context, LogicalGet &get, FunctionD
 	    MultiFileReader().ComplexFilterPushdown(context, file_list, data.options.file_options, info, filters);
 	if (filtered_list) {
 		data.files = filtered_list->GetAllFiles();
-		MultiFileReader::PruneReaders(data, file_list);
+		SimpleMultiFileList simple_filtered_list(data.files);
+		MultiFileReader::PruneReaders(data, simple_filtered_list);
 	} else {
 		data.files = file_list.GetAllFiles();
 	}

package/src/duckdb/src/function/table/sniff_csv.cpp

@@ -152,6 +152,13 @@ static void CSVSniffFunction(ClientContext &context, TableFunctionInput &data_p,
 	}
 	CSVSniffer sniffer(sniffer_options, buffer_manager, CSVStateMachineCache::Get(context));
 	auto sniffer_result = sniffer.SniffCSV(data.force_match);
+	if (sniffer.EmptyOrOnlyHeader()) {
+		for (auto &type : sniffer_result.return_types) {
+			D_ASSERT(type.id() == LogicalTypeId::BOOLEAN);
+			// we default to varchar if all files are empty or only have a header after all the sniffing
+			type = LogicalType::VARCHAR;
+		}
+	}
 	string str_opt;
 	string separator = ", ";
 	// Set output