duckdb 1.1.0 → 1.1.1

package/src/duckdb/src/parser/parser.cpp

@@ -339,8 +339,25 @@ vector<SimplifiedToken> Parser::Tokenize(const string &query) {
 	return result;
 }
 
-bool Parser::IsKeyword(const string &text) {
-	return PostgresParser::IsKeyword(text);
+KeywordCategory ToKeywordCategory(duckdb_libpgquery::PGKeywordCategory type) {
+	switch (type) {
+	case duckdb_libpgquery::PGKeywordCategory::PG_KEYWORD_RESERVED:
+		return KeywordCategory::KEYWORD_RESERVED;
+	case duckdb_libpgquery::PGKeywordCategory::PG_KEYWORD_UNRESERVED:
+		return KeywordCategory::KEYWORD_UNRESERVED;
+	case duckdb_libpgquery::PGKeywordCategory::PG_KEYWORD_TYPE_FUNC:
+		return KeywordCategory::KEYWORD_TYPE_FUNC;
+	case duckdb_libpgquery::PGKeywordCategory::PG_KEYWORD_COL_NAME:
+		return KeywordCategory::KEYWORD_COL_NAME;
+	case duckdb_libpgquery::PGKeywordCategory::PG_KEYWORD_NONE:
+		return KeywordCategory::KEYWORD_NONE;
+	default:
+		throw InternalException("Unrecognized keyword category");
+	}
+}
+
+KeywordCategory Parser::IsKeyword(const string &text) {
+	return ToKeywordCategory(PostgresParser::IsKeyword(text));
 }
 
 vector<ParserKeyword> Parser::KeywordList() {
@@ -349,22 +366,7 @@ vector<ParserKeyword> Parser::KeywordList() {
 	for (auto &kw : keywords) {
 		ParserKeyword res;
 		res.name = kw.text;
-		switch (kw.category) {
-		case duckdb_libpgquery::PGKeywordCategory::PG_KEYWORD_RESERVED:
-			res.category = KeywordCategory::KEYWORD_RESERVED;
-			break;
-		case duckdb_libpgquery::PGKeywordCategory::PG_KEYWORD_UNRESERVED:
-			res.category = KeywordCategory::KEYWORD_UNRESERVED;
-			break;
-		case duckdb_libpgquery::PGKeywordCategory::PG_KEYWORD_TYPE_FUNC:
-			res.category = KeywordCategory::KEYWORD_TYPE_FUNC;
-			break;
-		case duckdb_libpgquery::PGKeywordCategory::PG_KEYWORD_COL_NAME:
-			res.category = KeywordCategory::KEYWORD_COL_NAME;
-			break;
-		default:
-			throw InternalException("Unrecognized keyword category");
-		}
+		res.category = ToKeywordCategory(kw.category);
 		result.push_back(res);
 	}
 	return result;

package/src/duckdb/src/parser/transform/statement/transform_select_node.cpp

@@ -91,9 +91,14 @@ unique_ptr<QueryNode> Transformer::TransformSelectInternal(duckdb_libpgquery::PG
 			if (!stmt.targetList) {
 				throw ParserException("SELECT clause without selection list");
 			}
-			// select list
-			TransformExpressionList(*stmt.targetList, result.select_list);
-			result.from_table = TransformFrom(stmt.fromClause);
+			// transform in the specified order to ensure positional parameters are correctly set
+			if (stmt.from_first) {
+				result.from_table = TransformFrom(stmt.fromClause);
+				TransformExpressionList(*stmt.targetList, result.select_list);
+			} else {
+				TransformExpressionList(*stmt.targetList, result.select_list);
+				result.from_table = TransformFrom(stmt.fromClause);
+			}
 		}
 
 		// where

package/src/duckdb/src/planner/binder/expression/bind_function_expression.cpp

@@ -270,6 +270,9 @@ BindResult ExpressionBinder::BindUnnest(FunctionExpression &expr, idx_t depth, b
 	return BindUnsupportedExpression(expr, depth, UnsupportedUnnestMessage());
 }
 
+void ExpressionBinder::ThrowIfUnnestInLambda(const ColumnBinding &column_binding) {
+}
+
 string ExpressionBinder::UnsupportedAggregateMessage() {
 	return "Aggregate functions are not supported here";
 }

package/src/duckdb/src/planner/binder/expression/bind_lambda.cpp

@@ -178,7 +178,7 @@ void ExpressionBinder::CaptureLambdaColumns(BoundLambdaExpression &bound_lambda_
                                             const LogicalType &list_child_type) {
 
 	if (expr->expression_class == ExpressionClass::BOUND_SUBQUERY) {
-		throw InvalidInputException("Subqueries are not supported in lambda expressions!");
+		throw BinderException("subqueries in lambda expressions are not supported");
 	}
 
 	// these are bound depth-first
@@ -195,6 +195,12 @@ void ExpressionBinder::CaptureLambdaColumns(BoundLambdaExpression &bound_lambda_
 	    expr->expression_class == ExpressionClass::BOUND_PARAMETER ||
 	    expr->expression_class == ExpressionClass::BOUND_LAMBDA_REF) {
 
+		if (expr->expression_class == ExpressionClass::BOUND_COLUMN_REF) {
+			// Search for UNNEST.
+			auto &column_binding = expr->Cast<BoundColumnRefExpression>().binding;
+			ThrowIfUnnestInLambda(column_binding);
+		}
+
 		// move the expr because we are going to replace it
 		auto original = std::move(expr);
 		unique_ptr<Expression> replacement;

package/src/duckdb/src/planner/binder/expression/bind_unnest_expression.cpp

@@ -43,6 +43,19 @@ unique_ptr<Expression> CreateBoundStructExtractIndex(ClientContext &context, uni
 	return std::move(result);
 }
 
+void SelectBinder::ThrowIfUnnestInLambda(const ColumnBinding &column_binding) {
+	// Extract the unnests and check if any match the column index.
+	for (auto &node_pair : node.unnests) {
+		auto &unnest_node = node_pair.second;
+
+		if (unnest_node.index == column_binding.table_index) {
+			if (column_binding.column_index < unnest_node.expressions.size()) {
+				throw BinderException("UNNEST in lambda expressions is not supported");
+			}
+		}
+	}
+}
+
 BindResult SelectBinder::BindUnnest(FunctionExpression &function, idx_t depth, bool root_expression) {
 	// bind the children of the function expression
 	if (depth > 0) {

package/src/duckdb/src/planner/binder/statement/bind_copy_database.cpp

@@ -54,7 +54,7 @@ unique_ptr<LogicalOperator> Binder::BindCopyDatabaseData(Catalog &source_catalog
 	ExportEntries entries;
 	PhysicalExport::ExtractEntries(context, source_schemas, entries);
 
-	unique_ptr<LogicalOperator> result;
+	vector<unique_ptr<LogicalOperator>> insert_nodes;
 	for (auto &table_ref : entries.tables) {
 		auto &table = table_ref.get().Cast<TableCatalogEntry>();
 		// generate the insert statement
@@ -82,17 +82,10 @@ unique_ptr<LogicalOperator> Binder::BindCopyDatabaseData(Catalog &source_catalog
 		insert_stmt.select_statement = std::move(select_stmt);
 		auto bound_insert = Bind(insert_stmt);
 		auto insert_plan = std::move(bound_insert.plan);
-		if (result) {
-			// use UNION ALL to combine the individual copy statements into a single node
-			auto copy_union =
-			    make_uniq<LogicalSetOperation>(GenerateTableIndex(), 1U, std::move(insert_plan), std::move(result),
-			                                   LogicalOperatorType::LOGICAL_UNION, true, false);
-			result = std::move(copy_union);
-		} else {
-			result = std::move(insert_plan);
-		}
+		insert_nodes.push_back(std::move(insert_plan));
 	}
-	if (!result) {
+	unique_ptr<LogicalOperator> result;
+	if (insert_nodes.empty()) {
 		vector<LogicalType> result_types;
 		result_types.push_back(LogicalType::BIGINT);
 		vector<unique_ptr<Expression>> expression_list;
@@ -101,6 +94,9 @@ unique_ptr<LogicalOperator> Binder::BindCopyDatabaseData(Catalog &source_catalog
 		expressions.push_back(std::move(expression_list));
 		result = make_uniq<LogicalExpressionGet>(GenerateTableIndex(), std::move(result_types), std::move(expressions));
 		result->children.push_back(make_uniq<LogicalDummyScan>(GenerateTableIndex()));
+	} else {
+		// use UNION ALL to combine the individual copy statements into a single node
+		result = UnionOperators(std::move(insert_nodes));
 	}
 	return result;
 }

package/src/duckdb/src/planner/binder/statement/bind_create.cpp

@@ -249,6 +249,13 @@ SchemaCatalogEntry &Binder::BindCreateFunctionInfo(CreateInfo &info) {
 	return BindCreateSchema(info);
 }
 
+static bool IsValidUserType(optional_ptr<CatalogEntry> entry) {
+	if (!entry) {
+		return false;
+	}
+	return entry->Cast<TypeCatalogEntry>().user_type.id() != LogicalTypeId::INVALID;
+}
+
 void Binder::BindLogicalType(LogicalType &type, optional_ptr<Catalog> catalog, const string &schema) {
 	if (type.id() == LogicalTypeId::LIST || type.id() == LogicalTypeId::MAP) {
 		auto child_type = ListType::GetChildType(type);
@@ -297,24 +304,34 @@ void Binder::BindLogicalType(LogicalType &type, optional_ptr<Catalog> catalog, c
 		type.SetModifiers(modifiers);
 	} else if (type.id() == LogicalTypeId::USER) {
 		auto user_type_name = UserType::GetTypeName(type);
+		auto user_type_schema = UserType::GetSchema(type);
 		auto user_type_mods = UserType::GetTypeModifiers(type);
 
 		bind_type_modifiers_function_t user_bind_modifiers_func = nullptr;
 
 		if (catalog) {
 			// The search order is:
-			// 1) In the same schema as the table
-			// 2) In the same catalog
-			// 3) System catalog
-			auto entry = entry_retriever.GetEntry(CatalogType::TYPE_ENTRY, *catalog, schema, user_type_name,
-			                                      OnEntryNotFound::RETURN_NULL);
-			if (!entry || entry->Cast<TypeCatalogEntry>().user_type.id() == LogicalTypeId::INVALID) {
+			// 1) In the explicitly set schema (my_schema.my_type)
+			// 2) In the same schema as the table
+			// 3) In the same catalog
+			// 4) System catalog
+
+			optional_ptr<CatalogEntry> entry = nullptr;
+			if (!user_type_schema.empty()) {
+				entry = entry_retriever.GetEntry(CatalogType::TYPE_ENTRY, *catalog, user_type_schema, user_type_name,
+				                                 OnEntryNotFound::RETURN_NULL);
+			}
+			if (!IsValidUserType(entry)) {
+				entry = entry_retriever.GetEntry(CatalogType::TYPE_ENTRY, *catalog, schema, user_type_name,
+				                                 OnEntryNotFound::RETURN_NULL);
+			}
+			if (!IsValidUserType(entry)) {
 				entry = entry_retriever.GetEntry(CatalogType::TYPE_ENTRY, *catalog, INVALID_SCHEMA, user_type_name,
 				                                 OnEntryNotFound::RETURN_NULL);
-				if (!entry || entry->Cast<TypeCatalogEntry>().user_type.id() == LogicalTypeId::INVALID) {
-					entry = entry_retriever.GetEntry(CatalogType::TYPE_ENTRY, INVALID_CATALOG, INVALID_SCHEMA,
-					                                 user_type_name, OnEntryNotFound::THROW_EXCEPTION);
-				}
+			}
+			if (!IsValidUserType(entry)) {
+				entry = entry_retriever.GetEntry(CatalogType::TYPE_ENTRY, INVALID_CATALOG, INVALID_SCHEMA,
+				                                 user_type_name, OnEntryNotFound::THROW_EXCEPTION);
 			}
 			auto &type_entry = entry->Cast<TypeCatalogEntry>();
 			type = type_entry.user_type;

package/src/duckdb/src/planner/binder/statement/bind_export.cpp

@@ -136,6 +136,27 @@ static unique_ptr<QueryNode> CreateSelectStatement(CopyStatement &stmt, child_li
 	return std::move(statement);
 }
 
+unique_ptr<LogicalOperator> Binder::UnionOperators(vector<unique_ptr<LogicalOperator>> nodes) {
+	if (nodes.empty()) {
+		return nullptr;
+	}
+	while (nodes.size() > 1) {
+		vector<unique_ptr<LogicalOperator>> new_nodes;
+		for (idx_t i = 0; i < nodes.size(); i += 2) {
+			if (i + 1 == nodes.size()) {
+				new_nodes.push_back(std::move(nodes[i]));
+			} else {
+				auto copy_union = make_uniq<LogicalSetOperation>(GenerateTableIndex(), 1U, std::move(nodes[i]),
+				                                                 std::move(nodes[i + 1]),
+				                                                 LogicalOperatorType::LOGICAL_UNION, true, false);
+				new_nodes.push_back(std::move(copy_union));
+			}
+		}
+		nodes = std::move(new_nodes);
+	}
+	return std::move(nodes[0]);
+}
+
 BoundStatement Binder::Bind(ExportStatement &stmt) {
 	// COPY TO a file
 	auto &config = DBConfig::GetConfig(context);
@@ -170,11 +191,11 @@ BoundStatement Binder::Bind(ExportStatement &stmt) {
 
 	// now generate the COPY statements for each of the tables
 	auto &fs = FileSystem::GetFileSystem(context);
-	unique_ptr<LogicalOperator> child_operator;
 
 	BoundExportData exported_tables;
 
 	unordered_set<string> table_name_index;
+	vector<unique_ptr<LogicalOperator>> export_nodes;
 	for (auto &t : tables) {
 		auto &table = t.get().Cast<TableCatalogEntry>();
 		auto info = make_uniq<CopyInfo>();
@@ -237,15 +258,9 @@ BoundStatement Binder::Bind(ExportStatement &stmt) {
 
 		auto plan = std::move(bound_statement.plan);
 
-		if (child_operator) {
-			// use UNION ALL to combine the individual copy statements into a single node
-			auto copy_union = make_uniq<LogicalSetOperation>(GenerateTableIndex(), 1U, std::move(child_operator),
-			                                                 std::move(plan), LogicalOperatorType::LOGICAL_UNION, true);
-			child_operator = std::move(copy_union);
-		} else {
-			child_operator = std::move(plan);
-		}
+		export_nodes.push_back(std::move(plan));
 	}
+	auto child_operator = UnionOperators(std::move(export_nodes));
 
 	// try to create the directory, if it doesn't exist yet
 	// a bit hacky to do it here, but we need to create the directory BEFORE the copy statements run

package/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp

@@ -76,9 +76,7 @@ void LogicalComparisonJoin::ExtractJoinConditions(
 		auto total_side = JoinSide::GetJoinSide(*expr, left_bindings, right_bindings);
 		if (total_side != JoinSide::BOTH) {
 			// join condition does not reference both sides, add it as filter under the join
-			// BUT don't push right side filters into AsOf because it is really a table lookup
-			// and we shouldn't remove anything from the table.
-			if (type == JoinType::LEFT && total_side == JoinSide::RIGHT && ref_type != JoinRefType::ASOF) {
+			if ((type == JoinType::LEFT || ref_type == JoinRefType::ASOF) && total_side == JoinSide::RIGHT) {
 				// filter is on RHS and the join is a LEFT OUTER join, we can push it in the right child
 				if (right_child->type != LogicalOperatorType::LOGICAL_FILTER) {
 					// not a filter yet, push a new empty filter

package/src/duckdb/src/planner/binder.cpp

@@ -6,6 +6,7 @@
 #include "duckdb/common/enum_util.hpp"
 #include "duckdb/common/helper.hpp"
 #include "duckdb/main/config.hpp"
+#include "duckdb/main/database.hpp"
 #include "duckdb/parser/expression/function_expression.hpp"
 #include "duckdb/parser/expression/subquery_expression.hpp"
 #include "duckdb/parser/parsed_expression_iterator.hpp"
@@ -21,7 +22,6 @@
 #include "duckdb/planner/operator/logical_sample.hpp"
 #include "duckdb/planner/query_node/list.hpp"
 #include "duckdb/planner/tableref/list.hpp"
-#include "duckdb/main/database.hpp"
 
 #include <algorithm>
 
@@ -134,10 +134,8 @@ BoundStatement Binder::BindWithCTE(T &statement) {
 		}
 		MoveCorrelatedExpressions(*tail.child_binder);
 
-		// extract operator below root operation
-		auto plan = std::move(bound_statement.plan->children[0]);
-		bound_statement.plan->children.clear();
-		bound_statement.plan->children.push_back(CreatePlan(*bound_cte, std::move(plan)));
+		auto plan = std::move(bound_statement.plan);
+		bound_statement.plan = CreatePlan(*bound_cte, std::move(plan));
 	} else {
 		bound_statement = Bind(statement.template Cast<T>());
 	}
@@ -344,7 +342,8 @@ unique_ptr<BoundQueryNode> Binder::BindNode(QueryNode &node) {
 
 BoundStatement Binder::Bind(QueryNode &node) {
 	BoundStatement result;
-	if (context.db->config.options.disabled_optimizers.find(OptimizerType::MATERIALIZED_CTE) ==
+	if (node.type != QueryNodeType::CTE_NODE && // Issue #13850 - Don't auto-materialize if users materialize (for now)
+	    context.db->config.options.disabled_optimizers.find(OptimizerType::MATERIALIZED_CTE) ==
 	        context.db->config.options.disabled_optimizers.end() &&
 	    context.config.enable_optimizer && OptimizeCTEs(node)) {
 		switch (node.type) {

package/src/duckdb/src/planner/expression/bound_cast_expression.cpp

@@ -170,6 +170,7 @@ bool BoundCastExpression::CastIsInvertible(const LogicalType &source_type, const
 		break;
 	case LogicalTypeId::VARCHAR:
 	case LogicalTypeId::BIT:
+	case LogicalTypeId::TIME_TZ:
 		return false;
 	default:
 		break;

package/src/duckdb/src/planner/expression_binder/select_binder.cpp

@@ -8,6 +8,15 @@ SelectBinder::SelectBinder(Binder &binder, ClientContext &context, BoundSelectNo
     : BaseSelectBinder(binder, context, node, info) {
 }
 
+unique_ptr<ParsedExpression> SelectBinder::GetSQLValueFunction(const string &column_name) {
+	auto alias_entry = node.bind_state.alias_map.find(column_name);
+	if (alias_entry != node.bind_state.alias_map.end()) {
+		// don't replace SQL value functions if they are in the alias map
+		return nullptr;
+	}
+	return ExpressionBinder::GetSQLValueFunction(column_name);
+}
+
 BindResult SelectBinder::BindColumnRef(unique_ptr<ParsedExpression> &expr_ptr, idx_t depth, bool root_expression) {
 	// first try to bind the column reference regularly
 	auto result = BaseSelectBinder::BindColumnRef(expr_ptr, depth, root_expression);

package/src/duckdb/src/planner/operator/logical_copy_to_file.cpp

@@ -65,7 +65,7 @@ void LogicalCopyToFile::Serialize(Serializer &serializer) const {
 	serializer.WriteProperty(213, "file_extension", file_extension);
 	serializer.WriteProperty(214, "rotate", rotate);
 	serializer.WriteProperty(215, "return_type", return_type);
-	serializer.WriteProperty(216, "write_partition_columns", write_partition_columns);
+	serializer.WritePropertyWithDefault(216, "write_partition_columns", write_partition_columns, true);
 }
 
 unique_ptr<LogicalOperator> LogicalCopyToFile::Deserialize(Deserializer &deserializer) {
@@ -109,7 +109,7 @@ unique_ptr<LogicalOperator> LogicalCopyToFile::Deserialize(Deserializer &deseria
 	auto rotate = deserializer.ReadPropertyWithExplicitDefault(214, "rotate", false);
 	auto return_type =
 	    deserializer.ReadPropertyWithExplicitDefault(215, "return_type", CopyFunctionReturnType::CHANGED_ROWS);
-	auto write_partition_columns = deserializer.ReadProperty<bool>(216, "write_partition_columns");
+	auto write_partition_columns = deserializer.ReadPropertyWithExplicitDefault(216, "write_partition_columns", true);
 
 	if (!has_serialize) {
 		// If not serialized, re-bind with the copy info

package/src/duckdb/src/planner/operator/logical_positional_join.cpp

@@ -4,6 +4,7 @@ namespace duckdb {
 
 LogicalPositionalJoin::LogicalPositionalJoin(unique_ptr<LogicalOperator> left, unique_ptr<LogicalOperator> right)
     : LogicalUnconditionalJoin(LogicalOperatorType::LOGICAL_POSITIONAL_JOIN, std::move(left), std::move(right)) {
+	SetEstimatedCardinality(MaxValue(children[0]->estimated_cardinality, children[1]->estimated_cardinality));
 }
 
 unique_ptr<LogicalOperator> LogicalPositionalJoin::Create(unique_ptr<LogicalOperator> left,

package/src/duckdb/src/storage/buffer/block_handle.cpp

@@ -49,7 +49,7 @@ BlockHandle::~BlockHandle() { // NOLINT: allow internal exceptions
 		D_ASSERT(memory_charge.size == 0);
 	}
 
-	block_manager.UnregisterBlock(block_id);
+	block_manager.UnregisterBlock(*this);
 }
 
 unique_ptr<Block> AllocateBlock(BlockManager &block_manager, unique_ptr<FileBuffer> reusable_buffer,
@@ -71,39 +71,36 @@ unique_ptr<Block> AllocateBlock(BlockManager &block_manager, unique_ptr<FileBuff
 	}
 }
 
-BufferHandle BlockHandle::LoadFromBuffer(shared_ptr<BlockHandle> &handle, data_ptr_t data,
-                                         unique_ptr<FileBuffer> reusable_buffer) {
-	D_ASSERT(handle->state != BlockState::BLOCK_LOADED);
+BufferHandle BlockHandle::LoadFromBuffer(data_ptr_t data, unique_ptr<FileBuffer> reusable_buffer) {
+	D_ASSERT(state != BlockState::BLOCK_LOADED);
 	// copy over the data into the block from the file buffer
-	auto block = AllocateBlock(handle->block_manager, std::move(reusable_buffer), handle->block_id);
+	auto block = AllocateBlock(block_manager, std::move(reusable_buffer), block_id);
 	memcpy(block->InternalBuffer(), data, block->AllocSize());
-	handle->buffer = std::move(block);
-	handle->state = BlockState::BLOCK_LOADED;
-	return BufferHandle(handle, handle->buffer.get());
+	buffer = std::move(block);
+	state = BlockState::BLOCK_LOADED;
+	return BufferHandle(shared_from_this());
 }
 
-BufferHandle BlockHandle::Load(shared_ptr<BlockHandle> &handle, unique_ptr<FileBuffer> reusable_buffer) {
-	if (handle->state == BlockState::BLOCK_LOADED) {
+BufferHandle BlockHandle::Load(unique_ptr<FileBuffer> reusable_buffer) {
+	if (state == BlockState::BLOCK_LOADED) {
 		// already loaded
-		D_ASSERT(handle->buffer);
-		return BufferHandle(handle, handle->buffer.get());
+		D_ASSERT(buffer);
+		return BufferHandle(shared_from_this());
 	}
 
-	auto &block_manager = handle->block_manager;
-	if (handle->block_id < MAXIMUM_BLOCK) {
-		auto block = AllocateBlock(block_manager, std::move(reusable_buffer), handle->block_id);
+	if (block_id < MAXIMUM_BLOCK) {
+		auto block = AllocateBlock(block_manager, std::move(reusable_buffer), block_id);
 		block_manager.Read(*block);
-		handle->buffer = std::move(block);
+		buffer = std::move(block);
 	} else {
-		if (handle->MustWriteToTemporaryFile()) {
-			handle->buffer = block_manager.buffer_manager.ReadTemporaryBuffer(handle->tag, handle->block_id,
-			                                                                  std::move(reusable_buffer));
+		if (MustWriteToTemporaryFile()) {
+			buffer = block_manager.buffer_manager.ReadTemporaryBuffer(tag, *this, std::move(reusable_buffer));
 		} else {
 			return BufferHandle(); // Destroyed upon unpin/evict, so there is no temp buffer to read
 		}
 	}
-	handle->state = BlockState::BLOCK_LOADED;
-	return BufferHandle(handle, handle->buffer.get());
+	state = BlockState::BLOCK_LOADED;
+	return BufferHandle(shared_from_this());
 }
 
 unique_ptr<FileBuffer> BlockHandle::UnloadAndTakeBlock() {

package/src/duckdb/src/storage/buffer/block_manager.cpp

@@ -70,14 +70,22 @@ shared_ptr<BlockHandle> BlockManager::ConvertToPersistent(block_id_t block_id, s
 	return new_block;
 }
 
-void BlockManager::UnregisterBlock(block_id_t block_id) {
-	if (block_id >= MAXIMUM_BLOCK) {
+void BlockManager::UnregisterBlock(block_id_t id) {
+	D_ASSERT(id < MAXIMUM_BLOCK);
+	lock_guard<mutex> lock(blocks_lock);
+	// on-disk block: erase from list of blocks in manager
+	blocks.erase(id);
+}
+
+void BlockManager::UnregisterBlock(BlockHandle &block) {
+	auto id = block.BlockId();
+	if (id >= MAXIMUM_BLOCK) {
 		// in-memory buffer: buffer could have been offloaded to disk: remove the file
-		buffer_manager.DeleteTemporaryFile(block_id);
+		buffer_manager.DeleteTemporaryFile(block);
 	} else {
 		lock_guard<mutex> lock(blocks_lock);
 		// on-disk block: erase from list of blocks in manager
-		blocks.erase(block_id);
+		blocks.erase(id);
 	}
 }
 

package/src/duckdb/src/storage/buffer/buffer_handle.cpp

@@ -7,8 +7,8 @@ namespace duckdb {
 BufferHandle::BufferHandle() : handle(nullptr), node(nullptr) {
 }
 
-BufferHandle::BufferHandle(shared_ptr<BlockHandle> handle_p, FileBuffer *node_p)
-    : handle(std::move(handle_p)), node(node_p) {
+BufferHandle::BufferHandle(shared_ptr<BlockHandle> handle_p)
+    : handle(std::move(handle_p)), node(handle ? handle->buffer.get() : nullptr) {
 }
 
 BufferHandle::BufferHandle(BufferHandle &&other) noexcept : node(nullptr) {

package/src/duckdb/src/storage/buffer/buffer_pool.cpp

@@ -194,8 +194,11 @@ void EvictionQueue::PurgeIteration(const idx_t purge_size) {
 	total_dead_nodes -= actually_dequeued - alive_nodes;
 }
 
-BufferPool::BufferPool(idx_t maximum_memory, bool track_eviction_timestamps)
-    : maximum_memory(maximum_memory), track_eviction_timestamps(track_eviction_timestamps),
+BufferPool::BufferPool(idx_t maximum_memory, bool track_eviction_timestamps,
+                       idx_t allocator_bulk_deallocation_flush_threshold)
+    : maximum_memory(maximum_memory),
+      allocator_bulk_deallocation_flush_threshold(allocator_bulk_deallocation_flush_threshold),
+      track_eviction_timestamps(track_eviction_timestamps),
       temporary_memory_manager(make_uniq<TemporaryMemoryManager>()) {
 	queues.reserve(FILE_BUFFER_TYPE_COUNT);
 	for (idx_t i = 0; i < FILE_BUFFER_TYPE_COUNT; i++) {
@@ -283,6 +286,9 @@ BufferPool::EvictionResult BufferPool::EvictBlocksInternal(EvictionQueue &queue,
 	bool found = false;
 
 	if (memory_usage.GetUsedMemory(MemoryUsageCaches::NO_FLUSH) <= memory_limit) {
+		if (Allocator::SupportsFlush() && extra_memory > allocator_bulk_deallocation_flush_threshold) {
+			Allocator::FlushAll();
+		}
 		return {true, std::move(r)};
 	}
 
@@ -407,6 +413,10 @@ void BufferPool::SetAllocatorBulkDeallocationFlushThreshold(idx_t threshold) {
 	allocator_bulk_deallocation_flush_threshold = threshold;
 }
 
+idx_t BufferPool::GetAllocatorBulkDeallocationFlushThreshold() {
+	return allocator_bulk_deallocation_flush_threshold;
+}
+
 BufferPool::MemoryUsage::MemoryUsage() {
 	for (auto &v : memory_usage) {
 		v = 0;

package/src/duckdb/src/storage/buffer_manager.cpp

@@ -79,11 +79,12 @@ void BufferManager::WriteTemporaryBuffer(MemoryTag tag, block_id_t block_id, Fil
 	throw NotImplementedException("This type of BufferManager does not support 'WriteTemporaryBuffer");
 }
 
-unique_ptr<FileBuffer> BufferManager::ReadTemporaryBuffer(MemoryTag tag, block_id_t id, unique_ptr<FileBuffer> buffer) {
+unique_ptr<FileBuffer> BufferManager::ReadTemporaryBuffer(MemoryTag tag, BlockHandle &block,
+                                                          unique_ptr<FileBuffer> buffer) {
 	throw NotImplementedException("This type of BufferManager does not support 'ReadTemporaryBuffer");
 }
 
-void BufferManager::DeleteTemporaryFile(block_id_t id) {
+void BufferManager::DeleteTemporaryFile(BlockHandle &block) {
 	throw NotImplementedException("This type of BufferManager does not support 'DeleteTemporaryFile");
 }
 

package/src/duckdb/src/storage/compression/rle.cpp

@@ -57,11 +57,14 @@ public:
 			} else {
 				// the values are different
 				// issue the callback on the last value
-				Flush<OP>();
+				// edge case: if a value has exactly 2^16 repeated values, we can end up here with last_seen_count = 0
+				if (last_seen_count > 0) {
+					Flush<OP>();
+					seen_count++;
+				}
 
 				// increment the seen_count and put the new value into the RLE slot
 				last_value = data[idx];
-				seen_count++;
 				last_seen_count = 1;
 			}
 		} else {

package/src/duckdb/src/storage/compression/string_uncompressed.cpp

@@ -315,7 +315,8 @@ void UncompressedStringStorage::WriteStringMemory(ColumnSegment &segment, string
 		new_block->offset = 0;
 		new_block->size = alloc_size;
 		// allocate an in-memory buffer for it
-		handle = buffer_manager.Allocate(MemoryTag::OVERFLOW_STRINGS, alloc_size, false, &block);
+		handle = buffer_manager.Allocate(MemoryTag::OVERFLOW_STRINGS, alloc_size, false);
+		block = handle.GetBlockHandle();
 		state.overflow_blocks.insert(make_pair(block->BlockId(), reference<StringBlock>(*new_block)));
 		new_block->block = std::move(block);
 		new_block->next = std::move(state.head);

package/src/duckdb/src/storage/metadata/metadata_manager.cpp

@@ -61,27 +61,28 @@ MetadataHandle MetadataManager::Pin(MetadataPointer pointer) {
 	return handle;
 }
 
-void MetadataManager::ConvertToTransient(MetadataBlock &block) {
+void MetadataManager::ConvertToTransient(MetadataBlock &metadata_block) {
 	// pin the old block
-	auto old_buffer = buffer_manager.Pin(block.block);
+	auto old_buffer = buffer_manager.Pin(metadata_block.block);
 
 	// allocate a new transient block to replace it
-	shared_ptr<BlockHandle> new_block;
-	auto new_buffer = buffer_manager.Allocate(MemoryTag::METADATA, block_manager.GetBlockSize(), false, &new_block);
+	auto new_buffer = buffer_manager.Allocate(MemoryTag::METADATA, block_manager.GetBlockSize(), false);
+	auto new_block = new_buffer.GetBlockHandle();
 
 	// copy the data to the transient block
 	memcpy(new_buffer.Ptr(), old_buffer.Ptr(), block_manager.GetBlockSize());
-	block.block = std::move(new_block);
+	metadata_block.block = std::move(new_block);
 
 	// unregister the old block
-	block_manager.UnregisterBlock(block.block_id);
+	block_manager.UnregisterBlock(metadata_block.block_id);
 }
 
 block_id_t MetadataManager::AllocateNewBlock() {
 	auto new_block_id = GetNextBlockId();
 
 	MetadataBlock new_block;
-	auto handle = buffer_manager.Allocate(MemoryTag::METADATA, block_manager.GetBlockSize(), false, &new_block.block);
+	auto handle = buffer_manager.Allocate(MemoryTag::METADATA, block_manager.GetBlockSize(), false);
+	new_block.block = handle.GetBlockHandle();
 	new_block.block_id = new_block_id;
 	for (idx_t i = 0; i < METADATA_BLOCK_COUNT; i++) {
 		new_block.free_blocks.push_back(NumericCast<uint8_t>(METADATA_BLOCK_COUNT - i - 1));