duckdb 1.4.2-dev4.0 → 1.4.2

package/src/duckdb/src/optimizer/join_order/relation_manager.cpp

@@ -54,6 +54,13 @@ void RelationManager::AddRelation(LogicalOperator &op, optional_ptr<LogicalOpera
 	auto relation_id = relations.size();
 
 	auto table_indexes = op.GetTableIndex();
+	bool is_unnest_or_get_with_unnest = op.type == LogicalOperatorType::LOGICAL_UNNEST;
+	if (op.type == LogicalOperatorType::LOGICAL_GET) {
+		auto &get = op.Cast<LogicalGet>();
+		if (get.function.name == "unnest") {
+			is_unnest_or_get_with_unnest = true;
+		}
+	}
 	if (table_indexes.empty()) {
 		// relation represents a non-reorderable relation, most likely a join relation
 		// Get the tables referenced in the non-reorderable relation and add them to the relation mapping
@@ -65,7 +72,7 @@ void RelationManager::AddRelation(LogicalOperator &op, optional_ptr<LogicalOpera
 			D_ASSERT(relation_mapping.find(reference) == relation_mapping.end());
 			relation_mapping[reference] = relation_id;
 		}
-	} else if (op.type == LogicalOperatorType::LOGICAL_UNNEST) {
+	} else if (is_unnest_or_get_with_unnest) {
 		// logical unnest has a logical_unnest index, but other bindings can refer to
 		// columns that are not unnested.
 		auto bindings = op.GetColumnBindings();
@@ -182,6 +189,21 @@ static void ModifyStatsIfLimit(optional_ptr<LogicalOperator> limit_op, RelationS
 	}
 }
 
+void RelationManager::AddUnnestRelation(JoinOrderOptimizer &optimizer, LogicalOperator &op, LogicalOperator &input_op,
+                                        optional_ptr<LogicalOperator> parent, RelationStats &child_stats,
+                                        optional_ptr<LogicalOperator> limit_op,
+                                        vector<reference<LogicalOperator>> &datasource_filters) {
+	D_ASSERT(!op.children.empty());
+	auto child_optimizer = optimizer.CreateChildOptimizer();
+	op.children[0] = child_optimizer.Optimize(std::move(op.children[0]), &child_stats);
+	if (!datasource_filters.empty()) {
+		child_stats.cardinality = LossyNumericCast<idx_t>(static_cast<double>(child_stats.cardinality) *
+		                                                  RelationStatisticsHelper::DEFAULT_SELECTIVITY);
+	}
+	ModifyStatsIfLimit(limit_op.get(), child_stats);
+	AddRelation(input_op, parent, child_stats);
+}
+
 bool RelationManager::ExtractJoinRelations(JoinOrderOptimizer &optimizer, LogicalOperator &input_op,
                                            vector<reference<LogicalOperator>> &filter_operators,
                                            optional_ptr<LogicalOperator> parent) {
@@ -279,15 +301,7 @@ bool RelationManager::ExtractJoinRelations(JoinOrderOptimizer &optimizer, Logica
 	case LogicalOperatorType::LOGICAL_UNNEST: {
 		// optimize children of unnest
 		RelationStats child_stats;
-		auto child_optimizer = optimizer.CreateChildOptimizer();
-		op->children[0] = child_optimizer.Optimize(std::move(op->children[0]), &child_stats);
-		// the extracted cardinality should be set for window
-		if (!datasource_filters.empty()) {
-			child_stats.cardinality = LossyNumericCast<idx_t>(static_cast<double>(child_stats.cardinality) *
-			                                                  RelationStatisticsHelper::DEFAULT_SELECTIVITY);
-		}
-		ModifyStatsIfLimit(limit_op.get(), child_stats);
-		AddRelation(input_op, parent, child_stats);
+		AddUnnestRelation(optimizer, *op, input_op, parent, child_stats, limit_op, datasource_filters);
 		return true;
 	}
 	case LogicalOperatorType::LOGICAL_COMPARISON_JOIN: {
@@ -345,6 +359,11 @@ bool RelationManager::ExtractJoinRelations(JoinOrderOptimizer &optimizer, Logica
 	case LogicalOperatorType::LOGICAL_GET: {
 		// TODO: Get stats from a logical GET
 		auto &get = op->Cast<LogicalGet>();
+		if (get.function.name == "unnest" && !op->children.empty()) {
+			RelationStats child_stats;
+			AddUnnestRelation(optimizer, *op, input_op, parent, child_stats, limit_op, datasource_filters);
+			return true;
+		}
 		auto stats = RelationStatisticsHelper::ExtractGetStats(get, context);
 		// if there is another logical filter that could not be pushed down into the
 		// table scan, apply another selectivity.

package/src/duckdb/src/optimizer/rule/regex_optimizations.cpp

@@ -184,6 +184,13 @@ unique_ptr<Expression> RegexOptimizationRule::Apply(LogicalOperator &op, vector<
 		if (!escaped_like_string.exists) {
 			return nullptr;
 		}
+
+		// if regexp had options, remove them so the new Contains Expression can be matched for other optimizers.
+		if (root.children.size() == 3) {
+			root.children.pop_back();
+			D_ASSERT(root.children.size() == 2);
+		}
+
 		auto parameter = make_uniq<BoundConstantExpression>(Value(std::move(escaped_like_string.like_string)));
 		auto contains = make_uniq<BoundFunctionExpression>(root.return_type, GetStringContains(),
 		                                                   std::move(root.children), nullptr);

package/src/duckdb/src/parallel/task_executor.cpp

@@ -69,8 +69,10 @@ TaskExecutionResult BaseExecutorTask::Execute(TaskExecutionMode mode) {
 		return TaskExecutionResult::TASK_FINISHED;
 	}
 	try {
-		TaskNotifier task_notifier {executor.context};
-		ExecuteTask();
+		{
+			TaskNotifier task_notifier {executor.context};
+			ExecuteTask();
+		}
 		executor.FinishTask();
 		return TaskExecutionResult::TASK_FINISHED;
 	} catch (std::exception &ex) {

package/src/duckdb/src/parser/query_node/cte_node.cpp

@@ -1,6 +1,7 @@
 #include "duckdb/parser/query_node/cte_node.hpp"
 #include "duckdb/common/serializer/serializer.hpp"
 #include "duckdb/common/serializer/deserializer.hpp"
+#include "duckdb/parser/statement/select_statement.hpp"
 
 namespace duckdb {
 
@@ -39,4 +40,82 @@ unique_ptr<QueryNode> CTENode::Copy() const {
 	return std::move(result);
 }
 
+// TEMPORARY BUGFIX WARNING - none of this code should make it into main - this is a temporary work-around for v1.4
+// TEMPORARY BUGFIX START
+// the below code fixes backwards and forwards compatibility of CTEs with the somewhat broken version of CTEs in v1.4
+// all of this code has been made obsolete with the CTE binding rework
+void QueryNode::ExtractCTENodes(unique_ptr<QueryNode> &query_node) {
+	if (query_node->cte_map.map.empty()) {
+		return;
+	}
+	vector<unique_ptr<CTENode>> materialized_ctes;
+	for (auto &cte : query_node->cte_map.map) {
+		auto &cte_entry = cte.second;
+		auto mat_cte = make_uniq<CTENode>();
+		mat_cte->ctename = cte.first;
+		mat_cte->query = cte_entry->query->node->Copy();
+		mat_cte->aliases = cte_entry->aliases;
+		mat_cte->materialized = cte_entry->materialized;
+		materialized_ctes.push_back(std::move(mat_cte));
+	}
+
+	auto root = std::move(query_node);
+	while (!materialized_ctes.empty()) {
+		unique_ptr<CTENode> node_result;
+		node_result = std::move(materialized_ctes.back());
+		node_result->cte_map = root->cte_map.Copy();
+		node_result->child = std::move(root);
+		root = std::move(node_result);
+		materialized_ctes.pop_back();
+	}
+	query_node = std::move(root);
+}
+
+void EraseDuplicateCTE(unique_ptr<QueryNode> &node, const string &ctename) {
+	if (node->type != QueryNodeType::CTE_NODE) {
+		// not a CTE
+		return;
+	}
+	auto &cte_node = node->Cast<CTENode>();
+	if (cte_node.ctename == ctename) {
+		// duplicate CTE - erase this CTE node
+		node = std::move(cte_node.child);
+		EraseDuplicateCTE(node, ctename);
+	} else {
+		// not a duplicate - recurse into child
+		EraseDuplicateCTE(cte_node.child, ctename);
+	}
+}
+
+void CTENode::Serialize(Serializer &serializer) const {
+	if (materialized != CTEMaterialize::CTE_MATERIALIZE_ALWAYS) {
+		// for non-materialized CTEs - don't serialize CTENode
+		// older DuckDB versions only expect a CTENode to be there for materialized CTEs
+		child->Serialize(serializer);
+		return;
+	}
+	auto child_copy = child->Copy();
+	EraseDuplicateCTE(child_copy, ctename);
+
+	QueryNode::Serialize(serializer);
+	serializer.WritePropertyWithDefault<string>(200, "cte_name", ctename);
+	serializer.WritePropertyWithDefault<unique_ptr<QueryNode>>(201, "query", query);
+	serializer.WritePropertyWithDefault<unique_ptr<QueryNode>>(202, "child", child_copy);
+	serializer.WritePropertyWithDefault<vector<string>>(203, "aliases", aliases);
+}
+
+unique_ptr<QueryNode> CTENode::Deserialize(Deserializer &deserializer) {
+	auto result = duckdb::unique_ptr<CTENode>(new CTENode());
+	deserializer.ReadPropertyWithDefault<string>(200, "cte_name", result->ctename);
+	deserializer.ReadPropertyWithDefault<unique_ptr<QueryNode>>(201, "query", result->query);
+	deserializer.ReadPropertyWithDefault<unique_ptr<QueryNode>>(202, "child", result->child);
+	deserializer.ReadPropertyWithDefault<vector<string>>(203, "aliases", result->aliases);
+	// v1.4.0 and v1.4.1 wrote this property - deserialize it for BC with these versions
+	deserializer.ReadPropertyWithExplicitDefault<CTEMaterialize>(204, "materialized", result->materialized,
+	                                                             CTEMaterialize::CTE_MATERIALIZE_DEFAULT);
+	return std::move(result->child);
+}
+// TEMPORARY BUGFIX WARNING - none of this code should make it into main - this is a temporary work-around for v1.4
+// TEMPORARY BUGFIX END
+
 } // namespace duckdb

package/src/duckdb/src/parser/transform/expression/transform_cast.cpp

@@ -21,7 +21,9 @@ unique_ptr<ParsedExpression> Transformer::TransformTypeCast(duckdb_libpgquery::P
 				parameters.query_location = NumericCast<idx_t>(root.location);
 			}
 			auto blob_data = Blob::ToBlob(string(c->val.val.str), parameters);
-			return make_uniq<ConstantExpression>(Value::BLOB_RAW(blob_data));
+			auto result = make_uniq<ConstantExpression>(Value::BLOB_RAW(blob_data));
+			SetQueryLocation(*result, root.location);
+			return std::move(result);
 		}
 	}
 	// transform the expression node

package/src/duckdb/src/planner/binder/expression/bind_macro_expression.cpp

@@ -98,6 +98,7 @@ void ExpressionBinder::UnfoldMacroExpression(FunctionExpression &function, Scala
 	// validate the arguments and separate positional and default arguments
 	vector<unique_ptr<ParsedExpression>> positional_arguments;
 	InsertionOrderPreservingMap<unique_ptr<ParsedExpression>> named_arguments;
+	binder.lambda_bindings = lambda_bindings;
 	auto bind_result = MacroFunction::BindMacroFunction(binder, macro_func.macros, macro_func.name, function,
 	                                                    positional_arguments, named_arguments, depth);
 	if (!bind_result.error.empty()) {

package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp

@@ -40,10 +40,18 @@ static void VerifyCompressionType(ClientContext &context, optional_ptr<StorageMa
 	auto &base = info.base->Cast<CreateTableInfo>();
 	for (auto &col : base.columns.Logical()) {
 		auto compression_type = col.CompressionType();
-		if (CompressionTypeIsDeprecated(compression_type, storage_manager)) {
-			throw BinderException("Can't compress using user-provided compression type '%s', that type is deprecated "
-			                      "and only has decompress support",
-			                      CompressionTypeToString(compression_type));
+		auto compression_availability_result = CompressionTypeIsAvailable(compression_type, storage_manager);
+		if (!compression_availability_result.IsAvailable()) {
+			if (compression_availability_result.IsDeprecated()) {
+				throw BinderException(
+				    "Can't compress using user-provided compression type '%s', that type is deprecated "
+				    "and only has decompress support",
+				    CompressionTypeToString(compression_type));
+			} else {
+				throw BinderException(
+				    "Can't compress using user-provided compression type '%s', that type is not available yet",
+				    CompressionTypeToString(compression_type));
+			}
 		}
 		auto logical_type = col.GetType();
 		if (logical_type.id() == LogicalTypeId::USER && logical_type.HasAlias()) {

package/src/duckdb/src/planner/binder/statement/bind_insert.cpp

@@ -277,7 +277,7 @@ unique_ptr<MergeIntoStatement> Binder::GenerateMergeInto(InsertStatement &stmt,
 	auto storage_info = table.GetStorageInfo(context);
 	auto &columns = table.GetColumns();
 	// set up the columns on which to join
-	vector<string> distinct_on_columns;
+	vector<vector<string>> all_distinct_on_columns;
 	if (on_conflict_info.indexed_columns.empty()) {
 		// When omitting the conflict target, we derive the join columns from the primary key/unique constraints
 		// traverse the primary key/unique constraints
@@ -292,6 +292,7 @@ unique_ptr<MergeIntoStatement> Binder::GenerateMergeInto(InsertStatement &stmt,
 
 			vector<unique_ptr<ParsedExpression>> and_children;
 			auto &indexed_columns = index.column_set;
+			vector<string> distinct_on_columns;
 			for (auto &column : columns.Physical()) {
 				if (!indexed_columns.count(column.Physical().index)) {
 					continue;
@@ -303,6 +304,7 @@ unique_ptr<MergeIntoStatement> Binder::GenerateMergeInto(InsertStatement &stmt,
 				and_children.push_back(std::move(new_condition));
 				distinct_on_columns.push_back(column.Name());
 			}
+			all_distinct_on_columns.push_back(std::move(distinct_on_columns));
 			if (and_children.empty()) {
 				continue;
 			}
@@ -377,7 +379,7 @@ unique_ptr<MergeIntoStatement> Binder::GenerateMergeInto(InsertStatement &stmt,
 			throw BinderException("The specified columns as conflict target are not referenced by a UNIQUE/PRIMARY KEY "
 			                      "CONSTRAINT or INDEX");
 		}
-		distinct_on_columns = on_conflict_info.indexed_columns;
+		all_distinct_on_columns.push_back(on_conflict_info.indexed_columns);
 		merge_into->using_columns = std::move(on_conflict_info.indexed_columns);
 	}
 
@@ -445,17 +447,19 @@ unique_ptr<MergeIntoStatement> Binder::GenerateMergeInto(InsertStatement &stmt,
 		}
 	}
 	// push DISTINCT ON(unique_columns)
-	auto distinct_stmt = make_uniq<SelectStatement>();
-	auto select_node = make_uniq<SelectNode>();
-	auto distinct = make_uniq<DistinctModifier>();
-	for (auto &col : distinct_on_columns) {
-		distinct->distinct_on_targets.push_back(make_uniq<ColumnRefExpression>(col));
+	for (auto &distinct_on_columns : all_distinct_on_columns) {
+		auto distinct_stmt = make_uniq<SelectStatement>();
+		auto select_node = make_uniq<SelectNode>();
+		auto distinct = make_uniq<DistinctModifier>();
+		for (auto &col : distinct_on_columns) {
+			distinct->distinct_on_targets.push_back(make_uniq<ColumnRefExpression>(col));
+		}
+		select_node->modifiers.push_back(std::move(distinct));
+		select_node->select_list.push_back(make_uniq<StarExpression>());
+		select_node->from_table = std::move(source);
+		distinct_stmt->node = std::move(select_node);
+		source = make_uniq<SubqueryRef>(std::move(distinct_stmt), "excluded");
 	}
-	select_node->modifiers.push_back(std::move(distinct));
-	select_node->select_list.push_back(make_uniq<StarExpression>());
-	select_node->from_table = std::move(source);
-	distinct_stmt->node = std::move(select_node);
-	source = make_uniq<SubqueryRef>(std::move(distinct_stmt), "excluded");
 
 	merge_into->source = std::move(source);
 

package/src/duckdb/src/planner/binder/statement/bind_merge_into.cpp

@@ -41,10 +41,20 @@ unique_ptr<BoundMergeIntoAction> Binder::BindMergeAction(LogicalMergeInto &merge
 	auto result = make_uniq<BoundMergeIntoAction>();
 	result->action_type = action.action_type;
 	if (action.condition) {
-		ProjectionBinder proj_binder(*this, context, proj_index, expressions, "WHERE clause");
-		proj_binder.target_type = LogicalType::BOOLEAN;
-		auto cond = proj_binder.Bind(action.condition);
-		result->condition = std::move(cond);
+		if (action.condition->HasSubquery()) {
+			// if we have a subquery we need to execute the condition outside of the MERGE INTO statement
+			WhereBinder where_binder(*this, context);
+			auto cond = where_binder.Bind(action.condition);
+			PlanSubqueries(cond, root);
+			result->condition =
+			    make_uniq<BoundColumnRefExpression>(cond->return_type, ColumnBinding(proj_index, expressions.size()));
+			expressions.push_back(std::move(cond));
+		} else {
+			ProjectionBinder proj_binder(*this, context, proj_index, expressions, "WHERE clause");
+			proj_binder.target_type = LogicalType::BOOLEAN;
+			auto cond = proj_binder.Bind(action.condition);
+			result->condition = std::move(cond);
+		}
 	}
 	switch (action.action_type) {
 	case MergeActionType::MERGE_UPDATE: {
@@ -173,6 +183,32 @@ void RewriteMergeBindings(LogicalOperator &op, const vector<ColumnBinding> &sour
 	    op, [&](unique_ptr<Expression> *child) { RewriteMergeBindings(*child, source_bindings, new_table_index); });
 }
 
+LogicalGet &ExtractLogicalGet(LogicalOperator &op) {
+	reference<LogicalOperator> current_op(op);
+	while (current_op.get().type == LogicalOperatorType::LOGICAL_FILTER) {
+		current_op = *current_op.get().children[0];
+	}
+	if (current_op.get().type != LogicalOperatorType::LOGICAL_GET) {
+		throw InvalidInputException("BindMerge - expected to find an operator of type LOGICAL_GET but got %s",
+		                            op.ToString());
+	}
+	return current_op.get().Cast<LogicalGet>();
+}
+
+void CheckMergeAction(MergeActionCondition condition, MergeActionType action_type) {
+	if (condition == MergeActionCondition::WHEN_NOT_MATCHED_BY_TARGET) {
+		switch (action_type) {
+		case MergeActionType::MERGE_UPDATE:
+		case MergeActionType::MERGE_DELETE:
+			throw ParserException("WHEN NOT MATCHED (BY TARGET) cannot be combined with UPDATE or DELETE actions - as "
+			                      "there is no corresponding row in the target to update or delete.\nDid you mean to "
+			                      "use WHEN MATCHED or WHEN NOT MATCHED BY SOURCE?");
+		default:
+			break;
+		}
+	}
+}
+
 BoundStatement Binder::Bind(MergeIntoStatement &stmt) {
 	// bind the target table
 	auto target_binder = Binder::CreateBinder(context, this);
@@ -243,7 +279,7 @@ BoundStatement Binder::Bind(MergeIntoStatement &stmt) {
 	// kind of hacky, CreatePlan turns a RIGHT join into a LEFT join so the children get reversed from what we need
 	bool inverted = join.type == JoinType::RIGHT;
 	auto &source = join_ref.get().children[inverted ? 1 : 0];
-	auto &get = join_ref.get().children[inverted ? 0 : 1]->Cast<LogicalGet>();
+	auto &get = ExtractLogicalGet(*join_ref.get().children[inverted ? 0 : 1]);
 
 	auto merge_into = make_uniq<LogicalMergeInto>(table);
 	merge_into->table_index = GenerateTableIndex();
@@ -265,6 +301,7 @@ BoundStatement Binder::Bind(MergeIntoStatement &stmt) {
 	for (auto &entry : stmt.actions) {
 		vector<unique_ptr<BoundMergeIntoAction>> bound_actions;
 		for (auto &action : entry.second) {
+			CheckMergeAction(entry.first, action->action_type);
 			bound_actions.push_back(BindMergeAction(*merge_into, table, get, proj_index, projection_expressions, root,
 			                                        *action, source_aliases, source_names));
 		}

package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp

@@ -326,30 +326,6 @@ unique_ptr<BoundTableRef> Binder::Bind(BaseTableRef &ref) {
 
 		// The view may contain CTEs, but maybe only in the cte_map, so we need create CTE nodes for them
 		auto query = view_catalog_entry.GetQuery().Copy();
-		auto &select_stmt = query->Cast<SelectStatement>();
-
-		vector<unique_ptr<CTENode>> materialized_ctes;
-		for (auto &cte : select_stmt.node->cte_map.map) {
-			auto &cte_entry = cte.second;
-			auto mat_cte = make_uniq<CTENode>();
-			mat_cte->ctename = cte.first;
-			mat_cte->query = cte_entry->query->node->Copy();
-			mat_cte->aliases = cte_entry->aliases;
-			mat_cte->materialized = cte_entry->materialized;
-			materialized_ctes.push_back(std::move(mat_cte));
-		}
-
-		auto root = std::move(select_stmt.node);
-		while (!materialized_ctes.empty()) {
-			unique_ptr<CTENode> node_result;
-			node_result = std::move(materialized_ctes.back());
-			node_result->cte_map = root->cte_map.Copy();
-			node_result->child = std::move(root);
-			root = std::move(node_result);
-			materialized_ctes.pop_back();
-		}
-		select_stmt.node = std::move(root);
-
 		SubqueryRef subquery(unique_ptr_cast<SQLStatement, SelectStatement>(std::move(query)));
 
 		subquery.alias = ref.alias;

package/src/duckdb/src/planner/binder/tableref/bind_table_function.cpp

@@ -147,7 +147,7 @@ bool Binder::BindTableFunctionParameters(TableFunctionCatalogEntry &table_functi
 			MoveCorrelatedExpressions(*subquery->binder);
 			seen_subquery = true;
 			arguments.emplace_back(LogicalTypeId::TABLE);
-			parameters.emplace_back(Value());
+			parameters.emplace_back();
 			continue;
 		}
 

package/src/duckdb/src/planner/binder.cpp

@@ -469,7 +469,6 @@ optional_ptr<Binding> Binder::GetMatchingBinding(const string &catalog_name, con
                                                  const string &table_name, const string &column_name,
                                                  ErrorData &error) {
 	optional_ptr<Binding> binding;
-	D_ASSERT(!lambda_bindings);
 	if (macro_binding && table_name == macro_binding->GetAlias()) {
 		binding = optional_ptr<Binding>(macro_binding.get());
 	} else {

package/src/duckdb/src/planner/expression_binder/having_binder.cpp

@@ -3,7 +3,6 @@
 #include "duckdb/parser/expression/columnref_expression.hpp"
 #include "duckdb/parser/expression/window_expression.hpp"
 #include "duckdb/planner/binder.hpp"
-#include "duckdb/planner/expression_binder/aggregate_binder.hpp"
 #include "duckdb/common/string_util.hpp"
 #include "duckdb/planner/query_node/bound_select_node.hpp"
 
@@ -91,7 +90,7 @@ BindResult HavingBinder::BindColumnRef(unique_ptr<ParsedExpression> &expr_ptr, i
 }
 
 BindResult HavingBinder::BindWindow(WindowExpression &expr, idx_t depth) {
-	return BindResult(BinderException::Unsupported(expr, "HAVING clause cannot contain window functions!"));
+	throw BinderException::Unsupported(expr, "HAVING clause cannot contain window functions!");
 }
 
 } // namespace duckdb

package/src/duckdb/src/storage/buffer/block_manager.cpp

@@ -34,19 +34,32 @@ shared_ptr<BlockHandle> BlockManager::RegisterBlock(block_id_t block_id) {
 }
 
 shared_ptr<BlockHandle> BlockManager::ConvertToPersistent(QueryContext context, block_id_t block_id,
-                                                          shared_ptr<BlockHandle> old_block, BufferHandle old_handle) {
+                                                          shared_ptr<BlockHandle> old_block, BufferHandle old_handle,
+                                                          ConvertToPersistentMode mode) {
 	// register a block with the new block id
 	auto new_block = RegisterBlock(block_id);
 	D_ASSERT(new_block->GetState() == BlockState::BLOCK_UNLOADED);
 	D_ASSERT(new_block->Readers() == 0);
 
+	if (mode == ConvertToPersistentMode::THREAD_SAFE) {
+		// safe mode - create a copy of the old block and operate on that
+		// this ensures we don't modify the old block - which allows other concurrent operations on the old block to
+		// continue
+		auto old_block_copy = buffer_manager.AllocateMemory(old_block->GetMemoryTag(), this, false);
+		auto copy_pin = buffer_manager.Pin(old_block_copy);
+		memcpy(copy_pin.Ptr(), old_handle.Ptr(), GetBlockSize());
+		old_block = std::move(old_block_copy);
+		old_handle = std::move(copy_pin);
+	}
+
 	auto lock = old_block->GetLock();
 	D_ASSERT(old_block->GetState() == BlockState::BLOCK_LOADED);
 	D_ASSERT(old_block->GetBuffer(lock));
 	if (old_block->Readers() > 1) {
-		throw InternalException("BlockManager::ConvertToPersistent - cannot be called for block %d as old_block has "
-		                        "multiple readers active",
-		                        block_id);
+		throw InternalException(
+		    "BlockManager::ConvertToPersistent in destructive mode - cannot be called for block %d as old_block has "
+		    "multiple readers active",
+		    block_id);
 	}
 
 	// Temp buffers can be larger than the storage block size.
@@ -76,10 +89,11 @@ shared_ptr<BlockHandle> BlockManager::ConvertToPersistent(QueryContext context,
 }
 
 shared_ptr<BlockHandle> BlockManager::ConvertToPersistent(QueryContext context, block_id_t block_id,
-                                                          shared_ptr<BlockHandle> old_block) {
+                                                          shared_ptr<BlockHandle> old_block,
+                                                          ConvertToPersistentMode mode) {
 	// pin the old block to ensure we have it loaded in memory
 	auto handle = buffer_manager.Pin(old_block);
-	return ConvertToPersistent(context, block_id, std::move(old_block), std::move(handle));
+	return ConvertToPersistent(context, block_id, std::move(old_block), std::move(handle), mode);
 }
 
 void BlockManager::UnregisterBlock(block_id_t id) {

package/src/duckdb/src/storage/checkpoint/table_data_writer.cpp

@@ -4,6 +4,7 @@
 #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
 #include "duckdb/common/serializer/binary_serializer.hpp"
 #include "duckdb/main/database.hpp"
+#include "duckdb/main/settings.hpp"
 #include "duckdb/parallel/task_scheduler.hpp"
 #include "duckdb/storage/table/column_checkpoint_state.hpp"
 #include "duckdb/storage/table/table_statistics.hpp"
@@ -119,15 +120,16 @@ void SingleFileTableDataWriter::FinalizeTable(const TableStatistics &global_stat
 	}
 	auto index_storage_infos = info.GetIndexes().SerializeToDisk(context, options);
 
-#ifdef DUCKDB_BLOCK_VERIFICATION
-	for (auto &entry : index_storage_infos) {
-		for (auto &allocator : entry.allocator_infos) {
-			for (auto &block : allocator.block_pointers) {
-				checkpoint_manager.verify_block_usage_count[block.block_id]++;
+	auto debug_verify_blocks = DBConfig::GetSetting<DebugVerifyBlocksSetting>(GetDatabase());
+	if (debug_verify_blocks) {
+		for (auto &entry : index_storage_infos) {
+			for (auto &allocator : entry.allocator_infos) {
+				for (auto &block : allocator.block_pointers) {
+					checkpoint_manager.verify_block_usage_count[block.block_id]++;
+				}
 			}
 		}
 	}
-#endif
 
 	// write empty block pointers for forwards compatibility
 	vector<BlockPointer> compat_block_pointers;

package/src/duckdb/src/storage/checkpoint_manager.cpp

@@ -214,33 +214,35 @@ void SingleFileCheckpointWriter::CreateCheckpoint() {
 	header.vector_size = STANDARD_VECTOR_SIZE;
 	block_manager.WriteHeader(context, header);
 
-#ifdef DUCKDB_BLOCK_VERIFICATION
-	// extend verify_block_usage_count
-	auto metadata_info = storage_manager.GetMetadataInfo();
-	for (auto &info : metadata_info) {
-		verify_block_usage_count[info.block_id]++;
-	}
-	for (auto &entry_ref : catalog_entries) {
-		auto &entry = entry_ref.get();
-		if (entry.type == CatalogType::TABLE_ENTRY) {
-			auto &table = entry.Cast<DuckTableEntry>();
-			auto &storage = table.GetStorage();
-			auto segment_info = storage.GetColumnSegmentInfo();
-			for (auto &segment : segment_info) {
-				verify_block_usage_count[segment.block_id]++;
-				if (StringUtil::Contains(segment.segment_info, "Overflow String Block Ids: ")) {
-					auto overflow_blocks = StringUtil::Replace(segment.segment_info, "Overflow String Block Ids: ", "");
-					auto splits = StringUtil::Split(overflow_blocks, ", ");
-					for (auto &split : splits) {
-						auto overflow_block_id = std::stoll(split);
-						verify_block_usage_count[overflow_block_id]++;
+	auto debug_verify_blocks = DBConfig::GetSetting<DebugVerifyBlocksSetting>(db.GetDatabase());
+	if (debug_verify_blocks) {
+		// extend verify_block_usage_count
+		auto metadata_info = storage_manager.GetMetadataInfo();
+		for (auto &info : metadata_info) {
+			verify_block_usage_count[info.block_id]++;
+		}
+		for (auto &entry_ref : catalog_entries) {
+			auto &entry = entry_ref.get();
+			if (entry.type == CatalogType::TABLE_ENTRY) {
+				auto &table = entry.Cast<DuckTableEntry>();
+				auto &storage = table.GetStorage();
+				auto segment_info = storage.GetColumnSegmentInfo();
+				for (auto &segment : segment_info) {
+					verify_block_usage_count[segment.block_id]++;
+					if (StringUtil::Contains(segment.segment_info, "Overflow String Block Ids: ")) {
+						auto overflow_blocks =
+						    StringUtil::Replace(segment.segment_info, "Overflow String Block Ids: ", "");
+						auto splits = StringUtil::Split(overflow_blocks, ", ");
+						for (auto &split : splits) {
+							auto overflow_block_id = std::stoll(split);
+							verify_block_usage_count[overflow_block_id]++;
+						}
 					}
 				}
 			}
 		}
+		block_manager.VerifyBlocks(verify_block_usage_count);
 	}
-	block_manager.VerifyBlocks(verify_block_usage_count);
-#endif
 
 	if (debug_checkpoint_abort == CheckpointAbort::DEBUG_ABORT_BEFORE_TRUNCATE) {
 		throw FatalException("Checkpoint aborted before truncate because of PRAGMA checkpoint_abort flag");

package/src/duckdb/src/storage/compression/validity_uncompressed.cpp

@@ -287,6 +287,13 @@ void ValidityUncompressed::UnalignedScan(data_ptr_t input, idx_t input_size, idx
 			// otherwise the subsequent bitwise & will modify values outside of the range of values we want to alter
 			input_mask |= ValidityUncompressed::UPPER_MASKS[shift_amount];
 
+			if (pos == 0) {
+				// We also need to set the lower bits, which are to the left of the relevant bits (x), to 1
+				// These are the bits that are "behind" this scan window, and should not affect this scan
+				auto non_relevant_mask = ValidityUncompressed::LOWER_MASKS[result_idx];
+				input_mask |= non_relevant_mask;
+			}
+
 			// after this, we move to the next input_entry
 			offset = ValidityMask::BITS_PER_VALUE - input_idx;
 			input_entry++;