duckdb 1.1.4-dev9.0 → 1.2.0

package/src/duckdb/src/execution/index/art/art.cpp

@@ -45,7 +45,7 @@ ART::ART(const string &name, const IndexConstraintType index_constraint_type, co
          const shared_ptr<array<unsafe_unique_ptr<FixedSizeAllocator>, ALLOCATOR_COUNT>> &allocators_ptr,
          const IndexStorageInfo &info)
     : BoundIndex(name, ART::TYPE_NAME, index_constraint_type, column_ids, table_io_manager, unbound_expressions, db),
-      allocators(allocators_ptr), owns_data(false), append_mode(ARTAppendMode::DEFAULT) {
+      allocators(allocators_ptr), owns_data(false) {
 
 	// FIXME: Use the new byte representation function to support nested types.
 	for (idx_t i = 0; i < types.size(); i++) {
@@ -480,10 +480,11 @@ bool ART::Construct(unsafe_vector<ARTKey> &keys, unsafe_vector<ARTKey> &row_ids,
 //===--------------------------------------------------------------------===//
 
 ErrorData ART::Insert(IndexLock &l, DataChunk &chunk, Vector &row_ids) {
-	return Insert(l, chunk, row_ids, nullptr);
+	IndexAppendInfo info;
+	return Insert(l, chunk, row_ids, info);
 }
 
-ErrorData ART::Insert(IndexLock &l, DataChunk &chunk, Vector &row_ids, optional_ptr<BoundIndex> delete_index) {
+ErrorData ART::Insert(IndexLock &l, DataChunk &chunk, Vector &row_ids, IndexAppendInfo &info) {
 	D_ASSERT(row_ids.GetType().InternalType() == ROW_TYPE);
 	auto row_count = chunk.size();
 
@@ -493,8 +494,8 @@ ErrorData ART::Insert(IndexLock &l, DataChunk &chunk, Vector &row_ids, optional_
 	GenerateKeyVectors(allocator, chunk, row_ids, keys, row_id_keys);
 
 	optional_ptr<ART> delete_art;
-	if (delete_index) {
-		delete_art = delete_index->Cast<ART>();
+	if (info.delete_index) {
+		delete_art = info.delete_index->Cast<ART>();
 	}
 
 	auto conflict_type = ARTConflictType::NO_CONFLICT;
@@ -506,7 +507,7 @@ ErrorData ART::Insert(IndexLock &l, DataChunk &chunk, Vector &row_ids, optional_
 		if (keys[i].Empty()) {
 			continue;
 		}
-		conflict_type = Insert(tree, keys[i], 0, row_id_keys[i], tree.GetGateStatus(), delete_art);
+		conflict_type = Insert(tree, keys[i], 0, row_id_keys[i], tree.GetGateStatus(), delete_art, info.append_mode);
 		if (conflict_type != ARTConflictType::NO_CONFLICT) {
 			conflict_idx = i;
 			break;
@@ -557,27 +558,27 @@ ErrorData ART::Append(IndexLock &l, DataChunk &chunk, Vector &row_ids) {
 	ExecuteExpressions(chunk, expr_chunk);
 
 	// Now insert the data chunk.
-	return Insert(l, expr_chunk, row_ids, nullptr);
+	IndexAppendInfo info;
+	return Insert(l, expr_chunk, row_ids, info);
 }
 
-ErrorData ART::AppendWithDeleteIndex(IndexLock &l, DataChunk &chunk, Vector &row_ids,
-                                     optional_ptr<BoundIndex> delete_index) {
+ErrorData ART::Append(IndexLock &l, DataChunk &chunk, Vector &row_ids, IndexAppendInfo &info) {
 	// Execute all column expressions before inserting the data chunk.
 	DataChunk expr_chunk;
 	expr_chunk.Initialize(Allocator::DefaultAllocator(), logical_types);
 	ExecuteExpressions(chunk, expr_chunk);
 
 	// Now insert the data chunk.
-	return Insert(l, expr_chunk, row_ids, delete_index);
+	return Insert(l, expr_chunk, row_ids, info);
 }
 
-void ART::VerifyAppend(DataChunk &chunk, optional_ptr<BoundIndex> delete_index, optional_ptr<ConflictManager> manager) {
+void ART::VerifyAppend(DataChunk &chunk, IndexAppendInfo &info, optional_ptr<ConflictManager> manager) {
 	if (manager) {
 		D_ASSERT(manager->LookupType() == VerifyExistenceType::APPEND);
-		return VerifyConstraint(chunk, delete_index, *manager);
+		return VerifyConstraint(chunk, info, *manager);
 	}
 	ConflictManager local_manager(VerifyExistenceType::APPEND, chunk.size());
-	VerifyConstraint(chunk, delete_index, local_manager);
+	VerifyConstraint(chunk, info, local_manager);
 }
 
 void ART::InsertIntoEmpty(Node &node, const ARTKey &key, const idx_t depth, const ARTKey &row_id,
@@ -598,15 +599,16 @@ void ART::InsertIntoEmpty(Node &node, const ARTKey &key, const idx_t depth, cons
 }
 
 ARTConflictType ART::InsertIntoInlined(Node &node, const ARTKey &key, const idx_t depth, const ARTKey &row_id,
-                                       const GateStatus status, optional_ptr<ART> delete_art) {
+                                       const GateStatus status, optional_ptr<ART> delete_art,
+                                       const IndexAppendMode append_mode) {
 
-	if (!IsUnique() || append_mode == ARTAppendMode::INSERT_DUPLICATES) {
+	if (!IsUnique() || append_mode == IndexAppendMode::INSERT_DUPLICATES) {
 		Leaf::InsertIntoInlined(*this, node, row_id, depth, status);
 		return ARTConflictType::NO_CONFLICT;
 	}
 
 	if (!delete_art) {
-		if (append_mode == ARTAppendMode::IGNORE_DUPLICATES) {
+		if (append_mode == IndexAppendMode::IGNORE_DUPLICATES) {
 			return ARTConflictType::NO_CONFLICT;
 		}
 		return ARTConflictType::CONSTRAINT;
@@ -633,14 +635,15 @@ ARTConflictType ART::InsertIntoInlined(Node &node, const ARTKey &key, const idx_
 }
 
 ARTConflictType ART::InsertIntoNode(Node &node, const ARTKey &key, const idx_t depth, const ARTKey &row_id,
-                                    const GateStatus status, optional_ptr<ART> delete_art) {
+                                    const GateStatus status, optional_ptr<ART> delete_art,
+                                    const IndexAppendMode append_mode) {
 	D_ASSERT(depth < key.len);
 	auto child = node.GetChildMutable(*this, key[depth]);
 
 	// Recurse, if a child exists at key[depth].
 	if (child) {
 		D_ASSERT(child->HasMetadata());
-		auto conflict_type = Insert(*child, key, depth + 1, row_id, status, delete_art);
+		auto conflict_type = Insert(*child, key, depth + 1, row_id, status, delete_art, append_mode);
 		node.ReplaceChild(*this, key[depth], *child);
 		return conflict_type;
 	}
@@ -649,7 +652,7 @@ ARTConflictType ART::InsertIntoNode(Node &node, const ARTKey &key, const idx_t d
 	if (status == GateStatus::GATE_SET) {
 		Node remainder;
 		auto byte = key[depth];
-		auto conflict_type = Insert(remainder, key, depth + 1, row_id, status, delete_art);
+		auto conflict_type = Insert(remainder, key, depth + 1, row_id, status, delete_art, append_mode);
 		Node::InsertChild(*this, node, byte, remainder);
 		return conflict_type;
 	}
@@ -671,7 +674,7 @@ ARTConflictType ART::InsertIntoNode(Node &node, const ARTKey &key, const idx_t d
 }
 
 ARTConflictType ART::Insert(Node &node, const ARTKey &key, idx_t depth, const ARTKey &row_id, const GateStatus status,
-                            optional_ptr<ART> delete_art) {
+                            optional_ptr<ART> delete_art, const IndexAppendMode append_mode) {
 	if (!node.HasMetadata()) {
 		InsertIntoEmpty(node, key, depth, row_id, status);
 		return ARTConflictType::NO_CONFLICT;
@@ -688,17 +691,17 @@ ARTConflictType ART::Insert(Node &node, const ARTKey &key, idx_t depth, const AR
 			// incoming transaction must fail here.
 			return ARTConflictType::TRANSACTION;
 		}
-		return Insert(node, row_id, 0, row_id, GateStatus::GATE_SET, delete_art);
+		return Insert(node, row_id, 0, row_id, GateStatus::GATE_SET, delete_art, append_mode);
 	}
 
 	auto type = node.GetType();
 	switch (type) {
 	case NType::LEAF_INLINED: {
-		return InsertIntoInlined(node, key, depth, row_id, status, delete_art);
+		return InsertIntoInlined(node, key, depth, row_id, status, delete_art, append_mode);
 	}
 	case NType::LEAF: {
 		Leaf::TransformToNested(*this, node);
-		return Insert(node, key, depth, row_id, status, delete_art);
+		return Insert(node, key, depth, row_id, status, delete_art, append_mode);
 	}
 	case NType::NODE_7_LEAF:
 	case NType::NODE_15_LEAF:
@@ -712,9 +715,9 @@ ARTConflictType ART::Insert(Node &node, const ARTKey &key, idx_t depth, const AR
 	case NType::NODE_16:
 	case NType::NODE_48:
 	case NType::NODE_256:
-		return InsertIntoNode(node, key, depth, row_id, status, delete_art);
+		return InsertIntoNode(node, key, depth, row_id, status, delete_art, append_mode);
 	case NType::PREFIX:
-		return Prefix::Insert(*this, node, key, depth, row_id, status, delete_art);
+		return Prefix::Insert(*this, node, key, depth, row_id, status, delete_art, append_mode);
 	default:
 		throw InternalException("Invalid node type for ART::Insert.");
 	}
@@ -732,6 +735,8 @@ void ART::CommitDrop(IndexLock &index_lock) {
 }
 
 void ART::Delete(IndexLock &state, DataChunk &input, Vector &row_ids) {
+	// FIXME: We could pass a row_count in here, as we sometimes don't have to delete all row IDs in the chunk,
+	// FIXME: but rather all row IDs up to the conflicting row.
 	auto row_count = input.size();
 
 	DataChunk expr_chunk;
@@ -777,10 +782,9 @@ void ART::Erase(Node &node, reference<const ARTKey> key, idx_t depth, reference<
 	// Traverse the prefix.
 	reference<Node> next(node);
 	if (next.get().GetType() == NType::PREFIX) {
-		Prefix::TraverseMutable(*this, next, key, depth);
-
-		// Prefixes don't match: nothing to erase.
-		if (next.get().GetType() == NType::PREFIX && next.get().GetGateStatus() == GateStatus::GATE_NOT_SET) {
+		auto pos = Prefix::TraverseMutable(*this, next, key, depth);
+		if (pos.IsValid()) {
+			// Prefixes don't match: nothing to erase.
 			return;
 		}
 	}
@@ -841,10 +845,9 @@ void ART::Erase(Node &node, reference<const ARTKey> key, idx_t depth, reference<
 	reference<Node> ref(*child);
 
 	if (ref.get().GetType() == NType::PREFIX) {
-		Prefix::TraverseMutable(*this, ref, key, temp_depth);
-
-		// Prefixes don't match: nothing to erase.
-		if (ref.get().GetType() == NType::PREFIX && ref.get().GetGateStatus() == GateStatus::GATE_NOT_SET) {
+		auto pos = Prefix::TraverseMutable(*this, ref, key, temp_depth);
+		if (pos.IsValid()) {
+			// Prefixes don't match: nothing to erase.
 			return;
 		}
 	}
@@ -880,8 +883,8 @@ const unsafe_optional_ptr<const Node> ART::Lookup(const Node &node, const ARTKey
 
 		// Traverse the prefix.
 		if (ref.get().GetType() == NType::PREFIX) {
-			Prefix::Traverse(*this, ref, key, depth);
-			if (ref.get().GetType() == NType::PREFIX && ref.get().GetGateStatus() == GateStatus::GATE_NOT_SET) {
+			auto pos = Prefix::Traverse(*this, ref, key, depth);
+			if (pos.IsValid()) {
 				// Prefix mismatch, return nullptr.
 				return nullptr;
 			}
@@ -1056,9 +1059,12 @@ void ART::VerifyLeaf(const Node &leaf, const ARTKey &key, optional_ptr<ART> dele
 
 	// Get the delete_leaf.
 	// All leaves in the delete ART are inlined.
-	auto deleted_leaf = delete_art->Lookup(delete_art->tree, key, 0);
+	unsafe_optional_ptr<const Node> deleted_leaf;
+	if (delete_art) {
+		deleted_leaf = delete_art->Lookup(delete_art->tree, key, 0);
+	}
 
-	// The leaf is inlined, and the same key does not exist in the delete ART.
+	// The leaf is inlined, and there is no deleted leaf with the same key.
 	if (leaf.GetType() == NType::LEAF_INLINED && !deleted_leaf) {
 		if (manager.AddHit(i, leaf.GetRowId())) {
 			conflict_idx = i;
@@ -1068,6 +1074,7 @@ void ART::VerifyLeaf(const Node &leaf, const ARTKey &key, optional_ptr<ART> dele
 
 	// The leaf is inlined, and the same key exists in the delete ART.
 	if (leaf.GetType() == NType::LEAF_INLINED && deleted_leaf) {
+		D_ASSERT(deleted_leaf->GetType() == NType::LEAF_INLINED);
 		auto deleted_row_id = deleted_leaf->GetRowId();
 		auto this_row_id = leaf.GetRowId();
 
@@ -1084,6 +1091,10 @@ void ART::VerifyLeaf(const Node &leaf, const ARTKey &key, optional_ptr<ART> dele
 		return;
 	}
 
+	// FIXME: proper foreign key + delete ART support.
+	// This implicitly works for foreign keys, as we do not have to consider the actual row IDs.
+	// We only need to know that there are conflicts (for now), as we still perform over-eager constraint checking.
+
 	// Scan the two row IDs in the leaf.
 	Iterator it(*this);
 	it.FindMinimum(leaf);
@@ -1092,14 +1103,13 @@ void ART::VerifyLeaf(const Node &leaf, const ARTKey &key, optional_ptr<ART> dele
 	it.Scan(empty_key, 2, row_ids, false);
 
 	if (!deleted_leaf) {
-		if (manager.AddHit(i, row_ids[0]) || manager.AddHit(i, row_ids[0])) {
+		if (manager.AddHit(i, row_ids[0]) || manager.AddHit(i, row_ids[1])) {
 			conflict_idx = i;
 		}
 		return;
 	}
 
 	auto deleted_row_id = deleted_leaf->GetRowId();
-
 	if (deleted_row_id == row_ids[0] || deleted_row_id == row_ids[1]) {
 		if (manager.AddMiss(i)) {
 			conflict_idx = i;
@@ -1112,7 +1122,7 @@ void ART::VerifyLeaf(const Node &leaf, const ARTKey &key, optional_ptr<ART> dele
 	}
 }
 
-void ART::VerifyConstraint(DataChunk &chunk, optional_ptr<BoundIndex> delete_index, ConflictManager &manager) {
+void ART::VerifyConstraint(DataChunk &chunk, IndexAppendInfo &info, ConflictManager &manager) {
 	// Lock the index during constraint checking.
 	lock_guard<mutex> l(lock);
 
@@ -1125,8 +1135,8 @@ void ART::VerifyConstraint(DataChunk &chunk, optional_ptr<BoundIndex> delete_ind
 	GenerateKeys<>(arena_allocator, expr_chunk, keys);
 
 	optional_ptr<ART> delete_art;
-	if (delete_index) {
-		delete_art = delete_index->Cast<ART>();
+	if (info.delete_index) {
+		delete_art = info.delete_index->Cast<ART>();
 	}
 
 	optional_idx conflict_idx;

package/src/duckdb/src/execution/index/art/leaf.cpp

@@ -26,7 +26,7 @@ void Leaf::New(ART &art, reference<Node> &node, const unsafe_vector<ARTKey> &row
 	// We cannot recurse into the leaf during Construct(...) because row IDs are not sorted.
 	for (idx_t i = 0; i < count; i++) {
 		idx_t offset = start + i;
-		art.Insert(node, row_ids[offset], 0, row_ids[offset], GateStatus::GATE_SET, nullptr);
+		art.Insert(node, row_ids[offset], 0, row_ids[offset], GateStatus::GATE_SET, nullptr, IndexAppendMode::DEFAULT);
 	}
 	node.get().SetGateStatus(GateStatus::GATE_SET);
 }
@@ -36,7 +36,7 @@ void Leaf::MergeInlined(ART &art, Node &l_node, Node &r_node) {
 
 	ArenaAllocator arena_allocator(Allocator::Get(art.db));
 	auto key = ARTKey::CreateARTKey<row_t>(arena_allocator, r_node.GetRowId());
-	art.Insert(l_node, key, 0, key, l_node.GetGateStatus(), nullptr);
+	art.Insert(l_node, key, 0, key, l_node.GetGateStatus(), nullptr, IndexAppendMode::DEFAULT);
 	r_node.Clear();
 }
 
@@ -96,18 +96,14 @@ void Leaf::TransformToNested(ART &art, Node &node) {
 	ArenaAllocator allocator(Allocator::Get(art.db));
 	Node root = Node();
 
-	// Temporarily disable constraint checking.
-	if (art.IsUnique() && art.append_mode == ARTAppendMode::DEFAULT) {
-		art.append_mode = ARTAppendMode::INSERT_DUPLICATES;
-	}
-
 	// Move all row IDs into the nested leaf.
 	reference<const Node> leaf_ref(node);
 	while (leaf_ref.get().HasMetadata()) {
 		auto &leaf = Node::Ref<const Leaf>(art, leaf_ref, LEAF);
 		for (uint8_t i = 0; i < leaf.count; i++) {
 			auto row_id = ARTKey::CreateARTKey<row_t>(allocator, leaf.row_ids[i]);
-			auto conflict_type = art.Insert(root, row_id, 0, row_id, GateStatus::GATE_SET, nullptr);
+			auto conflict_type =
+			    art.Insert(root, row_id, 0, row_id, GateStatus::GATE_SET, nullptr, IndexAppendMode::INSERT_DUPLICATES);
 			if (conflict_type != ARTConflictType::NO_CONFLICT) {
 				throw InternalException("invalid conflict type in Leaf::TransformToNested");
 			}
@@ -115,7 +111,6 @@ void Leaf::TransformToNested(ART &art, Node &node) {
 		leaf_ref = leaf.ptr;
 	}
 
-	art.append_mode = ARTAppendMode::DEFAULT;
 	root.SetGateStatus(GateStatus::GATE_SET);
 	Node::Free(art, node);
 	node = root;

package/src/duckdb/src/execution/index/art/node.cpp

@@ -44,7 +44,7 @@ void Node::New(ART &art, Node &node, NType type) {
 		Node256::New(art, node);
 		break;
 	default:
-		throw InternalException("Invalid node type for New: %d.", static_cast<uint8_t>(type));
+		throw InternalException("Invalid node type for New: %s.", EnumUtil::ToString(type));
 	}
 }
 
@@ -113,7 +113,7 @@ uint8_t Node::GetAllocatorIdx(const NType type) {
 	case NType::NODE_256_LEAF:
 		return 8;
 	default:
-		throw InternalException("Invalid node type for GetAllocatorIdx: %d.", static_cast<uint8_t>(type));
+		throw InternalException("Invalid node type for GetAllocatorIdx: %s.", EnumUtil::ToString(type));
 	}
 }
 
@@ -135,7 +135,7 @@ void Node::ReplaceChild(const ART &art, const uint8_t byte, const Node child) co
 	case NType::NODE_256:
 		return Ref<Node256>(art, *this, type).ReplaceChild(byte, child);
 	default:
-		throw InternalException("Invalid node type for ReplaceChild: %d.", static_cast<uint8_t>(type));
+		throw InternalException("Invalid node type for ReplaceChild: %s.", EnumUtil::ToString(type));
 	}
 }
 
@@ -159,7 +159,7 @@ void Node::InsertChild(ART &art, Node &node, const uint8_t byte, const Node chil
 	case NType::NODE_256_LEAF:
 		return Node256Leaf::InsertByte(art, node, byte);
 	default:
-		throw InternalException("Invalid node type for InsertChild: %d.", static_cast<uint8_t>(type));
+		throw InternalException("Invalid node type for InsertChild: %s.", EnumUtil::ToString(type));
 	}
 }
 
@@ -188,7 +188,7 @@ void Node::DeleteChild(ART &art, Node &node, Node &prefix, const uint8_t byte, c
 	case NType::NODE_256_LEAF:
 		return Node256Leaf::DeleteByte(art, node, byte);
 	default:
-		throw InternalException("Invalid node type for DeleteChild: %d.", static_cast<uint8_t>(type));
+		throw InternalException("Invalid node type for DeleteChild: %s.", EnumUtil::ToString(type));
 	}
 }
 
@@ -212,7 +212,7 @@ unsafe_optional_ptr<Node> GetChildInternal(ART &art, NODE &node, const uint8_t b
 		return Node256::GetChild(Node::Ref<Node256>(art, node, type), byte);
 	}
 	default:
-		throw InternalException("Invalid node type for GetChildInternal: %d.", static_cast<uint8_t>(type));
+		throw InternalException("Invalid node type for GetChildInternal: %s.", EnumUtil::ToString(type));
 	}
 }
 
@@ -239,7 +239,7 @@ unsafe_optional_ptr<Node> GetNextChildInternal(ART &art, NODE &node, uint8_t &by
 	case NType::NODE_256:
 		return Node256::GetNextChild(Node::Ref<Node256>(art, node, type), byte);
 	default:
-		throw InternalException("Invalid node type for GetNextChildInternal: %d.", static_cast<uint8_t>(type));
+		throw InternalException("Invalid node type for GetNextChildInternal: %s.", EnumUtil::ToString(type));
 	}
 }
 
@@ -263,7 +263,7 @@ bool Node::HasByte(ART &art, uint8_t &byte) const {
 	case NType::NODE_256_LEAF:
 		return Ref<Node256Leaf>(art, *this, NType::NODE_256_LEAF).HasByte(byte);
 	default:
-		throw InternalException("Invalid node type for GetNextByte: %d.", static_cast<uint8_t>(type));
+		throw InternalException("Invalid node type for GetNextByte: %s.", EnumUtil::ToString(type));
 	}
 }
 
@@ -279,7 +279,7 @@ bool Node::GetNextByte(ART &art, uint8_t &byte) const {
 	case NType::NODE_256_LEAF:
 		return Ref<Node256Leaf>(art, *this, NType::NODE_256_LEAF).GetNextByte(byte);
 	default:
-		throw InternalException("Invalid node type for GetNextByte: %d.", static_cast<uint8_t>(type));
+		throw InternalException("Invalid node type for GetNextByte: %s.", EnumUtil::ToString(type));
 	}
 }
 
@@ -304,7 +304,7 @@ idx_t GetCapacity(NType type) {
 	case NType::NODE_256:
 		return Node256::CAPACITY;
 	default:
-		throw InternalException("Invalid node type for GetCapacity: %d.", static_cast<uint8_t>(type));
+		throw InternalException("Invalid node type for GetCapacity: %s.", EnumUtil::ToString(type));
 	}
 }
 
@@ -572,7 +572,7 @@ bool Node::MergeInternal(ART &art, Node &other, const GateStatus status) {
 		ArenaAllocator allocator(Allocator::Get(art.db));
 		for (idx_t i = 0; i < row_ids.size(); i++) {
 			auto row_id = ARTKey::CreateARTKey<row_t>(allocator, row_ids[i]);
-			art.Insert(*this, row_id, 0, row_id, GateStatus::GATE_SET, nullptr);
+			art.Insert(*this, row_id, 0, row_id, GateStatus::GATE_SET, nullptr, IndexAppendMode::DEFAULT);
 		}
 		return true;
 	}
@@ -637,7 +637,7 @@ void Node::Vacuum(ART &art, const unordered_set<uint8_t> &indexes) {
 	case NType::NODE_256_LEAF:
 		return;
 	default:
-		throw InternalException("Invalid node type for Vacuum: %d.", static_cast<uint8_t>(type));
+		throw InternalException("Invalid node type for Vacuum: %s.", EnumUtil::ToString(type));
 	}
 }
 
@@ -670,7 +670,7 @@ void Node::TransformToDeprecated(ART &art, Node &node, unsafe_unique_ptr<FixedSi
 	case NType::NODE_256:
 		return TransformToDeprecatedInternal(art, InMemoryRef<Node256>(art, node, type), allocator);
 	default:
-		throw InternalException("Invalid node type for TransformToDeprecated: %d.", static_cast<uint8_t>(type));
+		throw InternalException("Invalid node type for TransformToDeprecated: %s.", EnumUtil::ToString(type));
 	}
 }
 

package/src/duckdb/src/execution/index/art/prefix.cpp

@@ -40,7 +40,7 @@ idx_t Prefix::GetMismatchWithOther(const Prefix &l_prefix, const Prefix &r_prefi
 	return DConstants::INVALID_INDEX;
 }
 
-idx_t Prefix::GetMismatchWithKey(ART &art, const Node &node, const ARTKey &key, idx_t &depth) {
+optional_idx Prefix::GetMismatchWithKey(ART &art, const Node &node, const ARTKey &key, idx_t &depth) {
 	Prefix prefix(art, node);
 	for (idx_t i = 0; i < prefix.data[Prefix::Count(art)]; i++) {
 		if (prefix.data[i] != key[depth]) {
@@ -48,7 +48,7 @@ idx_t Prefix::GetMismatchWithKey(ART &art, const Node &node, const ARTKey &key,
 		}
 		depth++;
 	}
-	return DConstants::INVALID_INDEX;
+	return optional_idx::Invalid();
 }
 
 uint8_t Prefix::GetByte(const ART &art, const Node &node, const uint8_t pos) {
@@ -160,14 +160,14 @@ void Prefix::Concat(ART &art, Node &parent, uint8_t byte, const GateStatus old_s
 }
 
 template <class NODE>
-idx_t TraverseInternal(ART &art, reference<NODE> &node, const ARTKey &key, idx_t &depth,
-                       const bool is_mutable = false) {
+optional_idx TraverseInternal(ART &art, reference<NODE> &node, const ARTKey &key, idx_t &depth,
+                              const bool is_mutable = false) {
 	D_ASSERT(node.get().HasMetadata());
 	D_ASSERT(node.get().GetType() == NType::PREFIX);
 
 	while (node.get().GetType() == NType::PREFIX) {
 		auto pos = Prefix::GetMismatchWithKey(art, node, key, depth);
-		if (pos != DConstants::INVALID_INDEX) {
+		if (pos.IsValid()) {
 			return pos;
 		}
 
@@ -177,14 +177,18 @@ idx_t TraverseInternal(ART &art, reference<NODE> &node, const ARTKey &key, idx_t
 			break;
 		}
 	}
-	return DConstants::INVALID_INDEX;
+
+	// We return an invalid index, if (and only if) the next node is:
+	// 1. not a prefix, or
+	// 2. a gate.
+	return optional_idx::Invalid();
 }
 
-idx_t Prefix::Traverse(ART &art, reference<const Node> &node, const ARTKey &key, idx_t &depth) {
+optional_idx Prefix::Traverse(ART &art, reference<const Node> &node, const ARTKey &key, idx_t &depth) {
 	return TraverseInternal<const Node>(art, node, key, depth);
 }
 
-idx_t Prefix::TraverseMutable(ART &art, reference<Node> &node, const ARTKey &key, idx_t &depth) {
+optional_idx Prefix::TraverseMutable(ART &art, reference<Node> &node, const ARTKey &key, idx_t &depth) {
 	return TraverseInternal<Node>(art, node, key, depth, true);
 }
 
@@ -292,22 +296,22 @@ GateStatus Prefix::Split(ART &art, reference<Node> &node, Node &child, const uin
 }
 
 ARTConflictType Prefix::Insert(ART &art, Node &node, const ARTKey &key, idx_t depth, const ARTKey &row_id,
-                               const GateStatus status, optional_ptr<ART> delete_art) {
+                               const GateStatus status, optional_ptr<ART> delete_art,
+                               const IndexAppendMode append_mode) {
 	reference<Node> next(node);
 	auto pos = TraverseMutable(art, next, key, depth);
 
 	// We recurse into the next node, if
 	// (1) the prefix matches the key.
 	// (2) we reach a gate.
-	if (pos == DConstants::INVALID_INDEX) {
-		if (next.get().GetType() != NType::PREFIX || next.get().GetGateStatus() == GateStatus::GATE_SET) {
-			return art.Insert(next, key, depth, row_id, status, delete_art);
-		}
+	if (!pos.IsValid()) {
+		D_ASSERT(next.get().GetType() != NType::PREFIX || next.get().GetGateStatus() == GateStatus::GATE_SET);
+		return art.Insert(next, key, depth, row_id, status, delete_art, append_mode);
 	}
 
 	Node remainder;
-	auto byte = GetByte(art, next, UnsafeNumericCast<uint8_t>(pos));
-	auto split_status = Split(art, next, remainder, UnsafeNumericCast<uint8_t>(pos));
+	auto byte = GetByte(art, next, UnsafeNumericCast<uint8_t>(pos.GetIndex()));
+	auto split_status = Split(art, next, remainder, UnsafeNumericCast<uint8_t>(pos.GetIndex()));
 	Node4::New(art, next);
 	next.get().SetGateStatus(split_status);
 
@@ -315,7 +319,7 @@ ARTConflictType Prefix::Insert(ART &art, Node &node, const ARTKey &key, idx_t de
 	Node4::InsertChild(art, next, byte, remainder);
 
 	if (status == GateStatus::GATE_SET) {
-		D_ASSERT(pos != ROW_ID_COUNT);
+		D_ASSERT(pos.GetIndex() != ROW_ID_COUNT);
 		Node new_row_id;
 		Leaf::New(new_row_id, key.GetRowId());
 		Node::InsertChild(art, next, key[depth], new_row_id);
@@ -329,6 +333,7 @@ ARTConflictType Prefix::Insert(ART &art, Node &node, const ARTKey &key, idx_t de
 		auto count = key.len - depth - 1;
 		Prefix::New(art, ref, key, depth + 1, count);
 	}
+
 	// Create the inlined leaf.
 	Leaf::New(ref, row_id.GetRowId());
 	Node4::InsertChild(art, next, key[depth], leaf);

package/src/duckdb/src/execution/index/bound_index.cpp

@@ -38,24 +38,22 @@ ErrorData BoundIndex::Append(DataChunk &chunk, Vector &row_ids) {
 	return Append(l, chunk, row_ids);
 }
 
-ErrorData BoundIndex::AppendWithDeleteIndex(IndexLock &l, DataChunk &chunk, Vector &row_ids,
-                                            optional_ptr<BoundIndex> delete_index) {
+ErrorData BoundIndex::Append(IndexLock &l, DataChunk &chunk, Vector &row_ids, IndexAppendInfo &info) {
 	// Fallback to the old Append.
 	return Append(l, chunk, row_ids);
 }
 
-ErrorData BoundIndex::AppendWithDeleteIndex(DataChunk &chunk, Vector &row_ids, optional_ptr<BoundIndex> delete_index) {
+ErrorData BoundIndex::Append(DataChunk &chunk, Vector &row_ids, IndexAppendInfo &info) {
 	IndexLock l;
 	InitializeLock(l);
-	return AppendWithDeleteIndex(l, chunk, row_ids, delete_index);
+	return Append(l, chunk, row_ids, info);
 }
 
-void BoundIndex::VerifyAppend(DataChunk &chunk, optional_ptr<BoundIndex> delete_index,
-                              optional_ptr<ConflictManager> manager) {
+void BoundIndex::VerifyAppend(DataChunk &chunk, IndexAppendInfo &info, optional_ptr<ConflictManager> manager) {
 	throw NotImplementedException("this implementation of VerifyAppend does not exist.");
 }
 
-void BoundIndex::VerifyConstraint(DataChunk &chunk, optional_ptr<BoundIndex> delete_index, ConflictManager &manager) {
+void BoundIndex::VerifyConstraint(DataChunk &chunk, IndexAppendInfo &info, ConflictManager &manager) {
 	throw NotImplementedException("this implementation of VerifyConstraint does not exist.");
 }
 
@@ -71,7 +69,7 @@ void BoundIndex::Delete(DataChunk &entries, Vector &row_identifiers) {
 	Delete(state, entries, row_identifiers);
 }
 
-ErrorData BoundIndex::Insert(IndexLock &l, DataChunk &chunk, Vector &row_ids, optional_ptr<BoundIndex> delete_index) {
+ErrorData BoundIndex::Insert(IndexLock &l, DataChunk &chunk, Vector &row_ids, IndexAppendInfo &info) {
 	throw NotImplementedException("this implementation of Insert does not exist.");
 }