duckdb 0.6.2-dev971.0 → 0.7.1-dev2.0

package/src/duckdb/src/execution/index/art/art.cpp

@@ -5,6 +5,7 @@
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/storage/arena_allocator.hpp"
 #include "duckdb/execution/index/art/art_key.hpp"
+#include "duckdb/common/types/conflict_manager.hpp"
 
 #include <algorithm>
 #include <cstring>
@@ -14,79 +15,78 @@ namespace duckdb {
 
 ART::ART(const vector<column_t> &column_ids, TableIOManager &table_io_manager,
          const vector<unique_ptr<Expression>> &unbound_expressions, IndexConstraintType constraint_type,
-         AttachedDatabase &db, idx_t block_id, idx_t block_offset)
-    : Index(IndexType::ART, table_io_manager, column_ids, unbound_expressions, constraint_type), db(db),
-      estimated_art_size(0), estimated_key_size(16) {
+         AttachedDatabase &db, bool track_memory, idx_t block_id, idx_t block_offset)
+
+    : Index(db, IndexType::ART, table_io_manager, column_ids, unbound_expressions, constraint_type, track_memory) {
+
 	if (!Radix::IsLittleEndian()) {
 		throw NotImplementedException("ART indexes are not supported on big endian architectures");
 	}
+
+	// set the root node of the tree
+	tree = nullptr;
 	if (block_id != DConstants::INVALID_INDEX) {
 		tree = Node::Deserialize(*this, block_id, block_offset);
-	} else {
-		tree = nullptr;
+		ART::Verify();
 	}
 	serialized_data_pointer = BlockPointer(block_id, block_offset);
+
+	// validate the types of the key columns
 	for (idx_t i = 0; i < types.size(); i++) {
 		switch (types[i]) {
 		case PhysicalType::BOOL:
 		case PhysicalType::INT8:
-		case PhysicalType::UINT8:
-			estimated_key_size += sizeof(int8_t);
-			break;
 		case PhysicalType::INT16:
-		case PhysicalType::UINT16:
-			estimated_key_size += sizeof(int16_t);
-			break;
 		case PhysicalType::INT32:
-		case PhysicalType::UINT32:
-		case PhysicalType::FLOAT:
-			estimated_key_size += sizeof(int32_t);
-			break;
 		case PhysicalType::INT64:
+		case PhysicalType::INT128:
+		case PhysicalType::UINT8:
+		case PhysicalType::UINT16:
+		case PhysicalType::UINT32:
 		case PhysicalType::UINT64:
+		case PhysicalType::FLOAT:
 		case PhysicalType::DOUBLE:
-			estimated_key_size += sizeof(int64_t);
-			break;
-		case PhysicalType::INT128:
-			estimated_key_size += sizeof(hugeint_t);
-			break;
 		case PhysicalType::VARCHAR:
-			estimated_key_size += 16; // oh well
 			break;
 		default:
-			throw InvalidTypeException(logical_types[i], "Invalid type for index");
+			throw InvalidTypeException(logical_types[i], "Invalid type for index key.");
 		}
 	}
 }
 
 ART::~ART() {
-	if (estimated_art_size > 0) {
-		BufferManager::GetBufferManager(db).FreeReservedMemory(estimated_art_size);
-		estimated_art_size = 0;
+	if (!tree) {
+		return;
 	}
-	if (tree) {
-		Node::Delete(tree);
-		tree = nullptr;
+	ART::Verify();
+	if (track_memory) {
+		buffer_manager.DecreaseUsedMemory(memory_size);
 	}
+	Node::Delete(tree);
+	tree = nullptr;
 }
 
-unique_ptr<IndexScanState> ART::InitializeScanSinglePredicate(Transaction &transaction, Value value,
+//===--------------------------------------------------------------------===//
+// Initialize Predicate Scans
+//===--------------------------------------------------------------------===//
+
+unique_ptr<IndexScanState> ART::InitializeScanSinglePredicate(const Transaction &transaction, const Value &value,
                                                               ExpressionType expression_type) {
 	auto result = make_unique<ARTIndexScanState>();
 	result->values[0] = value;
 	result->expressions[0] = expression_type;
-	return move(result);
+	return std::move(result);
 }
 
-unique_ptr<IndexScanState> ART::InitializeScanTwoPredicates(Transaction &transaction, Value low_value,
-                                                            ExpressionType low_expression_type, Value high_value,
+unique_ptr<IndexScanState> ART::InitializeScanTwoPredicates(Transaction &transaction, const Value &low_value,
+                                                            ExpressionType low_expression_type, const Value &high_value,
                                                             ExpressionType high_expression_type) {
 	auto result = make_unique<ARTIndexScanState>();
 	result->values[0] = low_value;
 	result->expressions[0] = low_expression_type;
 	result->values[1] = high_value;
 	result->expressions[1] = high_expression_type;
-	return move(result);
+	return std::move(result);
 }
 
 //===--------------------------------------------------------------------===//
@@ -225,7 +225,7 @@ void ART::GenerateKeys(ArenaAllocator &allocator, DataChunk &input, vector<Key>
 }
 
 //===--------------------------------------------------------------------===//
-// Insert
+// Construct from sorted data
 //===--------------------------------------------------------------------===//
 
 struct KeySection {
@@ -251,7 +251,8 @@ void GetChildSections(vector<KeySection> &child_sections, vector<Key> &keys, Key
 	child_sections.emplace_back(child_start_idx, key_section.end, keys, key_section);
 }
 
-void Construct(vector<Key> &keys, row_t *row_ids, Node *&node, KeySection &key_section, bool &has_constraint) {
+bool Construct(ART &art, vector<Key> &keys, row_t *row_ids, Node *&node, KeySection &key_section,
+               bool &has_constraint) {
 
 	D_ASSERT(key_section.start < keys.size());
 	D_ASSERT(key_section.end < keys.size());
@@ -272,90 +273,61 @@ void Construct(vector<Key> &keys, row_t *row_ids, Node *&node, KeySection &key_s
 		auto num_row_ids = key_section.end - key_section.start + 1;
 
 		// check for possible constraint violation
-		if (has_constraint && num_row_ids != 1) {
-			throw ConstraintException("New data contains duplicates on indexed column(s)");
+		auto single_row_id = num_row_ids == 1;
+		if (has_constraint && !single_row_id) {
+			return false;
 		}
 
-		node = Leaf::New(start_key, prefix_start, row_ids + key_section.start, num_row_ids);
-	} else { // create a new node and recurse
-
-		// we will find at least two child entries of this node, otherwise we'd have reached a leaf
-		vector<KeySection> child_sections;
-		GetChildSections(child_sections, keys, key_section);
-
-		auto node_type = Node::GetTypeBySize(child_sections.size());
-		Node::New(node_type, node);
-
-		auto prefix_length = key_section.depth - prefix_start;
-		node->prefix = Prefix(start_key, prefix_start, prefix_length);
-
-		// recurse on each child section
-		for (auto &child_section : child_sections) {
-			Node *new_child = nullptr;
-			Construct(keys, row_ids, new_child, child_section, has_constraint);
-			Node::InsertChild(node, child_section.key_byte, new_child);
+		if (single_row_id) {
+			node = Leaf::New(start_key, prefix_start, row_ids[key_section.start]);
+		} else {
+			node = Leaf::New(start_key, prefix_start, row_ids + key_section.start, num_row_ids);
 		}
+		art.memory_size += node->MemorySize(art, false);
+		return true;
 	}
-}
+	// create a new node and recurse
 
-void ART::ConstructAndMerge(IndexLock &lock, PayloadScanner &scanner, Allocator &allocator) {
+	// we will find at least two child entries of this node, otherwise we'd have reached a leaf
+	vector<KeySection> child_sections;
+	GetChildSections(child_sections, keys, key_section);
 
-	auto payload_types = logical_types;
-	payload_types.emplace_back(LogicalType::ROW_TYPE);
+	auto node_type = Node::GetTypeBySize(child_sections.size());
+	Node::New(node_type, node);
 
-	ArenaAllocator arena_allocator(BufferAllocator::Get(db));
-	vector<Key> keys(STANDARD_VECTOR_SIZE);
-
-	auto temp_art = make_unique<ART>(this->column_ids, this->table_io_manager, this->unbound_expressions,
-	                                 this->constraint_type, this->db);
+	auto prefix_length = key_section.depth - prefix_start;
+	node->prefix = Prefix(start_key, prefix_start, prefix_length);
+	art.memory_size += node->MemorySize(art, false);
 
-	for (;;) {
-		DataChunk ordered_chunk;
-		ordered_chunk.Initialize(allocator, payload_types);
-		ordered_chunk.SetCardinality(0);
-		scanner.Scan(ordered_chunk);
-		if (ordered_chunk.size() == 0) {
-			break;
+	// recurse on each child section
+	for (auto &child_section : child_sections) {
+		Node *new_child = nullptr;
+		auto no_violation = Construct(art, keys, row_ids, new_child, child_section, has_constraint);
+		Node::InsertChild(art, node, child_section.key_byte, new_child);
+		if (!no_violation) {
+			return false;
 		}
+	}
+	return true;
+}
 
-		// get the key chunk and the row_identifiers vector
-		DataChunk row_id_chunk;
-		ordered_chunk.Split(row_id_chunk, ordered_chunk.ColumnCount() - 1);
-		auto &row_identifiers = row_id_chunk.data[0];
-
-		D_ASSERT(row_identifiers.GetType().InternalType() == ROW_TYPE);
-		D_ASSERT(logical_types[0] == ordered_chunk.data[0].GetType());
-
-		// generate the keys for the given input
-		arena_allocator.Reset();
-		GenerateKeys(arena_allocator, ordered_chunk, keys);
-
-		// prepare the row_identifiers
-		row_identifiers.Flatten(ordered_chunk.size());
-		auto row_ids = FlatVector::GetData<row_t>(row_identifiers);
-
-		// construct the ART of this chunk
-		auto art = make_unique<ART>(this->column_ids, this->table_io_manager, this->unbound_expressions,
-		                            this->constraint_type, this->db);
-		auto key_section = KeySection(0, ordered_chunk.size() - 1, 0, 0);
-		auto has_constraint = IsUnique();
-		Construct(keys, row_ids, art->tree, key_section, has_constraint);
+bool ART::ConstructFromSorted(idx_t count, vector<Key> &keys, Vector &row_identifiers) {
 
-		// merge art into temp_art
-		if (!temp_art->MergeIndexes(lock, art.get())) {
-			throw ConstraintException("Data contains duplicates on indexed column(s)");
-		}
-	}
+	// prepare the row_identifiers
+	row_identifiers.Flatten(count);
+	auto row_ids = FlatVector::GetData<row_t>(row_identifiers);
 
-	// NOTE: currently this code is only used for index creation, so we can assume that there are no
-	// duplicate violations between the existing index and the new data,
-	// so we do not need to revert any changes
-	if (!this->MergeIndexes(lock, temp_art.get())) {
-		throw ConstraintException("Data contains duplicates on indexed column(s)");
-	}
+	auto key_section = KeySection(0, count - 1, 0, 0);
+	auto has_constraint = IsUnique();
+	return Construct(*this, keys, row_ids, this->tree, key_section, has_constraint);
 }
 
+//===--------------------------------------------------------------------===//
+// Insert
+//===--------------------------------------------------------------------===//
+
 bool ART::Insert(IndexLock &lock, DataChunk &input, Vector &row_ids) {
+
 	D_ASSERT(row_ids.GetType().InternalType() == ROW_TYPE);
 	D_ASSERT(logical_types[0] == input.data[0].GetType());
 
@@ -364,13 +336,13 @@ bool ART::Insert(IndexLock &lock, DataChunk &input, Vector &row_ids) {
 	vector<Key> keys(input.size());
 	GenerateKeys(arena_allocator, input, keys);
 
-	idx_t extra_memory = estimated_key_size * input.size();
-	BufferManager::GetBufferManager(db).ReserveMemory(extra_memory);
-	estimated_art_size += extra_memory;
+	auto old_memory_size = this->memory_size;
 
-	// now insert the elements into the index
+	// get the corresponding row IDs
 	row_ids.Flatten(input.size());
 	auto row_identifiers = FlatVector::GetData<row_t>(row_ids);
+
+	// now insert the elements into the index
 	idx_t failed_index = DConstants::INVALID_INDEX;
 	for (idx_t i = 0; i < input.size(); i++) {
 		if (keys[i].Empty()) {
@@ -394,8 +366,15 @@ bool ART::Insert(IndexLock &lock, DataChunk &input, Vector &row_ids) {
 			row_t row_id = row_identifiers[i];
 			Erase(tree, keys[i], 0, row_id);
 		}
+		// nothing changed, no need to update the buffer memory size
 		return false;
 	}
+
+	D_ASSERT(old_memory_size <= memory_size);
+	Verify();
+	if (track_memory) {
+		buffer_manager.IncreaseUsedMemory(memory_size - old_memory_size);
+	}
 	return true;
 }
 
@@ -411,15 +390,26 @@ bool ART::Append(IndexLock &lock, DataChunk &appended_data, Vector &row_identifi
 }
 
 void ART::VerifyAppend(DataChunk &chunk) {
-	VerifyExistence(chunk, VerifyExistenceType::APPEND);
+	ConflictManager conflict_manager(VerifyExistenceType::APPEND, chunk.size());
+	LookupValues(chunk, conflict_manager);
+}
+
+void ART::VerifyAppend(DataChunk &chunk, ConflictManager &conflict_manager) {
+	D_ASSERT(conflict_manager.LookupType() == VerifyExistenceType::APPEND);
+	LookupValues(chunk, conflict_manager);
 }
 
-void ART::VerifyAppendForeignKey(DataChunk &chunk, string *err_msg_ptr) {
-	VerifyExistence(chunk, VerifyExistenceType::APPEND_FK, err_msg_ptr);
+void ART::VerifyAppendForeignKey(DataChunk &chunk) {
+	ConflictManager conflict_manager(VerifyExistenceType::APPEND_FK, chunk.size());
+	LookupValues(chunk, conflict_manager);
 }
 
-void ART::VerifyDeleteForeignKey(DataChunk &chunk, string *err_msg_ptr) {
-	VerifyExistence(chunk, VerifyExistenceType::DELETE_FK, err_msg_ptr);
+void ART::VerifyDeleteForeignKey(DataChunk &chunk) {
+	if (!IsUnique()) {
+		return;
+	}
+	ConflictManager conflict_manager(VerifyExistenceType::DELETE_FK, chunk.size());
+	LookupValues(chunk, conflict_manager);
 }
 
 bool ART::InsertToLeaf(Leaf &leaf, row_t row_id) {
@@ -431,7 +421,7 @@ bool ART::InsertToLeaf(Leaf &leaf, row_t row_id) {
 	if (IsUnique() && leaf.count != 0) {
 		return false;
 	}
-	leaf.Insert(row_id);
+	leaf.Insert(*this, row_id);
 	return true;
 }
 
@@ -440,23 +430,24 @@ bool ART::Insert(Node *&node, Key &key, idx_t depth, row_t row_id) {
 	if (!node) {
 		// node is currently empty, create a leaf here with the key
 		node = Leaf::New(key, depth, row_id);
+		this->memory_size += node->MemorySize(*this, false);
 		return true;
 	}
 
 	if (node->type == NodeType::NLeaf) {
-		// Replace leaf with Node4 and store both leaves in it
+		// replace leaf with Node4 and store both leaves in it
+		// or add a row ID to a leaf, if they have the same key
 		auto leaf = (Leaf *)node;
-
-		auto &leaf_prefix = leaf->prefix;
 		uint32_t new_prefix_length = 0;
 
-		// Leaf node is already there (its key matches the current key), update row_id vector
+		// FIXME: this code (if and while) can be optimized, less branching, see Construct
+		// leaf node is already there (its key matches the current key), update row_id vector
 		if (new_prefix_length == leaf->prefix.Size() && depth + leaf->prefix.Size() == key.len) {
 			return InsertToLeaf(*leaf, row_id);
 		}
-		while (leaf_prefix[new_prefix_length] == key[depth + new_prefix_length]) {
+		while (leaf->prefix[new_prefix_length] == key[depth + new_prefix_length]) {
 			new_prefix_length++;
-			// Leaf node is already there (its key matches the current key), update row_id vector
+			// leaf node is already there (its key matches the current key), update row_id vector
 			if (new_prefix_length == leaf->prefix.Size() && depth + leaf->prefix.Size() == key.len) {
 				return InsertToLeaf(*leaf, row_id);
 			}
@@ -464,34 +455,44 @@ bool ART::Insert(Node *&node, Key &key, idx_t depth, row_t row_id) {
 
 		Node *new_node = Node4::New();
 		new_node->prefix = Prefix(key, depth, new_prefix_length);
-		auto key_byte = node->prefix.Reduce(new_prefix_length);
-		Node4::InsertChild(new_node, key_byte, node);
+		this->memory_size += new_node->MemorySize(*this, false);
+
+		auto key_byte = node->prefix.Reduce(*this, new_prefix_length);
+		Node4::InsertChild(*this, new_node, key_byte, node);
+
 		Node *leaf_node = Leaf::New(key, depth + new_prefix_length + 1, row_id);
-		Node4::InsertChild(new_node, key[depth + new_prefix_length], leaf_node);
+		Node4::InsertChild(*this, new_node, key[depth + new_prefix_length], leaf_node);
+		this->memory_size += leaf_node->MemorySize(*this, false);
+
 		node = new_node;
 		return true;
 	}
 
-	// Handle prefix of inner node
+	// handle prefix of inner node
 	if (node->prefix.Size()) {
+
 		uint32_t mismatch_pos = node->prefix.KeyMismatchPosition(key, depth);
 		if (mismatch_pos != node->prefix.Size()) {
-			// Prefix differs, create new node
+			// prefix differs, create new node
 			Node *new_node = Node4::New();
 			new_node->prefix = Prefix(key, depth, mismatch_pos);
-			// Break up prefix
-			auto key_byte = node->prefix.Reduce(mismatch_pos);
-			Node4::InsertChild(new_node, key_byte, node);
+			this->memory_size += new_node->MemorySize(*this, false);
+
+			// break up prefix
+			auto key_byte = node->prefix.Reduce(*this, mismatch_pos);
+			Node4::InsertChild(*this, new_node, key_byte, node);
 
 			Node *leaf_node = Leaf::New(key, depth + mismatch_pos + 1, row_id);
-			Node4::InsertChild(new_node, key[depth + mismatch_pos], leaf_node);
+			Node4::InsertChild(*this, new_node, key[depth + mismatch_pos], leaf_node);
+			this->memory_size += leaf_node->MemorySize(*this, false);
+
 			node = new_node;
 			return true;
 		}
 		depth += node->prefix.Size();
 	}
 
-	// Recurse
+	// recurse
 	D_ASSERT(depth < key.len);
 	idx_t pos = node->GetChildPos(key[depth]);
 	if (pos != DConstants::INVALID_INDEX) {
@@ -500,30 +501,32 @@ bool ART::Insert(Node *&node, Key &key, idx_t depth, row_t row_id) {
 		node->ReplaceChildPointer(pos, child);
 		return insertion_result;
 	}
-	Node *new_node = Leaf::New(key, depth + 1, row_id);
-	Node::InsertChild(node, key[depth], new_node);
+
+	Node *leaf_node = Leaf::New(key, depth + 1, row_id);
+	Node::InsertChild(*this, node, key[depth], leaf_node);
+	this->memory_size += leaf_node->MemorySize(*this, false);
 	return true;
 }
 
 //===--------------------------------------------------------------------===//
 // Delete
 //===--------------------------------------------------------------------===//
+
 void ART::Delete(IndexLock &state, DataChunk &input, Vector &row_ids) {
+
 	DataChunk expression;
 	expression.Initialize(Allocator::DefaultAllocator(), logical_types);
 
 	// first resolve the expressions
 	ExecuteExpressions(input, expression);
 
-	idx_t released_memory = MinValue<idx_t>(estimated_art_size, estimated_key_size * input.size());
-	BufferManager::GetBufferManager(db).FreeReservedMemory(released_memory);
-	estimated_art_size -= released_memory;
-
 	// then generate the keys for the given input
 	ArenaAllocator arena_allocator(BufferAllocator::Get(db));
 	vector<Key> keys(expression.size());
 	GenerateKeys(arena_allocator, expression, keys);
 
+	auto old_memory_size = this->memory_size;
+
 	// now erase the elements from the database
 	row_ids.Flatten(input.size());
 	auto row_identifiers = FlatVector::GetData<row_t>(row_ids);
@@ -536,54 +539,66 @@ void ART::Delete(IndexLock &state, DataChunk &input, Vector &row_ids) {
 #ifdef DEBUG
 		auto node = Lookup(tree, keys[i], 0);
 		if (node) {
-			auto leaf = static_cast<Leaf *>(node);
+			auto leaf = (Leaf *)node;
 			for (idx_t k = 0; k < leaf->count; k++) {
 				D_ASSERT(leaf->GetRowId(k) != row_identifiers[i]);
 			}
 		}
 #endif
 	}
+
+	D_ASSERT(old_memory_size >= memory_size);
+	Verify();
+	if (track_memory) {
+		buffer_manager.DecreaseUsedMemory(old_memory_size - memory_size);
+	}
 }
 
 void ART::Erase(Node *&node, Key &key, idx_t depth, row_t row_id) {
+
 	if (!node) {
 		return;
 	}
-	// Delete a leaf from a tree
+
+	// delete a leaf from a tree
 	if (node->type == NodeType::NLeaf) {
-		// Make sure we have the right leaf
-		auto leaf = static_cast<Leaf *>(node);
-		leaf->Remove(row_id);
+		auto leaf = (Leaf *)node;
+		leaf->Remove(*this, row_id);
+
 		if (leaf->count == 0) {
+			D_ASSERT(this->memory_size >= leaf->MemorySize(*this, false));
+			this->memory_size -= leaf->MemorySize(*this, false);
 			Node::Delete(node);
 			node = nullptr;
 		}
-
 		return;
 	}
 
-	// Handle prefix
+	// handle prefix
 	if (node->prefix.Size()) {
 		if (node->prefix.KeyMismatchPosition(key, depth) != node->prefix.Size()) {
 			return;
 		}
 		depth += node->prefix.Size();
 	}
+
 	idx_t pos = node->GetChildPos(key[depth]);
 	if (pos != DConstants::INVALID_INDEX) {
 		auto child = node->GetChild(*this, pos);
 		D_ASSERT(child);
 
 		if (child->type == NodeType::NLeaf) {
-			// Leaf found, remove entry
+			// leaf found, remove entry
 			auto leaf = (Leaf *)child;
-			leaf->Remove(row_id);
+			leaf->Remove(*this, row_id);
+
 			if (leaf->count == 0) {
-				// Leaf is empty, delete leaf, decrement node counter and maybe shrink node
-				Node::EraseChild(node, pos, *this);
+				// leaf is empty, delete leaf, decrement node counter and maybe shrink node
+				Node::EraseChild(*this, node, pos);
 			}
+
 		} else {
-			// Recurse
+			// recurse
 			Erase(child, key, depth + 1, row_id);
 			node->ReplaceChildPointer(pos, child);
 		}
@@ -593,6 +608,7 @@ void ART::Erase(Node *&node, Key &key, idx_t depth, row_t row_id) {
 //===--------------------------------------------------------------------===//
 // Point Query
 //===--------------------------------------------------------------------===//
+
 static Key CreateKey(ArenaAllocator &allocator, PhysicalType type, Value &value) {
 	D_ASSERT(type == value.type().InternalType());
 	switch (type) {
@@ -629,7 +645,7 @@ static Key CreateKey(ArenaAllocator &allocator, PhysicalType type, Value &value)
 
 bool ART::SearchEqual(Key &key, idx_t max_count, vector<row_t> &result_ids) {
 
-	auto leaf = static_cast<Leaf *>(Lookup(tree, key, 0));
+	auto leaf = (Leaf *)(Lookup(tree, key, 0));
 	if (!leaf) {
 		return true;
 	}
@@ -690,6 +706,7 @@ Leaf *ART::Lookup(Node *node, Key &key, idx_t depth) {
 // Returns: True (If found leaf >= key)
 //          False (Otherwise)
 //===--------------------------------------------------------------------===//
+
 bool ART::SearchGreater(ARTIndexScanState *state, Key &key, bool inclusive, idx_t max_count,
                         vector<row_t> &result_ids) {
 
@@ -713,6 +730,7 @@ bool ART::SearchGreater(ARTIndexScanState *state, Key &key, bool inclusive, idx_
 //===--------------------------------------------------------------------===//
 // Less Than
 //===--------------------------------------------------------------------===//
+
 bool ART::SearchLess(ARTIndexScanState *state, Key &upper_bound, bool inclusive, idx_t max_count,
                      vector<row_t> &result_ids) {
 
@@ -738,6 +756,7 @@ bool ART::SearchLess(ARTIndexScanState *state, Key &upper_bound, bool inclusive,
 //===--------------------------------------------------------------------===//
 // Closed Range Query
 //===--------------------------------------------------------------------===//
+
 bool ART::SearchCloseRange(ARTIndexScanState *state, Key &lower_bound, Key &upper_bound, bool left_inclusive,
                            bool right_inclusive, idx_t max_count, vector<row_t> &result_ids) {
 
@@ -825,74 +844,97 @@ bool ART::Scan(Transaction &transaction, DataTable &table, IndexScanState &table
 	return true;
 }
 
-void ART::VerifyExistence(DataChunk &chunk, VerifyExistenceType verify_type, string *err_msg_ptr) {
-	if (verify_type != VerifyExistenceType::DELETE_FK && !IsUnique()) {
-		return;
-	}
-
+string ART::GenerateErrorKeyName(DataChunk &input, idx_t row) {
+	// re-executing the expressions is not very fast, but we're going to throw anyways, so we don't care
 	DataChunk expression_chunk;
 	expression_chunk.Initialize(Allocator::DefaultAllocator(), logical_types);
+	ExecuteExpressions(input, expression_chunk);
 
-	// unique index, check
+	string key_name;
+	for (idx_t k = 0; k < expression_chunk.ColumnCount(); k++) {
+		if (k > 0) {
+			key_name += ", ";
+		}
+		key_name += unbound_expressions[k]->GetName() + ": " + expression_chunk.data[k].GetValue(row).ToString();
+	}
+	return key_name;
+}
+
+string ART::GenerateConstraintErrorMessage(VerifyExistenceType verify_type, const string &key_name) {
+	switch (verify_type) {
+	case VerifyExistenceType::APPEND: {
+		// APPEND to PK/UNIQUE table, but node/key already exists in PK/UNIQUE table
+		string type = IsPrimary() ? "primary key" : "unique";
+		return StringUtil::Format("Duplicate key \"%s\" violates %s constraint", key_name, type);
+	}
+	case VerifyExistenceType::APPEND_FK: {
+		// APPEND_FK to FK table, node/key does not exist in PK/UNIQUE table
+		return StringUtil::Format(
+		    "Violates foreign key constraint because key \"%s\" does not exist in the referenced table", key_name);
+	}
+	case VerifyExistenceType::DELETE_FK: {
+		// DELETE_FK that still exists in a FK table, i.e., not a valid delete
+		return StringUtil::Format("Violates foreign key constraint because key \"%s\" is still referenced by a foreign "
+		                          "key in a different table",
+		                          key_name);
+	}
+	default:
+		throw NotImplementedException("Type not implemented for VerifyExistenceType");
+	}
+}
+
+void ART::LookupValues(DataChunk &input, ConflictManager &conflict_manager) {
+
+	// don't alter the index during constraint checking
 	lock_guard<mutex> l(lock);
+
 	// first resolve the expressions for the index
-	ExecuteExpressions(chunk, expression_chunk);
+	DataChunk expression_chunk;
+	expression_chunk.Initialize(Allocator::DefaultAllocator(), logical_types);
+	ExecuteExpressions(input, expression_chunk);
 
 	// generate the keys for the given input
 	ArenaAllocator arena_allocator(BufferAllocator::Get(db));
 	vector<Key> keys(expression_chunk.size());
 	GenerateKeys(arena_allocator, expression_chunk, keys);
 
-	for (idx_t i = 0; i < chunk.size(); i++) {
+	idx_t found_conflict = DConstants::INVALID_INDEX;
+	for (idx_t i = 0; found_conflict == DConstants::INVALID_INDEX && i < input.size(); i++) {
 		if (keys[i].Empty()) {
+			if (conflict_manager.AddNull(i)) {
+				found_conflict = i;
+			}
 			continue;
 		}
-		Node *node_ptr = Lookup(tree, keys[i], 0);
-		bool throw_exception =
-		    verify_type == VerifyExistenceType::APPEND_FK ? node_ptr == nullptr : node_ptr != nullptr;
-		if (!throw_exception) {
-			continue;
-		}
-		string key_name;
-		for (idx_t k = 0; k < expression_chunk.ColumnCount(); k++) {
-			if (k > 0) {
-				key_name += ", ";
+		Leaf *leaf_ptr = Lookup(tree, keys[i], 0);
+		if (leaf_ptr == nullptr) {
+			if (conflict_manager.AddMiss(i)) {
+				found_conflict = i;
 			}
-			key_name += unbound_expressions[k]->GetName() + ": " + expression_chunk.data[k].GetValue(i).ToString();
-		}
-		string exception_msg;
-		switch (verify_type) {
-		case VerifyExistenceType::APPEND: {
-			// node already exists in tree
-			string type = IsPrimary() ? "primary key" : "unique";
-			exception_msg = "duplicate key \"" + key_name + "\" violates ";
-			exception_msg += type + " constraint";
-			break;
-		}
-		case VerifyExistenceType::APPEND_FK: {
-			// found node no exists in tree
-			exception_msg =
-			    "violates foreign key constraint because key \"" + key_name + "\" does not exist in referenced table";
-			break;
-		}
-		case VerifyExistenceType::DELETE_FK: {
-			// found node exists in tree
-			exception_msg =
-			    "violates foreign key constraint because key \"" + key_name + "\" exist in table has foreign key";
-			break;
-		}
+			continue;
 		}
-		if (err_msg_ptr) {
-			err_msg_ptr[i] = exception_msg;
-		} else {
-			throw ConstraintException(exception_msg);
+		// When we find a node, we need to update the 'matches' and 'row_ids'
+		// NOTE: Leafs can have more than one row_id, but for UNIQUE/PRIMARY KEY they will only have one
+		D_ASSERT(leaf_ptr->count == 1);
+		auto row_id = leaf_ptr->GetRowId(0);
+		if (conflict_manager.AddHit(i, row_id)) {
+			found_conflict = i;
 		}
 	}
+	conflict_manager.FinishLookup();
+	if (found_conflict == DConstants::INVALID_INDEX) {
+		// No conflicts detected
+		return;
+	}
+	auto key_name = GenerateErrorKeyName(input, found_conflict);
+	auto exception_msg = GenerateConstraintErrorMessage(conflict_manager.LookupType(), key_name);
+	throw ConstraintException(exception_msg);
 }
 
 //===--------------------------------------------------------------------===//
 // Serialization
 //===--------------------------------------------------------------------===//
+
 BlockPointer ART::Serialize(duckdb::MetaBlockWriter &writer) {
 	lock_guard<mutex> l(lock);
 	if (tree) {
@@ -904,13 +946,15 @@ BlockPointer ART::Serialize(duckdb::MetaBlockWriter &writer) {
 }
 
 //===--------------------------------------------------------------------===//
-// Merge ARTs
+// Merging
 //===--------------------------------------------------------------------===//
+
 bool ART::MergeIndexes(IndexLock &state, Index *other_index) {
+
 	auto other_art = (ART *)other_index;
-	estimated_art_size += other_art->estimated_art_size;
-	other_art->estimated_art_size = 0;
+
 	if (!this->tree) {
+		this->memory_size += other_art->memory_size;
 		this->tree = other_art->tree;
 		other_art->tree = nullptr;
 		return true;
@@ -919,6 +963,10 @@ bool ART::MergeIndexes(IndexLock &state, Index *other_index) {
 	return Node::MergeARTs(this, other_art);
 }
 
+//===--------------------------------------------------------------------===//
+// Utility
+//===--------------------------------------------------------------------===//
+
 string ART::ToString() {
 	if (tree) {
 		return tree->ToString(*this);
@@ -926,4 +974,17 @@ string ART::ToString() {
 	return "[empty]";
 }
 
+void ART::Verify() {
+#ifdef DEBUG
+	idx_t current_mem_size = 0;
+	if (tree) {
+		current_mem_size = tree->MemorySize(*this, true);
+	}
+	if (memory_size != current_mem_size) {
+		throw InternalException("Memory_size value (%d) does not match actual memory size (%d).", memory_size,
+		                        current_mem_size);
+	}
+#endif
+}
+
 } // namespace duckdb