duckdb 0.7.1-dev37.0 → 0.7.1-dev407.0

package/src/duckdb/src/execution/index/art/art.cpp

@@ -27,7 +27,10 @@ ART::ART(const vector<column_t> &column_ids, TableIOManager &table_io_manager,
 	tree = nullptr;
 	if (block_id != DConstants::INVALID_INDEX) {
 		tree = Node::Deserialize(*this, block_id, block_offset);
-		ART::Verify();
+		Verify();
+		if (track_memory) {
+			buffer_manager.IncreaseUsedMemory(memory_size);
+		}
 	}
 	serialized_data_pointer = BlockPointer(block_id, block_offset);
 
@@ -58,7 +61,7 @@ ART::~ART() {
 	if (!tree) {
 		return;
 	}
-	ART::Verify();
+	Verify();
 	if (track_memory) {
 		buffer_manager.DecreaseUsedMemory(memory_size);
 	}
@@ -72,6 +75,7 @@ ART::~ART() {
 
 unique_ptr<IndexScanState> ART::InitializeScanSinglePredicate(const Transaction &transaction, const Value &value,
                                                               ExpressionType expression_type) {
+	// initialize point lookup
 	auto result = make_unique<ARTIndexScanState>();
 	result->values[0] = value;
 	result->expressions[0] = expression_type;
@@ -81,6 +85,7 @@ unique_ptr<IndexScanState> ART::InitializeScanSinglePredicate(const Transaction
 unique_ptr<IndexScanState> ART::InitializeScanTwoPredicates(Transaction &transaction, const Value &low_value,
                                                             ExpressionType low_expression_type, const Value &high_value,
                                                             ExpressionType high_expression_type) {
+	// initialize range lookup
 	auto result = make_unique<ARTIndexScanState>();
 	result->values[0] = low_value;
 	result->expressions[0] = low_expression_type;
@@ -103,7 +108,7 @@ static void TemplatedGenerateKeys(ArenaAllocator &allocator, Vector &input, idx_
 	for (idx_t i = 0; i < count; i++) {
 		auto idx = idata.sel->get_index(i);
 		if (idata.validity.RowIsValid(idx)) {
-			Key::CreateKey<T>(allocator, keys[i], input_data[idx]);
+			Key::CreateKey<T>(allocator, input.GetType(), keys[i], input_data[idx]);
 		}
 	}
 }
@@ -123,7 +128,7 @@ static void ConcatenateKeys(ArenaAllocator &allocator, Vector &input, idx_t coun
 				// this column entry is NULL, set whole key to NULL
 				keys[i] = Key();
 			} else {
-				auto other_key = Key::CreateKey<T>(allocator, input_data[idx]);
+				auto other_key = Key::CreateKey<T>(allocator, input.GetType(), input_data[idx]);
 				keys[i].ConcatenateKey(allocator, other_key);
 			}
 		}
@@ -225,7 +230,7 @@ void ART::GenerateKeys(ArenaAllocator &allocator, DataChunk &input, vector<Key>
 }
 
 //===--------------------------------------------------------------------===//
-// Construct from sorted data
+// Construct from sorted data (only during CREATE (UNIQUE) INDEX statements)
 //===--------------------------------------------------------------------===//
 
 struct KeySection {
@@ -283,7 +288,7 @@ bool Construct(ART &art, vector<Key> &keys, row_t *row_ids, Node *&node, KeySect
 		} else {
 			node = Leaf::New(start_key, prefix_start, row_ids + key_section.start, num_row_ids);
 		}
-		art.memory_size += node->MemorySize(art, false);
+		art.IncreaseMemorySize(node->MemorySize(art, false));
 		return true;
 	}
 	// create a new node and recurse
@@ -297,7 +302,7 @@ bool Construct(ART &art, vector<Key> &keys, row_t *row_ids, Node *&node, KeySect
 
 	auto prefix_length = key_section.depth - prefix_start;
 	node->prefix = Prefix(start_key, prefix_start, prefix_length);
-	art.memory_size += node->MemorySize(art, false);
+	art.IncreaseMemorySize(node->MemorySize(art, false));
 
 	// recurse on each child section
 	for (auto &child_section : child_sections) {
@@ -323,7 +328,7 @@ bool ART::ConstructFromSorted(idx_t count, vector<Key> &keys, Vector &row_identi
 }
 
 //===--------------------------------------------------------------------===//
-// Insert
+// Insert / Verification / Constraint Checking
 //===--------------------------------------------------------------------===//
 
 bool ART::Insert(IndexLock &lock, DataChunk &input, Vector &row_ids) {
@@ -331,13 +336,13 @@ bool ART::Insert(IndexLock &lock, DataChunk &input, Vector &row_ids) {
 	D_ASSERT(row_ids.GetType().InternalType() == ROW_TYPE);
 	D_ASSERT(logical_types[0] == input.data[0].GetType());
 
+	auto old_memory_size = memory_size;
+
 	// generate the keys for the given input
 	ArenaAllocator arena_allocator(BufferAllocator::Get(db));
 	vector<Key> keys(input.size());
 	GenerateKeys(arena_allocator, input, keys);
 
-	auto old_memory_size = this->memory_size;
-
 	// get the corresponding row IDs
 	row_ids.Flatten(input.size());
 	auto row_identifiers = FlatVector::GetData<row_t>(row_ids);
@@ -356,9 +361,9 @@ bool ART::Insert(IndexLock &lock, DataChunk &input, Vector &row_ids) {
 			break;
 		}
 	}
-	if (failed_index != DConstants::INVALID_INDEX) {
 
-		// failed to insert because of constraint violation: remove previously inserted entries
+	// failed to insert because of constraint violation: remove previously inserted entries
+	if (failed_index != DConstants::INVALID_INDEX) {
 		for (idx_t i = 0; i < failed_index; i++) {
 			if (keys[i].Empty()) {
 				continue;
@@ -366,14 +371,11 @@ bool ART::Insert(IndexLock &lock, DataChunk &input, Vector &row_ids) {
 			row_t row_id = row_identifiers[i];
 			Erase(tree, keys[i], 0, row_id);
 		}
-		// nothing changed, no need to update the buffer memory size
-		return false;
 	}
 
-	D_ASSERT(old_memory_size <= memory_size);
-	Verify();
-	if (track_memory) {
-		buffer_manager.IncreaseUsedMemory(memory_size - old_memory_size);
+	IncreaseAndVerifyMemorySize(old_memory_size);
+	if (failed_index != DConstants::INVALID_INDEX) {
+		return false;
 	}
 	return true;
 }
@@ -391,25 +393,12 @@ bool ART::Append(IndexLock &lock, DataChunk &appended_data, Vector &row_identifi
 
 void ART::VerifyAppend(DataChunk &chunk) {
 	ConflictManager conflict_manager(VerifyExistenceType::APPEND, chunk.size());
-	LookupValues(chunk, conflict_manager);
+	CheckConstraintsForChunk(chunk, conflict_manager);
 }
 
 void ART::VerifyAppend(DataChunk &chunk, ConflictManager &conflict_manager) {
 	D_ASSERT(conflict_manager.LookupType() == VerifyExistenceType::APPEND);
-	LookupValues(chunk, conflict_manager);
-}
-
-void ART::VerifyAppendForeignKey(DataChunk &chunk) {
-	ConflictManager conflict_manager(VerifyExistenceType::APPEND_FK, chunk.size());
-	LookupValues(chunk, conflict_manager);
-}
-
-void ART::VerifyDeleteForeignKey(DataChunk &chunk) {
-	if (!IsUnique()) {
-		return;
-	}
-	ConflictManager conflict_manager(VerifyExistenceType::DELETE_FK, chunk.size());
-	LookupValues(chunk, conflict_manager);
+	CheckConstraintsForChunk(chunk, conflict_manager);
 }
 
 bool ART::InsertToLeaf(Leaf &leaf, row_t row_id) {
@@ -430,7 +419,7 @@ bool ART::Insert(Node *&node, Key &key, idx_t depth, row_t row_id) {
 	if (!node) {
 		// node is currently empty, create a leaf here with the key
 		node = Leaf::New(key, depth, row_id);
-		this->memory_size += node->MemorySize(*this, false);
+		IncreaseMemorySize(node->MemorySize(*this, false));
 		return true;
 	}
 
@@ -455,14 +444,14 @@ bool ART::Insert(Node *&node, Key &key, idx_t depth, row_t row_id) {
 
 		Node *new_node = Node4::New();
 		new_node->prefix = Prefix(key, depth, new_prefix_length);
-		this->memory_size += new_node->MemorySize(*this, false);
+		IncreaseMemorySize(new_node->MemorySize(*this, false));
 
 		auto key_byte = node->prefix.Reduce(*this, new_prefix_length);
 		Node4::InsertChild(*this, new_node, key_byte, node);
 
 		Node *leaf_node = Leaf::New(key, depth + new_prefix_length + 1, row_id);
 		Node4::InsertChild(*this, new_node, key[depth + new_prefix_length], leaf_node);
-		this->memory_size += leaf_node->MemorySize(*this, false);
+		IncreaseMemorySize(leaf_node->MemorySize(*this, false));
 
 		node = new_node;
 		return true;
@@ -476,7 +465,7 @@ bool ART::Insert(Node *&node, Key &key, idx_t depth, row_t row_id) {
 			// prefix differs, create new node
 			Node *new_node = Node4::New();
 			new_node->prefix = Prefix(key, depth, mismatch_pos);
-			this->memory_size += new_node->MemorySize(*this, false);
+			IncreaseMemorySize(new_node->MemorySize(*this, false));
 
 			// break up prefix
 			auto key_byte = node->prefix.Reduce(*this, mismatch_pos);
@@ -484,7 +473,7 @@ bool ART::Insert(Node *&node, Key &key, idx_t depth, row_t row_id) {
 
 			Node *leaf_node = Leaf::New(key, depth + mismatch_pos + 1, row_id);
 			Node4::InsertChild(*this, new_node, key[depth + mismatch_pos], leaf_node);
-			this->memory_size += leaf_node->MemorySize(*this, false);
+			IncreaseMemorySize(leaf_node->MemorySize(*this, false));
 
 			node = new_node;
 			return true;
@@ -504,7 +493,7 @@ bool ART::Insert(Node *&node, Key &key, idx_t depth, row_t row_id) {
 
 	Node *leaf_node = Leaf::New(key, depth + 1, row_id);
 	Node::InsertChild(*this, node, key[depth], leaf_node);
-	this->memory_size += leaf_node->MemorySize(*this, false);
+	IncreaseMemorySize(leaf_node->MemorySize(*this, false));
 	return true;
 }
 
@@ -525,7 +514,7 @@ void ART::Delete(IndexLock &state, DataChunk &input, Vector &row_ids) {
 	vector<Key> keys(expression.size());
 	GenerateKeys(arena_allocator, expression, keys);
 
-	auto old_memory_size = this->memory_size;
+	auto old_memory_size = memory_size;
 
 	// now erase the elements from the database
 	row_ids.Flatten(input.size());
@@ -547,10 +536,13 @@ void ART::Delete(IndexLock &state, DataChunk &input, Vector &row_ids) {
 #endif
 	}
 
-	D_ASSERT(old_memory_size >= memory_size);
+	// if we deserialize nodes while erasing, then we might end up with more
+	// memory afterwards, so we have to either increase or decrease the used memory
 	Verify();
-	if (track_memory) {
+	if (track_memory && old_memory_size >= memory_size) {
 		buffer_manager.DecreaseUsedMemory(old_memory_size - memory_size);
+	} else if (track_memory) {
+		buffer_manager.IncreaseUsedMemory(memory_size - old_memory_size);
 	}
 }
 
@@ -566,8 +558,7 @@ void ART::Erase(Node *&node, Key &key, idx_t depth, row_t row_id) {
 		leaf->Remove(*this, row_id);
 
 		if (leaf->count == 0) {
-			D_ASSERT(this->memory_size >= leaf->MemorySize(*this, false));
-			this->memory_size -= leaf->MemorySize(*this, false);
+			DecreaseMemorySize(leaf->MemorySize(*this, false));
 			Node::Delete(node);
 			node = nullptr;
 		}
@@ -606,38 +597,38 @@ void ART::Erase(Node *&node, Key &key, idx_t depth, row_t row_id) {
 }
 
 //===--------------------------------------------------------------------===//
-// Point Query
+// Point Query (Equal)
 //===--------------------------------------------------------------------===//
 
 static Key CreateKey(ArenaAllocator &allocator, PhysicalType type, Value &value) {
 	D_ASSERT(type == value.type().InternalType());
 	switch (type) {
 	case PhysicalType::BOOL:
-		return Key::CreateKey<bool>(allocator, value);
+		return Key::CreateKey<bool>(allocator, value.type(), value);
 	case PhysicalType::INT8:
-		return Key::CreateKey<int8_t>(allocator, value);
+		return Key::CreateKey<int8_t>(allocator, value.type(), value);
 	case PhysicalType::INT16:
-		return Key::CreateKey<int16_t>(allocator, value);
+		return Key::CreateKey<int16_t>(allocator, value.type(), value);
 	case PhysicalType::INT32:
-		return Key::CreateKey<int32_t>(allocator, value);
+		return Key::CreateKey<int32_t>(allocator, value.type(), value);
 	case PhysicalType::INT64:
-		return Key::CreateKey<int64_t>(allocator, value);
+		return Key::CreateKey<int64_t>(allocator, value.type(), value);
 	case PhysicalType::UINT8:
-		return Key::CreateKey<uint8_t>(allocator, value);
+		return Key::CreateKey<uint8_t>(allocator, value.type(), value);
 	case PhysicalType::UINT16:
-		return Key::CreateKey<uint16_t>(allocator, value);
+		return Key::CreateKey<uint16_t>(allocator, value.type(), value);
 	case PhysicalType::UINT32:
-		return Key::CreateKey<uint32_t>(allocator, value);
+		return Key::CreateKey<uint32_t>(allocator, value.type(), value);
 	case PhysicalType::UINT64:
-		return Key::CreateKey<uint64_t>(allocator, value);
+		return Key::CreateKey<uint64_t>(allocator, value.type(), value);
 	case PhysicalType::INT128:
-		return Key::CreateKey<hugeint_t>(allocator, value);
+		return Key::CreateKey<hugeint_t>(allocator, value.type(), value);
 	case PhysicalType::FLOAT:
-		return Key::CreateKey<float>(allocator, value);
+		return Key::CreateKey<float>(allocator, value.type(), value);
 	case PhysicalType::DOUBLE:
-		return Key::CreateKey<double>(allocator, value);
+		return Key::CreateKey<double>(allocator, value.type(), value);
 	case PhysicalType::VARCHAR:
-		return Key::CreateKey<string_t>(allocator, value);
+		return Key::CreateKey<string_t>(allocator, value.type(), value);
 	default:
 		throw InternalException("Invalid type for index");
 	}
@@ -645,7 +636,10 @@ static Key CreateKey(ArenaAllocator &allocator, PhysicalType type, Value &value)
 
 bool ART::SearchEqual(Key &key, idx_t max_count, vector<row_t> &result_ids) {
 
+	auto old_memory_size = memory_size;
 	auto leaf = (Leaf *)(Lookup(tree, key, 0));
+	IncreaseAndVerifyMemorySize(old_memory_size);
+
 	if (!leaf) {
 		return true;
 	}
@@ -662,19 +656,28 @@ bool ART::SearchEqual(Key &key, idx_t max_count, vector<row_t> &result_ids) {
 void ART::SearchEqualJoinNoFetch(Key &key, idx_t &result_size) {
 
 	// we need to look for a leaf
+	auto old_memory_size = memory_size;
 	auto leaf = Lookup(tree, key, 0);
+	IncreaseAndVerifyMemorySize(old_memory_size);
+
 	if (!leaf) {
 		return;
 	}
 	result_size = leaf->count;
 }
 
+//===--------------------------------------------------------------------===//
+// Lookup
+//===--------------------------------------------------------------------===//
+
 Leaf *ART::Lookup(Node *node, Key &key, idx_t depth) {
+
 	while (node) {
 		if (node->type == NodeType::NLeaf) {
 			auto leaf = (Leaf *)node;
 			auto &leaf_prefix = leaf->prefix;
-			//! Check leaf
+
+			// check if leaf contains key
 			for (idx_t i = 0; i < leaf->prefix.Size(); i++) {
 				if (leaf_prefix[i] != key[i + depth]) {
 					return nullptr;
@@ -682,22 +685,29 @@ Leaf *ART::Lookup(Node *node, Key &key, idx_t depth) {
 			}
 			return (Leaf *)node;
 		}
+
 		if (node->prefix.Size()) {
 			for (idx_t pos = 0; pos < node->prefix.Size(); pos++) {
 				if (key[depth + pos] != node->prefix[pos]) {
+					// prefix mismatch, does not contain key
 					return nullptr;
 				}
 			}
 			depth += node->prefix.Size();
 		}
+
+		// prefix matches key, but no child at byte, does not contain key
 		idx_t pos = node->GetChildPos(key[depth]);
 		if (pos == DConstants::INVALID_INDEX) {
 			return nullptr;
 		}
+
+		// recurse into child
 		node = node->GetChild(*this, pos);
 		D_ASSERT(node);
 		depth++;
 	}
+
 	return nullptr;
 }
 
@@ -710,6 +720,7 @@ Leaf *ART::Lookup(Node *node, Key &key, idx_t depth) {
 bool ART::SearchGreater(ARTIndexScanState *state, Key &key, bool inclusive, idx_t max_count,
                         vector<row_t> &result_ids) {
 
+	auto old_memory_size = memory_size;
 	Iterator *it = &state->iterator;
 
 	// greater than scan: first set the iterator to the node at which we will start our scan by finding the lowest node
@@ -718,13 +729,16 @@ bool ART::SearchGreater(ARTIndexScanState *state, Key &key, bool inclusive, idx_
 		it->art = this;
 		bool found = it->LowerBound(tree, key, inclusive);
 		if (!found) {
+			IncreaseAndVerifyMemorySize(old_memory_size);
 			return true;
 		}
 	}
 	// after that we continue the scan; we don't need to check the bounds as any value following this value is
 	// automatically bigger and hence satisfies our predicate
 	Key empty_key = Key();
-	return it->Scan(empty_key, max_count, result_ids, false);
+	auto success = it->Scan(empty_key, max_count, result_ids, false);
+	IncreaseAndVerifyMemorySize(old_memory_size);
+	return success;
 }
 
 //===--------------------------------------------------------------------===//
@@ -738,6 +752,7 @@ bool ART::SearchLess(ARTIndexScanState *state, Key &upper_bound, bool inclusive,
 		return true;
 	}
 
+	auto old_memory_size = memory_size;
 	Iterator *it = &state->iterator;
 
 	if (!it->art) {
@@ -746,11 +761,14 @@ bool ART::SearchLess(ARTIndexScanState *state, Key &upper_bound, bool inclusive,
 		it->FindMinimum(*tree);
 		// early out min value higher than upper bound query
 		if (it->cur_key > upper_bound) {
+			IncreaseAndVerifyMemorySize(old_memory_size);
 			return true;
 		}
 	}
 	// now continue the scan until we reach the upper bound
-	return it->Scan(upper_bound, max_count, result_ids, inclusive);
+	auto success = it->Scan(upper_bound, max_count, result_ids, inclusive);
+	IncreaseAndVerifyMemorySize(old_memory_size);
+	return success;
 }
 
 //===--------------------------------------------------------------------===//
@@ -760,6 +778,7 @@ bool ART::SearchLess(ARTIndexScanState *state, Key &upper_bound, bool inclusive,
 bool ART::SearchCloseRange(ARTIndexScanState *state, Key &lower_bound, Key &upper_bound, bool left_inclusive,
                            bool right_inclusive, idx_t max_count, vector<row_t> &result_ids) {
 
+	auto old_memory_size = memory_size;
 	Iterator *it = &state->iterator;
 
 	// first find the first node that satisfies the left predicate
@@ -767,11 +786,14 @@ bool ART::SearchCloseRange(ARTIndexScanState *state, Key &lower_bound, Key &uppe
 		it->art = this;
 		bool found = it->LowerBound(tree, lower_bound, left_inclusive);
 		if (!found) {
+			IncreaseAndVerifyMemorySize(old_memory_size);
 			return true;
 		}
 	}
 	// now continue the scan until we reach the upper bound
-	return it->Scan(upper_bound, max_count, result_ids, right_inclusive);
+	auto success = it->Scan(upper_bound, max_count, result_ids, right_inclusive);
+	IncreaseAndVerifyMemorySize(old_memory_size);
+	return success;
 }
 
 bool ART::Scan(Transaction &transaction, DataTable &table, IndexScanState &table_state, idx_t max_count,
@@ -844,8 +866,15 @@ bool ART::Scan(Transaction &transaction, DataTable &table, IndexScanState &table
 	return true;
 }
 
+//===--------------------------------------------------------------------===//
+// More Verification / Constraint Checking
+//===--------------------------------------------------------------------===//
+
 string ART::GenerateErrorKeyName(DataChunk &input, idx_t row) {
-	// re-executing the expressions is not very fast, but we're going to throw anyways, so we don't care
+
+	// FIXME: why exactly can we not pass the expression_chunk as an argument to this
+	// FIXME: function instead of re-executing?
+	// re-executing the expressions is not very fast, but we're going to throw, so we don't care
 	DataChunk expression_chunk;
 	expression_chunk.Initialize(Allocator::DefaultAllocator(), logical_types);
 	ExecuteExpressions(input, expression_chunk);
@@ -883,11 +912,13 @@ string ART::GenerateConstraintErrorMessage(VerifyExistenceType verify_type, cons
 	}
 }
 
-void ART::LookupValues(DataChunk &input, ConflictManager &conflict_manager) {
+void ART::CheckConstraintsForChunk(DataChunk &input, ConflictManager &conflict_manager) {
 
 	// don't alter the index during constraint checking
 	lock_guard<mutex> l(lock);
 
+	auto old_memory_size = memory_size;
+
 	// first resolve the expressions for the index
 	DataChunk expression_chunk;
 	expression_chunk.Initialize(Allocator::DefaultAllocator(), logical_types);
@@ -900,12 +931,14 @@ void ART::LookupValues(DataChunk &input, ConflictManager &conflict_manager) {
 
 	idx_t found_conflict = DConstants::INVALID_INDEX;
 	for (idx_t i = 0; found_conflict == DConstants::INVALID_INDEX && i < input.size(); i++) {
+
 		if (keys[i].Empty()) {
 			if (conflict_manager.AddNull(i)) {
 				found_conflict = i;
 			}
 			continue;
 		}
+
 		Leaf *leaf_ptr = Lookup(tree, keys[i], 0);
 		if (leaf_ptr == nullptr) {
 			if (conflict_manager.AddMiss(i)) {
@@ -913,6 +946,7 @@ void ART::LookupValues(DataChunk &input, ConflictManager &conflict_manager) {
 			}
 			continue;
 		}
+
 		// When we find a node, we need to update the 'matches' and 'row_ids'
 		// NOTE: Leafs can have more than one row_id, but for UNIQUE/PRIMARY KEY they will only have one
 		D_ASSERT(leaf_ptr->count == 1);
@@ -921,11 +955,15 @@ void ART::LookupValues(DataChunk &input, ConflictManager &conflict_manager) {
 			found_conflict = i;
 		}
 	}
+
 	conflict_manager.FinishLookup();
+	IncreaseAndVerifyMemorySize(old_memory_size);
+
 	if (found_conflict == DConstants::INVALID_INDEX) {
 		// No conflicts detected
 		return;
 	}
+
 	auto key_name = GenerateErrorKeyName(input, found_conflict);
 	auto exception_msg = GenerateConstraintErrorMessage(conflict_manager.LookupType(), key_name);
 	throw ConstraintException(exception_msg);
@@ -935,13 +973,15 @@ void ART::LookupValues(DataChunk &input, ConflictManager &conflict_manager) {
 // Serialization
 //===--------------------------------------------------------------------===//
 
-BlockPointer ART::Serialize(duckdb::MetaBlockWriter &writer) {
+BlockPointer ART::Serialize(MetaBlockWriter &writer) {
 	lock_guard<mutex> l(lock);
+	auto old_memory_size = memory_size;
 	if (tree) {
 		serialized_data_pointer = tree->Serialize(*this, writer);
 	} else {
 		serialized_data_pointer = {(block_id_t)DConstants::INVALID_INDEX, (uint32_t)DConstants::INVALID_INDEX};
 	}
+	IncreaseAndVerifyMemorySize(old_memory_size);
 	return serialized_data_pointer;
 }
 
@@ -954,8 +994,8 @@ bool ART::MergeIndexes(IndexLock &state, Index *other_index) {
 	auto other_art = (ART *)other_index;
 
 	if (!this->tree) {
-		this->memory_size += other_art->memory_size;
-		this->tree = other_art->tree;
+		IncreaseMemorySize(other_art->memory_size);
+		tree = other_art->tree;
 		other_art->tree = nullptr;
 		return true;
 	}
@@ -987,4 +1027,14 @@ void ART::Verify() {
 #endif
 }
 
+void ART::IncreaseAndVerifyMemorySize(idx_t old_memory_size) {
+	// since we lazily deserialize ART nodes, it is possible that its in-memory size
+	// increased during lookups
+	Verify();
+	D_ASSERT(memory_size >= old_memory_size);
+	if (track_memory) {
+		buffer_manager.IncreaseUsedMemory(memory_size - old_memory_size);
+	}
+}
+
 } // namespace duckdb

package/src/duckdb/src/execution/index/art/art_key.cpp

@@ -15,40 +15,52 @@ Key::Key(ArenaAllocator &allocator, idx_t len) : len(len) {
 }
 
 template <>
-Key Key::CreateKey(ArenaAllocator &allocator, string_t value) {
+Key Key::CreateKey(ArenaAllocator &allocator, const LogicalType &type, string_t value) {
 	idx_t len = value.GetSize() + 1;
 	auto data = allocator.Allocate(len);
 	memcpy(data, value.GetDataUnsafe(), len - 1);
 
-	if (len > 1 && data[len - 2] == '\0') {
-		// FIXME: rethink this
-		throw NotImplementedException("Indexes cannot have contain null-terminated decoded BLOBs.");
+	// FIXME: rethink this
+	if (type == LogicalType::BLOB || type == LogicalType::VARCHAR) {
+		// indexes cannot contain BLOBs (or BLOBs cast to VARCHARs) that contain null-terminated bytes
+		for (idx_t i = 0; i < len - 1; i++) {
+			if (data[i] == '\0') {
+				throw NotImplementedException("Indexes cannot contain BLOBs that contain null-terminated bytes.");
+			}
+		}
 	}
+
 	data[len - 1] = '\0';
 	return Key(data, len);
 }
 
 template <>
-Key Key::CreateKey(ArenaAllocator &allocator, const char *value) {
-	return Key::CreateKey(allocator, string_t(value, strlen(value)));
+Key Key::CreateKey(ArenaAllocator &allocator, const LogicalType &type, const char *value) {
+	return Key::CreateKey(allocator, type, string_t(value, strlen(value)));
 }
 
 template <>
-void Key::CreateKey(ArenaAllocator &allocator, Key &key, string_t value) {
+void Key::CreateKey(ArenaAllocator &allocator, const LogicalType &type, Key &key, string_t value) {
 	key.len = value.GetSize() + 1;
 	key.data = allocator.Allocate(key.len);
 	memcpy(key.data, value.GetDataUnsafe(), key.len - 1);
 
-	if (key.len > 1 && key.data[key.len - 2] == '\0') {
-		// FIXME: rethink this
-		throw NotImplementedException("Indexes cannot have contain null-terminated decoded BLOBs.");
+	// FIXME: rethink this
+	if (type == LogicalType::BLOB || type == LogicalType::VARCHAR) {
+		// indexes cannot contain BLOBs (or BLOBs cast to VARCHARs) that contain null-terminated bytes
+		for (idx_t i = 0; i < key.len - 1; i++) {
+			if (key.data[i] == '\0') {
+				throw NotImplementedException("Indexes cannot contain BLOBs that contain null-terminated bytes.");
+			}
+		}
 	}
+
 	key.data[key.len - 1] = '\0';
 }
 
 template <>
-void Key::CreateKey(ArenaAllocator &allocator, Key &key, const char *value) {
-	Key::CreateKey(allocator, key, string_t(value, strlen(value)));
+void Key::CreateKey(ArenaAllocator &allocator, const LogicalType &type, Key &key, const char *value) {
+	Key::CreateKey(allocator, type, key, string_t(value, strlen(value)));
 }
 
 bool Key::operator>(const Key &k) const {

package/src/duckdb/src/execution/index/art/leaf.cpp

@@ -109,9 +109,9 @@ void Leaf::Insert(ART &art, row_t row_id) {
 	if (count == capacity) {
 		// grow array
 		if (IsInlined()) {
-			art.memory_size += (capacity + 1) * sizeof(row_t);
+			art.IncreaseMemorySize((capacity + 1) * sizeof(row_t));
 		} else {
-			art.memory_size += capacity * sizeof(row_t);
+			art.IncreaseMemorySize(capacity * sizeof(row_t));
 		}
 		row_ids = Resize(row_ids, count, capacity * 2);
 	}
@@ -143,6 +143,7 @@ void Leaf::Remove(ART &art, row_t row_id) {
 		return;
 	}
 
+	auto capacity = GetCapacity();
 	count--;
 	if (count == 1) {
 		// after erasing we can now inline the leaf
@@ -150,18 +151,16 @@ void Leaf::Remove(ART &art, row_t row_id) {
 		auto remaining_row_id = row_ids[0] == row_id ? row_ids[1] : row_ids[0];
 		DeleteArray<row_t>(rowids.ptr, rowids.ptr[0] + 1);
 		rowids.inlined = remaining_row_id;
-		D_ASSERT(art.memory_size >= sizeof(row_t));
-		art.memory_size -= 2 * sizeof(row_t);
+		art.DecreaseMemorySize(capacity * sizeof(row_t));
 		return;
 	}
 
 	// shrink array, if less than half full
-	auto capacity = GetCapacity();
+	capacity = GetCapacity();
 	if (capacity > 2 && count < capacity / 2) {
 
 		auto new_capacity = capacity / 2;
-		D_ASSERT(art.memory_size >= (capacity - new_capacity) * sizeof(row_t));
-		art.memory_size -= (capacity - new_capacity) * sizeof(row_t);
+		art.DecreaseMemorySize((capacity - new_capacity) * sizeof(row_t));
 
 		auto new_allocation = AllocateArray<row_t>(new_capacity + 1);
 		new_allocation[0] = new_capacity;
@@ -200,7 +199,7 @@ void Leaf::Merge(ART &art, Node *&l_node, Node *&r_node) {
 	if (l_n->count + r_n->count > l_capacity) {
 		auto capacity = l_n->GetCapacity();
 		auto new_capacity = NextPowerOfTwo(l_n->count + r_n->count);
-		art.memory_size += sizeof(row_t) * (new_capacity - capacity);
+		art.IncreaseMemorySize(sizeof(row_t) * (new_capacity - capacity));
 		l_row_ids = l_n->Resize(l_row_ids, l_n->count, new_capacity);
 	}