duckdb 0.7.1-dev240.0 → 0.7.1-dev284.0

package/src/duckdb/src/execution/index/art/art_key.cpp

@@ -15,40 +15,52 @@ Key::Key(ArenaAllocator &allocator, idx_t len) : len(len) {
 }
 
 template <>
-Key Key::CreateKey(ArenaAllocator &allocator, string_t value) {
+Key Key::CreateKey(ArenaAllocator &allocator, const LogicalType &type, string_t value) {
 	idx_t len = value.GetSize() + 1;
 	auto data = allocator.Allocate(len);
 	memcpy(data, value.GetDataUnsafe(), len - 1);
 
-	if (len > 1 && data[len - 2] == '\0') {
-		// FIXME: rethink this
-		throw NotImplementedException("Indexes cannot have contain null-terminated decoded BLOBs.");
+	// FIXME: rethink this
+	if (type == LogicalType::BLOB || type == LogicalType::VARCHAR) {
+		// indexes cannot contain BLOBs (or BLOBs cast to VARCHARs) that contain null-terminated bytes
+		for (idx_t i = 0; i < len - 1; i++) {
+			if (data[i] == '\0') {
+				throw NotImplementedException("Indexes cannot contain BLOBs that contain null-terminated bytes.");
+			}
+		}
 	}
+
 	data[len - 1] = '\0';
 	return Key(data, len);
 }
 
 template <>
-Key Key::CreateKey(ArenaAllocator &allocator, const char *value) {
-	return Key::CreateKey(allocator, string_t(value, strlen(value)));
+Key Key::CreateKey(ArenaAllocator &allocator, const LogicalType &type, const char *value) {
+	return Key::CreateKey(allocator, type, string_t(value, strlen(value)));
 }
 
 template <>
-void Key::CreateKey(ArenaAllocator &allocator, Key &key, string_t value) {
+void Key::CreateKey(ArenaAllocator &allocator, const LogicalType &type, Key &key, string_t value) {
 	key.len = value.GetSize() + 1;
 	key.data = allocator.Allocate(key.len);
 	memcpy(key.data, value.GetDataUnsafe(), key.len - 1);
 
-	if (key.len > 1 && key.data[key.len - 2] == '\0') {
-		// FIXME: rethink this
-		throw NotImplementedException("Indexes cannot have contain null-terminated decoded BLOBs.");
+	// FIXME: rethink this
+	if (type == LogicalType::BLOB || type == LogicalType::VARCHAR) {
+		// indexes cannot contain BLOBs (or BLOBs cast to VARCHARs) that contain null-terminated bytes
+		for (idx_t i = 0; i < key.len - 1; i++) {
+			if (key.data[i] == '\0') {
+				throw NotImplementedException("Indexes cannot contain BLOBs that contain null-terminated bytes.");
+			}
+		}
 	}
+
 	key.data[key.len - 1] = '\0';
 }
 
 template <>
-void Key::CreateKey(ArenaAllocator &allocator, Key &key, const char *value) {
-	Key::CreateKey(allocator, key, string_t(value, strlen(value)));
+void Key::CreateKey(ArenaAllocator &allocator, const LogicalType &type, Key &key, const char *value) {
+	Key::CreateKey(allocator, type, key, string_t(value, strlen(value)));
 }
 
 bool Key::operator>(const Key &k) const {

package/src/duckdb/src/execution/index/art/leaf.cpp

@@ -109,9 +109,9 @@ void Leaf::Insert(ART &art, row_t row_id) {
 	if (count == capacity) {
 		// grow array
 		if (IsInlined()) {
-			art.memory_size += (capacity + 1) * sizeof(row_t);
+			art.IncreaseMemorySize((capacity + 1) * sizeof(row_t));
 		} else {
-			art.memory_size += capacity * sizeof(row_t);
+			art.IncreaseMemorySize(capacity * sizeof(row_t));
 		}
 		row_ids = Resize(row_ids, count, capacity * 2);
 	}
@@ -143,6 +143,7 @@ void Leaf::Remove(ART &art, row_t row_id) {
 		return;
 	}
 
+	auto capacity = GetCapacity();
 	count--;
 	if (count == 1) {
 		// after erasing we can now inline the leaf
@@ -150,18 +151,16 @@ void Leaf::Remove(ART &art, row_t row_id) {
 		auto remaining_row_id = row_ids[0] == row_id ? row_ids[1] : row_ids[0];
 		DeleteArray<row_t>(rowids.ptr, rowids.ptr[0] + 1);
 		rowids.inlined = remaining_row_id;
-		D_ASSERT(art.memory_size >= sizeof(row_t));
-		art.memory_size -= 2 * sizeof(row_t);
+		art.DecreaseMemorySize(capacity * sizeof(row_t));
 		return;
 	}
 
 	// shrink array, if less than half full
-	auto capacity = GetCapacity();
+	capacity = GetCapacity();
 	if (capacity > 2 && count < capacity / 2) {
 
 		auto new_capacity = capacity / 2;
-		D_ASSERT(art.memory_size >= (capacity - new_capacity) * sizeof(row_t));
-		art.memory_size -= (capacity - new_capacity) * sizeof(row_t);
+		art.DecreaseMemorySize((capacity - new_capacity) * sizeof(row_t));
 
 		auto new_allocation = AllocateArray<row_t>(new_capacity + 1);
 		new_allocation[0] = new_capacity;
@@ -200,7 +199,7 @@ void Leaf::Merge(ART &art, Node *&l_node, Node *&r_node) {
 	if (l_n->count + r_n->count > l_capacity) {
 		auto capacity = l_n->GetCapacity();
 		auto new_capacity = NextPowerOfTwo(l_n->count + r_n->count);
-		art.memory_size += sizeof(row_t) * (new_capacity - capacity);
+		art.IncreaseMemorySize(sizeof(row_t) * (new_capacity - capacity));
 		l_row_ids = l_n->Resize(l_row_ids, l_n->count, new_capacity);
 	}
 

package/src/duckdb/src/execution/index/art/node.cpp

@@ -299,16 +299,11 @@ Node *Node::Deserialize(ART &art, idx_t block_id, idx_t offset) {
 	NodeType node_type((NodeType)(n));
 
 	Node *deserialized_node = nullptr;
-	auto old_memory_size = art.memory_size;
 	switch (node_type) {
 	case NodeType::NLeaf: {
 		auto leaf = Leaf::New();
 		leaf->Deserialize(art, reader);
-		art.memory_size += leaf->MemorySize(art, false);
-		D_ASSERT(art.memory_size >= old_memory_size);
-		if (art.track_memory) {
-			art.buffer_manager.IncreaseUsedMemory(art.memory_size - old_memory_size);
-		}
+		art.IncreaseMemorySize(leaf->MemorySize(art, false));
 		return leaf;
 	}
 	case NodeType::N4: {
@@ -331,11 +326,7 @@ Node *Node::Deserialize(ART &art, idx_t block_id, idx_t offset) {
 		throw InternalException("Unrecognized node type");
 	}
 	deserialized_node->DeserializeInternal(art, reader);
-	art.memory_size += deserialized_node->MemorySize(art, false);
-	D_ASSERT(art.memory_size >= old_memory_size);
-	if (art.track_memory) {
-		art.buffer_manager.IncreaseUsedMemory(art.memory_size - old_memory_size);
-	}
+	art.IncreaseMemorySize(deserialized_node->MemorySize(art, false));
 	return deserialized_node;
 }
 
@@ -356,12 +347,10 @@ void SwapNodes(MergeInfo &info, ParentsOfNodes &parents) {
 	auto l_node_memory_size = info.l_node->MemorySize(*info.l_art, true);
 	auto r_node_memory_size = info.r_node->MemorySize(*info.r_art, true);
 
-	D_ASSERT(info.root_l_art->memory_size >= l_node_memory_size);
-	D_ASSERT(info.root_r_art->memory_size >= r_node_memory_size);
-	info.root_l_art->memory_size -= l_node_memory_size;
-	info.root_r_art->memory_size -= r_node_memory_size;
-	info.root_l_art->memory_size += r_node_memory_size;
-	info.root_r_art->memory_size += l_node_memory_size;
+	info.root_l_art->DecreaseMemorySize(l_node_memory_size);
+	info.root_r_art->DecreaseMemorySize(r_node_memory_size);
+	info.root_l_art->IncreaseMemorySize(r_node_memory_size);
+	info.root_r_art->IncreaseMemorySize(l_node_memory_size);
 
 	// actual swap
 	swap(info.l_art, info.r_art);
@@ -409,9 +398,8 @@ bool Merge(MergeInfo &info, ParentsOfNodes &parents) {
 			auto r_memory_size = r_child->MemorySize(*info.r_art, true);
 			Node::InsertChild(*info.root_l_art, info.l_node, key_byte, r_child);
 
-			info.root_l_art->memory_size += r_memory_size;
-			D_ASSERT(info.root_r_art->memory_size >= r_memory_size);
-			info.root_r_art->memory_size -= r_memory_size;
+			info.root_l_art->IncreaseMemorySize(r_memory_size);
+			info.root_r_art->DecreaseMemorySize(r_memory_size);
 			if (parents.l_parent) {
 				parents.l_parent->ReplaceChildPointer(parents.l_pos, info.l_node);
 			}
@@ -473,9 +461,8 @@ bool ResolvePrefixesAndMerge(MergeInfo &info, ParentsOfNodes &parents) {
 			auto r_memory_size = r_node->MemorySize(*info.r_art, true);
 			Node::InsertChild(*info.root_l_art, l_node, mismatch_byte, r_node);
 
-			info.root_l_art->memory_size += r_memory_size;
-			D_ASSERT(info.root_r_art->memory_size >= r_memory_size);
-			info.root_r_art->memory_size -= r_memory_size;
+			info.root_l_art->IncreaseMemorySize(r_memory_size);
+			info.root_r_art->DecreaseMemorySize(r_memory_size);
 			UpdateParentsOfNodes(l_node, null_parent, parents);
 			r_node = nullptr;
 			return true;
@@ -493,7 +480,7 @@ bool ResolvePrefixesAndMerge(MergeInfo &info, ParentsOfNodes &parents) {
 	// create new node
 	Node *new_node = Node4::New();
 	new_node->prefix = Prefix(l_node->prefix, mismatch_pos);
-	info.root_l_art->memory_size += new_node->MemorySize(*info.l_art, false);
+	info.root_l_art->IncreaseMemorySize(new_node->MemorySize(*info.l_art, false));
 
 	// insert l_node, break up prefix of l_node
 	auto key_byte = l_node->prefix.Reduce(*info.root_l_art, mismatch_pos);
@@ -504,9 +491,8 @@ bool ResolvePrefixesAndMerge(MergeInfo &info, ParentsOfNodes &parents) {
 	auto r_memory_size = r_node->MemorySize(*info.r_art, true);
 	Node4::InsertChild(*info.root_l_art, new_node, key_byte, r_node);
 
-	info.root_l_art->memory_size += r_memory_size;
-	D_ASSERT(info.root_r_art->memory_size >= r_memory_size);
-	info.root_r_art->memory_size -= r_memory_size;
+	info.root_l_art->IncreaseMemorySize(r_memory_size);
+	info.root_r_art->DecreaseMemorySize(r_memory_size);
 
 	l_node = new_node;
 	UpdateParentsOfNodes(l_node, null_parent, parents);

package/src/duckdb/src/execution/index/art/node16.cpp

@@ -104,7 +104,7 @@ void Node16::InsertChild(ART &art, Node *&node, uint8_t key_byte, Node *new_chil
 	} else {
 		// node is full, grow to Node48
 		auto new_node = Node48::New();
-		art.memory_size += new_node->MemorySize(art, false);
+		art.IncreaseMemorySize(new_node->MemorySize(art, false));
 		new_node->count = node->count;
 		new_node->prefix = std::move(n->prefix);
 
@@ -114,8 +114,7 @@ void Node16::InsertChild(ART &art, Node *&node, uint8_t key_byte, Node *new_chil
 			n->children[i] = nullptr;
 		}
 
-		D_ASSERT(art.memory_size >= node->MemorySize(art, false));
-		art.memory_size -= node->MemorySize(art, false);
+		art.DecreaseMemorySize(node->MemorySize(art, false));
 		Node::Delete(node);
 		node = new_node;
 		Node48::InsertChild(art, node, key_byte, new_child);
@@ -130,8 +129,7 @@ void Node16::EraseChild(ART &art, Node *&node, idx_t pos) {
 	// adjust the ART size
 	if (n->ChildIsInMemory(pos)) {
 		auto child = n->GetChild(art, pos);
-		D_ASSERT(art.memory_size >= child->MemorySize(art, true));
-		art.memory_size -= child->MemorySize(art, true);
+		art.DecreaseMemorySize(child->MemorySize(art, true));
 	}
 
 	// erase the child and decrease the count
@@ -155,7 +153,7 @@ void Node16::EraseChild(ART &art, Node *&node, idx_t pos) {
 	if (node->count < Node4::GetSize()) {
 
 		auto new_node = Node4::New();
-		art.memory_size += new_node->MemorySize(art, false);
+		art.IncreaseMemorySize(new_node->MemorySize(art, false));
 		new_node->prefix = std::move(n->prefix);
 
 		for (idx_t i = 0; i < n->count; i++) {
@@ -164,8 +162,7 @@ void Node16::EraseChild(ART &art, Node *&node, idx_t pos) {
 			n->children[i] = nullptr;
 		}
 
-		D_ASSERT(art.memory_size >= node->MemorySize(art, false));
-		art.memory_size -= node->MemorySize(art, false);
+		art.DecreaseMemorySize(node->MemorySize(art, false));
 		Node::Delete(node);
 		node = new_node;
 	}

package/src/duckdb/src/execution/index/art/node256.cpp

@@ -92,8 +92,7 @@ void Node256::EraseChild(ART &art, Node *&node, idx_t pos) {
 	// adjust the ART size
 	if (n->ChildIsInMemory(pos)) {
 		auto child = n->GetChild(art, pos);
-		D_ASSERT(art.memory_size >= child->MemorySize(art, true));
-		art.memory_size -= child->MemorySize(art, true);
+		art.DecreaseMemorySize(child->MemorySize(art, true));
 	}
 
 	// erase the child and decrease the count
@@ -104,7 +103,7 @@ void Node256::EraseChild(ART &art, Node *&node, idx_t pos) {
 	if (node->count <= NODE_256_SHRINK_THRESHOLD) {
 
 		auto new_node = Node48::New();
-		art.memory_size += new_node->MemorySize(art, false);
+		art.IncreaseMemorySize(new_node->MemorySize(art, false));
 		new_node->prefix = std::move(n->prefix);
 
 		for (idx_t i = 0; i < Node256::GetSize(); i++) {
@@ -115,8 +114,7 @@ void Node256::EraseChild(ART &art, Node *&node, idx_t pos) {
 			}
 		}
 
-		D_ASSERT(art.memory_size >= node->MemorySize(art, false));
-		art.memory_size -= node->MemorySize(art, false);
+		art.DecreaseMemorySize(node->MemorySize(art, false));
 		Node::Delete(node);
 		node = new_node;
 	}

package/src/duckdb/src/execution/index/art/node4.cpp

@@ -101,7 +101,7 @@ void Node4::InsertChild(ART &art, Node *&node, uint8_t key_byte, Node *new_child
 	} else {
 		// node is full, grow to Node16
 		auto new_node = Node16::New();
-		art.memory_size += new_node->MemorySize(art, false);
+		art.IncreaseMemorySize(new_node->MemorySize(art, false));
 		new_node->count = n->count;
 		new_node->prefix = std::move(node->prefix);
 
@@ -112,8 +112,7 @@ void Node4::InsertChild(ART &art, Node *&node, uint8_t key_byte, Node *new_child
 		}
 		n->count = 0;
 
-		D_ASSERT(art.memory_size >= node->MemorySize(art, false));
-		art.memory_size -= node->MemorySize(art, false);
+		art.DecreaseMemorySize(node->MemorySize(art, false));
 		Node::Delete(node);
 		node = new_node;
 		Node16::InsertChild(art, node, key_byte, new_child);
@@ -129,8 +128,7 @@ void Node4::EraseChild(ART &art, Node *&node, idx_t pos) {
 	// adjust the ART size
 	if (n->ChildIsInMemory(pos)) {
 		auto child = n->GetChild(art, pos);
-		D_ASSERT(art.memory_size >= child->MemorySize(art, true));
-		art.memory_size -= child->MemorySize(art, true);
+		art.DecreaseMemorySize(child->MemorySize(art, true));
 	}
 
 	// erase the child and decrease the count
@@ -158,8 +156,7 @@ void Node4::EraseChild(ART &art, Node *&node, idx_t pos) {
 		// ensure that when deleting the node, we do not delete the child (because we move it)
 		n->children[0] = nullptr;
 
-		D_ASSERT(art.memory_size >= n->MemorySize(art, false));
-		art.memory_size -= n->MemorySize(art, false);
+		art.DecreaseMemorySize(n->MemorySize(art, false));
 		Node::Delete(node);
 		node = child_ref;
 	}

package/src/duckdb/src/execution/index/art/node48.cpp

@@ -105,7 +105,7 @@ void Node48::InsertChild(ART &art, Node *&node, uint8_t key_byte, Node *new_chil
 	} else {
 		// node is full, grow to Node256
 		auto new_node = Node256::New();
-		art.memory_size += new_node->MemorySize(art, false);
+		art.IncreaseMemorySize(new_node->MemorySize(art, false));
 		new_node->count = n->count;
 		new_node->prefix = std::move(n->prefix);
 
@@ -116,8 +116,7 @@ void Node48::InsertChild(ART &art, Node *&node, uint8_t key_byte, Node *new_chil
 			}
 		}
 
-		D_ASSERT(art.memory_size >= node->MemorySize(art, false));
-		art.memory_size -= node->MemorySize(art, false);
+		art.DecreaseMemorySize(node->MemorySize(art, false));
 		Node::Delete(node);
 		node = new_node;
 		Node256::InsertChild(art, node, key_byte, new_child);
@@ -130,8 +129,7 @@ void Node48::EraseChild(ART &art, Node *&node, idx_t pos) {
 	// adjust the ART size
 	if (n->ChildIsInMemory(pos)) {
 		auto child = n->GetChild(art, pos);
-		D_ASSERT(art.memory_size >= child->MemorySize(art, true));
-		art.memory_size -= child->MemorySize(art, true);
+		art.DecreaseMemorySize(child->MemorySize(art, true));
 	}
 
 	// erase the child and decrease the count
@@ -143,7 +141,7 @@ void Node48::EraseChild(ART &art, Node *&node, idx_t pos) {
 	if (node->count < NODE_48_SHRINK_THRESHOLD) {
 
 		auto new_node = Node16::New();
-		art.memory_size += new_node->MemorySize(art, false);
+		art.IncreaseMemorySize(new_node->MemorySize(art, false));
 		new_node->prefix = std::move(n->prefix);
 
 		for (idx_t i = 0; i < Node256::GetSize(); i++) {
@@ -154,8 +152,7 @@ void Node48::EraseChild(ART &art, Node *&node, idx_t pos) {
 			}
 		}
 
-		D_ASSERT(art.memory_size >= node->MemorySize(art, false));
-		art.memory_size -= node->MemorySize(art, false);
+		art.DecreaseMemorySize(node->MemorySize(art, false));
 		Node::Delete(node);
 		node = new_node;
 	}

package/src/duckdb/src/execution/index/art/prefix.cpp

@@ -113,7 +113,7 @@ void Prefix::Overwrite(uint32_t new_size, uint8_t *data) {
 
 void Prefix::Concatenate(ART &art, uint8_t key, Prefix &other) {
 	auto new_size = size + 1 + other.size;
-	art.memory_size += (new_size - size) * sizeof(uint8_t);
+	art.IncreaseMemorySize((new_size - size) * sizeof(uint8_t));
 	// have to allocate space in our prefix array
 	auto new_prefix = AllocateArray<uint8_t>(new_size);
 	idx_t new_prefix_idx = 0;
@@ -136,8 +136,7 @@ void Prefix::Concatenate(ART &art, uint8_t key, Prefix &other) {
 
 uint8_t Prefix::Reduce(ART &art, uint32_t n) {
 	auto new_size = size - n - 1;
-	D_ASSERT(art.memory_size >= (size - new_size) * sizeof(uint8_t));
-	art.memory_size -= (size - new_size) * sizeof(uint8_t);
+	art.DecreaseMemorySize((size - new_size) * sizeof(uint8_t));
 	auto prefix = GetPrefixData();
 	auto partial_key = prefix[n];
 

package/src/duckdb/src/function/pragma/pragma_queries.cpp

@@ -22,8 +22,8 @@ string PragmaShowTablesExpanded(ClientContext &context, const FunctionParameters
 	return R"(
 			SELECT
 				t.table_name,
-				LIST(c.column_name order by c.column_name) AS column_names,
-				LIST(c.data_type order by c.column_name) AS column_types,
+				LIST(c.column_name order by c.column_index) AS column_names,
+				LIST(c.data_type order by c.column_index) AS column_types,
 				FIRST(t.temporary) AS temporary
 			FROM duckdb_tables t
 			JOIN duckdb_columns c

package/src/duckdb/src/function/table/read_csv.cpp

@@ -312,7 +312,7 @@ public:
 			progress = double(bytes_read) / double(file_size);
 		}
 		// now get the total percentage of files read
-		double percentage = double(file_index) / total_files;
+		double percentage = double(file_index - 1) / total_files;
 		percentage += (double(1) / double(total_files)) * progress;
 		return percentage * 100;
 	}
@@ -584,6 +584,7 @@ struct SingleThreadedCSVState : public GlobalTableFunctionState {
 		{
 			lock_guard<mutex> l(csv_lock);
 			if (initial_reader) {
+				total_size = initial_reader->file_handle ? initial_reader->file_handle->FileSize() : 0;
 				return std::move(initial_reader);
 			}
 			if (next_file >= total_files) {

package/src/duckdb/src/function/table/table_scan.cpp

@@ -287,9 +287,12 @@ void TableScanPushdownComplexFilter(ClientContext &context, LogicalGet &get, Fun
 	// behold
 	storage.info->indexes.Scan([&](Index &index) {
 		// first rewrite the index expression so the ColumnBindings align with the column bindings of the current table
+
 		if (index.unbound_expressions.size() > 1) {
+			// NOTE: index scans are not (yet) supported for compound index keys
 			return false;
 		}
+
 		auto index_expression = index.unbound_expressions[0]->Copy();
 		bool rewrite_possible = true;
 		RewriteIndexExpression(index, get, *index_expression, rewrite_possible);

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.7.1-dev240"
+#define DUCKDB_VERSION "0.7.1-dev284"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "661b09da10"
+#define DUCKDB_SOURCE_ID "6b7362d2ca"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/catalog/catalog.hpp

@@ -29,6 +29,7 @@ struct CreateFunctionInfo;
 struct CreateViewInfo;
 struct CreateSequenceInfo;
 struct CreateCollationInfo;
+struct CreateIndexInfo;
 struct CreateTypeInfo;
 struct CreateTableInfo;
 struct DatabaseSize;
@@ -137,6 +138,9 @@ public:
 	//! Creates a collation in the catalog
 	DUCKDB_API CatalogEntry *CreateCollation(CatalogTransaction transaction, CreateCollationInfo *info);
 	DUCKDB_API CatalogEntry *CreateCollation(ClientContext &context, CreateCollationInfo *info);
+	//! Creates an index in the catalog
+	DUCKDB_API CatalogEntry *CreateIndex(CatalogTransaction transaction, CreateIndexInfo *info);
+	DUCKDB_API CatalogEntry *CreateIndex(ClientContext &context, CreateIndexInfo *info);
 
 	//! Creates a table in the catalog.
 	DUCKDB_API CatalogEntry *CreateTable(CatalogTransaction transaction, SchemaCatalogEntry *schema,
@@ -153,7 +157,7 @@ public:
 	//! Create a scalar or aggregate function in the catalog
 	DUCKDB_API CatalogEntry *CreateFunction(CatalogTransaction transaction, SchemaCatalogEntry *schema,
 	                                        CreateFunctionInfo *info);
-	//! Creates a table in the catalog.
+	//! Creates a view in the catalog
 	DUCKDB_API CatalogEntry *CreateView(CatalogTransaction transaction, SchemaCatalogEntry *schema,
 	                                    CreateViewInfo *info);
 	//! Creates a table in the catalog.

package/src/duckdb/src/include/duckdb/catalog/catalog_entry/duck_index_entry.hpp

@@ -1,7 +1,7 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/catalog/catalog_entry/dindex_catalog_entry.hpp
+// duckdb/catalog/catalog_entry/duck_index_entry.hpp
 //
 //
 //===----------------------------------------------------------------------===//

package/src/duckdb/src/include/duckdb/catalog/catalog_entry/index_catalog_entry.hpp

@@ -34,7 +34,7 @@ public:
 
 public:
 	string ToSQL() override;
-	void Serialize(duckdb::MetaBlockWriter &serializer);
+	void Serialize(Serializer &serializer);
 	static unique_ptr<CreateIndexInfo> Deserialize(Deserializer &source, ClientContext &context);
 
 	virtual string GetSchemaName() = 0;

package/src/duckdb/src/include/duckdb/common/enums/wal_type.hpp

@@ -41,6 +41,9 @@ enum class WALType : uint8_t {
 	CREATE_TABLE_MACRO = 21,
 	DROP_TABLE_MACRO = 22,
 
+	CREATE_INDEX = 23,
+	DROP_INDEX = 24,
+
 	// -----------------------------
 	// Data
 	// -----------------------------

package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp

@@ -29,24 +29,21 @@ namespace duckdb {
 class ConflictManager;
 
 struct ARTIndexScanState : public IndexScanState {
-	ARTIndexScanState() : checked(false), result_index(0) {
-	}
 
+	//! Scan predicates (single predicate scan or range scan)
 	Value values[2];
+	//! Expressions of the scan predicates
 	ExpressionType expressions[2];
-	bool checked;
+	bool checked = false;
+	//! All scanned row IDs
 	vector<row_t> result_ids;
 	Iterator iterator;
-	//! Stores the current leaf
-	Leaf *cur_leaf = nullptr;
-	//! Offset to leaf
-	idx_t result_index = 0;
 };
 
 enum class VerifyExistenceType : uint8_t {
-	APPEND = 0,    // for purpose to append into table
-	APPEND_FK = 1, // for purpose to append into table has foreign key
-	DELETE_FK = 2  // for purpose to delete from table related to foreign key
+	APPEND = 0,    // appends to a table
+	APPEND_FK = 1, // appends to a table that has a foreign key
+	DELETE_FK = 2  // delete from a table that has a foreign key
 };
 
 class ART : public Index {
@@ -62,47 +59,44 @@ public:
 	Node *tree;
 
 public:
-	//! Initialize a scan on the index with the given expression and column ids
-	//! to fetch from the base table for a single predicate
+	//! Initialize a single predicate scan on the index with the given expression and column IDs
 	unique_ptr<IndexScanState> InitializeScanSinglePredicate(const Transaction &transaction, const Value &value,
 	                                                         ExpressionType expression_type) override;
-
-	//! Initialize a scan on the index with the given expression and column ids
-	//! to fetch from the base table for two predicates
+	//! Initialize a two predicate scan on the index with the given expression and column IDs
 	unique_ptr<IndexScanState> InitializeScanTwoPredicates(Transaction &transaction, const Value &low_value,
 	                                                       ExpressionType low_expression_type, const Value &high_value,
 	                                                       ExpressionType high_expression_type) override;
-
-	//! Perform a lookup on the index
+	//! Performs a lookup on the index, fetching up to max_count result IDs. Returns true if all row IDs were fetched,
+	//! and false otherwise
 	bool Scan(Transaction &transaction, DataTable &table, IndexScanState &state, idx_t max_count,
 	          vector<row_t> &result_ids) override;
-	//! Append entries to the index
+
+	//! Called when data is appended to the index. The lock obtained from InitializeLock must be held
 	bool Append(IndexLock &lock, DataChunk &entries, Vector &row_identifiers) override;
-	//! Verify that data can be appended to the index
+	//! Verify that data can be appended to the index without a constraint violation
 	void VerifyAppend(DataChunk &chunk) override;
-	//! Verify that data can be appended to the index
+	//! Verify that data can be appended to the index without a constraint violation using the conflict manager
 	void VerifyAppend(DataChunk &chunk, ConflictManager &conflict_manager) override;
-	//! Verify that data can be appended to the index for foreign key constraint
-	void VerifyAppendForeignKey(DataChunk &chunk) override;
-	//! Verify that data can be delete from the index for foreign key constraint
-	void VerifyDeleteForeignKey(DataChunk &chunk) override;
-	//! Delete entries in the index
+	//! Delete a chunk of entries from the index. The lock obtained from InitializeLock must be held
 	void Delete(IndexLock &lock, DataChunk &entries, Vector &row_identifiers) override;
-	//! Insert data into the index
+	//! Insert a chunk of entries into the index
 	bool Insert(IndexLock &lock, DataChunk &data, Vector &row_ids) override;
 
 	//! Construct an ART from a vector of sorted keys
 	bool ConstructFromSorted(idx_t count, vector<Key> &keys, Vector &row_identifiers);
 
-	//! Search Equal and fetches the row IDs
+	//! Search equal values and fetches the row IDs
 	bool SearchEqual(Key &key, idx_t max_count, vector<row_t> &result_ids);
-	//! Search Equal used for Joins that do not need to fetch data
+	//! Search equal values used for joins that do not need to fetch data
 	void SearchEqualJoinNoFetch(Key &key, idx_t &result_size);
-	//! Serialized the ART
+
+	//! Serializes the index and returns the pair of block_id offset positions
 	BlockPointer Serialize(duckdb::MetaBlockWriter &writer) override;
 
-	//! Merge two ARTs
+	//! Merge another index into this index. The lock obtained from InitializeLock must be held, and the other
+	//! index must also be locked during the merge
 	bool MergeIndexes(IndexLock &state, Index *other_index) override;
+
 	//! Generate ART keys for an input chunk
 	static void GenerateKeys(ArenaAllocator &allocator, DataChunk &input, vector<Key> &keys);
 
@@ -110,30 +104,32 @@ public:
 	string GenerateErrorKeyName(DataChunk &input, idx_t row);
 	//! Generate the matching error message for a constraint violation
 	string GenerateConstraintErrorMessage(VerifyExistenceType verify_type, const string &key_name);
+	//! Performs constraint checking for a chunk of input data
+	void CheckConstraintsForChunk(DataChunk &input, ConflictManager &conflict_manager) override;
 
 	//! Returns the string representation of an ART
 	string ToString() override;
-	//! Verifies that the memory_size value of the ART matches its actual size
+	//! Verifies that the in-memory size value of the index matches its actual size
 	void Verify() override;
+	//! Increases the memory size by the difference between the old size and the current size
+	//! and performs verifications
+	void IncreaseAndVerifyMemorySize(idx_t old_memory_size) override;
 
 private:
-	//! Insert a row id into a leaf node
+	//! Insert a row ID into a leaf
 	bool InsertToLeaf(Leaf &leaf, row_t row_id);
-	//! Insert the leaf value into the tree
+	//! Insert a key into the tree
 	bool Insert(Node *&node, Key &key, idx_t depth, row_t row_id);
-
-	//! Erase element from leaf (if leaf has more than one value) or eliminate the leaf itself
+	//! Erase a key from the tree (if a leaf has more than one value) or erase the leaf itself
 	void Erase(Node *&node, Key &key, idx_t depth, row_t row_id);
-
-	//! Perform 'Lookup' for an entire chunk, marking which succeeded
-	void LookupValues(DataChunk &input, ConflictManager &conflict_manager) final override;
-
-	//! Find the node with a matching key, optimistic version
+	//! Find the node with a matching key, or return nullptr if not found
 	Leaf *Lookup(Node *node, Key &key, idx_t depth);
-
+	//! Returns all row IDs belonging to a key greater (or equal) than the search key
 	bool SearchGreater(ARTIndexScanState *state, Key &key, bool inclusive, idx_t max_count, vector<row_t> &result_ids);
+	//! Returns all row IDs belonging to a key less (or equal) than the upper_bound
 	bool SearchLess(ARTIndexScanState *state, Key &upper_bound, bool inclusive, idx_t max_count,
 	                vector<row_t> &result_ids);
+	//! Returns all row IDs belonging to a key within the range of lower_bound and upper_bound
 	bool SearchCloseRange(ARTIndexScanState *state, Key &lower_bound, Key &upper_bound, bool left_inclusive,
 	                      bool right_inclusive, idx_t max_count, vector<row_t> &result_ids);
 };