duckdb 1.0.1-dev22.0 → 1.0.1-dev27.0

package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp

@@ -9,16 +9,14 @@
 #pragma once
 
 #include "duckdb/common/assert.hpp"
+#include "duckdb/common/limits.hpp"
 #include "duckdb/common/optional_ptr.hpp"
-#include "duckdb/common/to_string.hpp"
 #include "duckdb/common/typedefs.hpp"
-#include "duckdb/common/limits.hpp"
-#include "duckdb/execution/index/index_pointer.hpp"
 #include "duckdb/execution/index/fixed_size_allocator.hpp"
+#include "duckdb/execution/index/index_pointer.hpp"
 
 namespace duckdb {
 
-// classes
 enum class NType : uint8_t {
 	PREFIX = 1,
 	LEAF = 2,
@@ -27,107 +25,168 @@ enum class NType : uint8_t {
 	NODE_48 = 5,
 	NODE_256 = 6,
 	LEAF_INLINED = 7,
+	NODE_7_LEAF = 8,
+	NODE_15_LEAF = 9,
+	NODE_256_LEAF = 10,
+};
+
+enum class GateStatus : uint8_t {
+	GATE_NOT_SET = 0,
+	GATE_SET = 1,
 };
 
 class ART;
 class Prefix;
-class MetadataReader;
-class MetadataWriter;
-
-// structs
-struct BlockPointer;
-struct ARTFlags;
-struct MetaBlockPointer;
+class ARTKey;
 
 //! The Node is the pointer class of the ART index.
-//! It inherits from the IndexPointer, and adds ART-specific functionality
+//! It inherits from the IndexPointer, and adds ART-specific functionality.
 class Node : public IndexPointer {
+	friend class Prefix;
+
 public:
-	//! Node thresholds
-	static constexpr uint8_t NODE_48_SHRINK_THRESHOLD = 12;
-	static constexpr uint8_t NODE_256_SHRINK_THRESHOLD = 36;
-	//! Node sizes
-	static constexpr uint8_t NODE_4_CAPACITY = 4;
-	static constexpr uint8_t NODE_16_CAPACITY = 16;
-	static constexpr uint8_t NODE_48_CAPACITY = 48;
-	static constexpr uint16_t NODE_256_CAPACITY = 256;
-	//! Other constants
-	static constexpr uint8_t EMPTY_MARKER = 48;
-	static constexpr uint8_t LEAF_SIZE = 4;
-	static constexpr uint8_t PREFIX_SIZE = 15;
+	//! A gate sets the leftmost bit of the metadata, binary: 1000-0000.
+	static constexpr uint8_t AND_GATE = 0x80;
 	static constexpr idx_t AND_ROW_ID = 0x00FFFFFFFFFFFFFF;
 
 public:
-	//! Get a new pointer to a node, might cause a new buffer allocation, and initialize it
+	//! Get a new pointer to a node and initialize it.
 	static void New(ART &art, Node &node, const NType type);
-	//! Free the node (and its subtree)
+	//! Free the node and its children.
 	static void Free(ART &art, Node &node);
 
-	//! Get references to the allocator
+	//! Get a reference to the allocator.
 	static FixedSizeAllocator &GetAllocator(const ART &art, const NType type);
-	//! Get a (immutable) reference to the node. If dirty is false, then T should be a const class
+	//! Get the index of a node type's allocator.
+	static uint8_t GetAllocatorIdx(const NType type);
+
+	//! Get a reference to a node.
 	template <class NODE>
-	static inline const NODE &Ref(const ART &art, const Node ptr, const NType type) {
-		return *(GetAllocator(art, type).Get<const NODE>(ptr, false));
+	static inline NODE &Ref(const ART &art, const Node ptr, const NType type) {
+		D_ASSERT(ptr.GetType() != NType::PREFIX);
+		return *(GetAllocator(art, type).Get<NODE>(ptr, !std::is_const<NODE>::value));
 	}
-	//! Get a (const) reference to the node. If dirty is false, then T should be a const class
+	//! Get a node pointer, if the node is in memory, else nullptr.
 	template <class NODE>
-	static inline NODE &RefMutable(const ART &art, const Node ptr, const NType type) {
-		return *(GetAllocator(art, type).Get<NODE>(ptr));
+	static inline unsafe_optional_ptr<NODE> InMemoryRef(const ART &art, const Node ptr, const NType type) {
+		D_ASSERT(ptr.GetType() != NType::PREFIX);
+		return GetAllocator(art, type).GetIfLoaded<NODE>(ptr);
 	}
 
-	//! Replace the child node at byte
-	void ReplaceChild(const ART &art, const uint8_t byte, const Node child) const;
-	//! Insert the child node at byte
-	static void InsertChild(ART &art, Node &node, const uint8_t byte, const Node child);
-	//! Delete the child node at byte
-	static void DeleteChild(ART &art, Node &node, Node &prefix, const uint8_t byte);
-
-	//! Get the child (immutable) for the respective byte in the node
-	optional_ptr<const Node> GetChild(ART &art, const uint8_t byte) const;
-	//! Get the child for the respective byte in the node
-	optional_ptr<Node> GetChildMutable(ART &art, const uint8_t byte) const;
-	//! Get the first child (immutable) that is greater or equal to the specific byte
-	optional_ptr<const Node> GetNextChild(ART &art, uint8_t &byte) const;
-	//! Get the first child that is greater or equal to the specific byte
-	optional_ptr<Node> GetNextChildMutable(ART &art, uint8_t &byte) const;
-
-	//! Returns the string representation of the node, or only traverses and verifies the node and its subtree
+	//! Replace the child at byte.
+	void ReplaceChild(const ART &art, const uint8_t byte, const Node child = Node()) const;
+	//! Insert the child at byte.
+	static void InsertChild(ART &art, Node &node, const uint8_t byte, const Node child = Node());
+	//! Delete the child at byte.
+	static void DeleteChild(ART &art, Node &node, Node &prefix, const uint8_t byte, const GateStatus status,
+	                        const ARTKey &row_id);
+
+	//! Get the immutable child at byte.
+	const unsafe_optional_ptr<Node> GetChild(ART &art, const uint8_t byte) const;
+	//! Get the child at byte.
+	unsafe_optional_ptr<Node> GetChildMutable(ART &art, const uint8_t byte) const;
+	//! Get the first immutable child greater than or equal to the byte.
+	const unsafe_optional_ptr<Node> GetNextChild(ART &art, uint8_t &byte) const;
+	//! Get the first child greater than or equal to the byte.
+	unsafe_optional_ptr<Node> GetNextChildMutable(ART &art, uint8_t &byte) const;
+	//! Returns true, if the byte exists, else false.
+	bool HasByte(ART &art, uint8_t &byte) const;
+	//! Get the first byte greater than or equal to the byte.
+	bool GetNextByte(ART &art, uint8_t &byte) const;
+
+	//! Returns the string representation of the node, if only_verify is false.
+	//! Else, it traverses and verifies the node.
 	string VerifyAndToString(ART &art, const bool only_verify) const;
-	//! Returns the capacity of the node
-	idx_t GetCapacity() const;
-	//! Returns the matching node type for a given count
-	static NType GetARTNodeTypeByCount(const idx_t count);
-
-	//! Initializes a merge by incrementing the buffer IDs of a node and its subtree
-	void InitializeMerge(ART &art, const ARTFlags &flags);
-	//! Merge another node into this node
-	bool Merge(ART &art, Node &other);
-	//! Merge two nodes by first resolving their prefixes
-	bool ResolvePrefixes(ART &art, Node &other);
-	//! Merge two nodes that have no prefix or the same prefix
-	bool MergeInternal(ART &art, Node &other);
-
-	//! Vacuum all nodes that exceed their respective vacuum thresholds
-	void Vacuum(ART &art, const ARTFlags &flags);
-
-	//! Get the row ID (8th to 63rd bit)
+	//! Counts each node type.
+	void VerifyAllocations(ART &art, unordered_map<uint8_t, idx_t> &node_counts) const;
+
+	//! Returns the node type for a count.
+	static NType GetNodeType(const idx_t count);
+
+	//! Initialize a merge by incrementing the buffer IDs of a node and its children.
+	void InitMerge(ART &art, const unsafe_vector<idx_t> &upper_bounds);
+	//! Merge a node into this node.
+	bool Merge(ART &art, Node &other, const GateStatus status);
+
+	//! Vacuum all nodes exceeding their vacuum threshold.
+	void Vacuum(ART &art, const unordered_set<uint8_t> &indexes);
+
+	//! Transform the node storage to deprecated storage.
+	static void TransformToDeprecated(ART &art, Node &node, unsafe_unique_ptr<FixedSizeAllocator> &allocator);
+
+	//! Returns the node type.
+	inline NType GetType() const {
+		return NType(GetMetadata() & ~AND_GATE);
+	}
+
+	//! True, if the node is a Node4, Node16, Node48, or Node256.
+	bool IsNode() const;
+	//! True, if the node is a Node7Leaf, Node15Leaf, or Node256Leaf.
+	bool IsLeafNode() const;
+	//! True, if the node is any leaf.
+	bool IsAnyLeaf() const;
+
+	//! Get the row ID (8th to 63rd bit).
 	inline row_t GetRowId() const {
-		return Get() & AND_ROW_ID;
+		return UnsafeNumericCast<row_t>(Get() & AND_ROW_ID);
 	}
-	//! Set the row ID (8th to 63rd bit)
+	//! Set the row ID (8th to 63rd bit).
 	inline void SetRowId(const row_t row_id) {
 		Set((Get() & AND_METADATA) | UnsafeNumericCast<idx_t>(row_id));
 	}
 
-	//! Returns the type of the node, which is held in the metadata
-	inline NType GetType() const {
-		return NType(GetMetadata());
+	//! Returns the gate status of a node.
+	inline GateStatus GetGateStatus() const {
+		return (GetMetadata() & AND_GATE) == 0 ? GateStatus::GATE_NOT_SET : GateStatus::GATE_SET;
+	}
+	//! Sets the gate status of a node.
+	inline void SetGateStatus(const GateStatus status) {
+		switch (status) {
+		case GateStatus::GATE_SET:
+			SetMetadata(GetMetadata() | AND_GATE);
+			break;
+		case GateStatus::GATE_NOT_SET:
+			SetMetadata(GetMetadata() & ~AND_GATE);
+			break;
+		}
 	}
 
-	//! Assign operator
+	//! Assign operator.
 	inline void operator=(const IndexPointer &ptr) {
 		Set(ptr.Get());
 	}
+
+private:
+	bool MergeNormalNodes(ART &art, Node &l_node, Node &r_node, uint8_t &byte, const GateStatus status);
+	void MergeLeafNodes(ART &art, Node &l_node, Node &r_node, uint8_t &byte);
+	bool MergeNodes(ART &art, Node &other, const GateStatus status);
+	bool PrefixContainsOther(ART &art, Node &l_node, Node &r_node, const uint8_t pos, const GateStatus status);
+	void MergeIntoNode4(ART &art, Node &l_node, Node &r_node, const uint8_t pos);
+	bool MergePrefixes(ART &art, Node &other, const GateStatus status);
+	bool MergeInternal(ART &art, Node &other, const GateStatus status);
+
+private:
+	template <class NODE>
+	static void InitMergeInternal(ART &art, NODE &n, const unsafe_vector<idx_t> &upper_bounds) {
+		NODE::Iterator(n, [&](Node &child) { child.InitMerge(art, upper_bounds); });
+	}
+
+	template <class NODE>
+	static void VacuumInternal(ART &art, NODE &n, const unordered_set<uint8_t> &indexes) {
+		NODE::Iterator(n, [&](Node &child) { child.Vacuum(art, indexes); });
+	}
+
+	template <class NODE>
+	static void TransformToDeprecatedInternal(ART &art, unsafe_optional_ptr<NODE> ptr,
+	                                          unsafe_unique_ptr<FixedSizeAllocator> &allocator) {
+		if (ptr) {
+			NODE::Iterator(*ptr, [&](Node &child) { Node::TransformToDeprecated(art, child, allocator); });
+		}
+	}
+
+	template <class NODE>
+	static void VerifyAllocationsInternal(ART &art, NODE &n, unordered_map<uint8_t, idx_t> &node_counts) {
+		NODE::Iterator(n, [&](const Node &child) { child.VerifyAllocations(art, node_counts); });
+	}
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/index/art/node256.hpp

@@ -14,50 +14,67 @@
 
 namespace duckdb {
 
-//! Node256 holds up to 256 Node children which can be directly indexed by the key byte
+//! Node256 holds up to 256 children. They are indexed by their key byte.
 class Node256 {
+	friend class Node48;
+
+public:
+	static constexpr NType NODE_256 = NType::NODE_256;
+	static constexpr uint16_t CAPACITY = 256;
+	static constexpr uint8_t SHRINK_THRESHOLD = 36;
+
 public:
-	//! Delete copy constructors, as any Node256 can never own its memory
+	Node256() = delete;
 	Node256(const Node256 &) = delete;
 	Node256 &operator=(const Node256 &) = delete;
 
-	//! Number of non-null children
+private:
 	uint16_t count;
-	//! Node pointers to the child nodes
-	Node children[Node::NODE_256_CAPACITY];
+	Node children[CAPACITY];
 
 public:
-	//! Get a new Node256, might cause a new buffer allocation, and initialize it
+	//! Get a new Node256 and initialize it.
 	static Node256 &New(ART &art, Node &node);
-	//! Free the node (and its subtree)
+	//! Free the node and its children.
 	static void Free(ART &art, Node &node);
 
-	//! Initializes all the fields of the node while growing a Node48 to a Node256
-	static Node256 &GrowNode48(ART &art, Node &node256, Node &node48);
-
-	//! Initializes a merge by incrementing the buffer IDs of the node
-	void InitializeMerge(ART &art, const ARTFlags &flags);
-
-	//! Insert a child node at byte
+	//! Insert a child at byte.
 	static void InsertChild(ART &art, Node &node, const uint8_t byte, const Node child);
-	//! Delete the child node at byte
+	//! Delete the child at byte.
 	static void DeleteChild(ART &art, Node &node, const uint8_t byte);
+	//! Replace the child at byte.
+	void ReplaceChild(const uint8_t byte, const Node child);
 
-	//! Replace the child node at byte
-	inline void ReplaceChild(const uint8_t byte, const Node child) {
-		children[byte] = child;
+public:
+	template <class F, class NODE>
+	static void Iterator(NODE &n, F &&lambda) {
+		for (idx_t i = 0; i < CAPACITY; i++) {
+			if (n.children[i].HasMetadata()) {
+				lambda(n.children[i]);
+			}
+		}
+	}
+
+	template <class NODE>
+	static unsafe_optional_ptr<Node> GetChild(NODE &n, const uint8_t byte) {
+		if (n.children[byte].HasMetadata()) {
+			return &n.children[byte];
+		}
+		return nullptr;
 	}
 
-	//! Get the (immutable) child for the respective byte in the node
-	optional_ptr<const Node> GetChild(const uint8_t byte) const;
-	//! Get the child for the respective byte in the node
-	optional_ptr<Node> GetChildMutable(const uint8_t byte);
-	//! Get the first (immutable) child that is greater or equal to the specific byte
-	optional_ptr<const Node> GetNextChild(uint8_t &byte) const;
-	//! Get the first child that is greater or equal to the specific byte
-	optional_ptr<Node> GetNextChildMutable(uint8_t &byte);
-
-	//! Vacuum the children of the node
-	void Vacuum(ART &art, const ARTFlags &flags);
+	template <class NODE>
+	static unsafe_optional_ptr<Node> GetNextChild(NODE &n, uint8_t &byte) {
+		for (idx_t i = byte; i < CAPACITY; i++) {
+			if (n.children[i].HasMetadata()) {
+				byte = UnsafeNumericCast<uint8_t>(i);
+				return &n.children[i];
+			}
+		}
+		return nullptr;
+	}
+
+private:
+	static Node256 &GrowNode48(ART &art, Node &node256, Node &node48);
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/index/art/node256_leaf.hpp

@@ -0,0 +1,53 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/index/art/node256_leaf.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/execution/index/art/art.hpp"
+#include "duckdb/execution/index/art/node.hpp"
+#include "duckdb/execution/index/art/node256.hpp"
+#include "duckdb/execution/index/fixed_size_allocator.hpp"
+
+namespace duckdb {
+
+//! Node256Leaf is a bitmask containing 256 bits.
+class Node256Leaf {
+	friend class Node15Leaf;
+
+public:
+	static constexpr NType NODE_256_LEAF = NType::NODE_256_LEAF;
+	static constexpr uint16_t CAPACITY = Node256::CAPACITY;
+
+public:
+	Node256Leaf() = delete;
+	Node256Leaf(const Node256Leaf &) = delete;
+	Node256Leaf &operator=(const Node256Leaf &) = delete;
+
+private:
+	uint16_t count;
+	validity_t mask[CAPACITY / sizeof(validity_t)];
+
+public:
+	//! Get a new Node256Leaf and initialize it.
+	static Node256Leaf &New(ART &art, Node &node);
+
+	//! Insert a byte.
+	static void InsertByte(ART &art, Node &node, const uint8_t byte);
+	//! Delete a byte.
+	static void DeleteByte(ART &art, Node &node, const uint8_t byte);
+
+	//! Returns true, if the byte exists, else false.
+	bool HasByte(uint8_t &byte);
+	//! Get the first byte greater or equal to the byte.
+	bool GetNextByte(uint8_t &byte);
+
+private:
+	static Node256Leaf &GrowNode15Leaf(ART &art, Node &node256_leaf, Node &node15_leaf);
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/index/art/node48.hpp

@@ -8,62 +8,79 @@
 
 #pragma once
 
-#include "duckdb/execution/index/fixed_size_allocator.hpp"
 #include "duckdb/execution/index/art/art.hpp"
 #include "duckdb/execution/index/art/node.hpp"
+#include "duckdb/execution/index/art/node256.hpp"
+#include "duckdb/execution/index/fixed_size_allocator.hpp"
 
 namespace duckdb {
 
-//! Node48 holds up to 48 Node children. It contains a child_index array which can be directly indexed by the key
-//! byte, and which contains the position of the child node in the children array
+//! Node48 holds up to 48 children. The child_index array is indexed by the key byte.
+//! It contains the position of the child node in the children array.
 class Node48 {
+	friend class Node16;
+	friend class Node256;
+
+public:
+	static constexpr NType NODE_48 = NType::NODE_48;
+	static constexpr uint8_t CAPACITY = 48;
+	static constexpr uint8_t EMPTY_MARKER = 48;
+	static constexpr uint8_t SHRINK_THRESHOLD = 12;
+
 public:
-	//! Delete copy constructors, as any Node48 can never own its memory
+	Node48() = delete;
 	Node48(const Node48 &) = delete;
 	Node48 &operator=(const Node48 &) = delete;
 
-	//! Number of non-null children
+private:
 	uint8_t count;
-	//! Array containing all possible partial key bytes, those not set have an EMPTY_MARKER
-	uint8_t child_index[Node::NODE_256_CAPACITY];
-	//! Node pointers to the child nodes
-	Node children[Node::NODE_48_CAPACITY];
+	uint8_t child_index[Node256::CAPACITY];
+	Node children[CAPACITY];
 
 public:
-	//! Get a new Node48, might cause a new buffer allocation, and initialize it
+	//! Get a new Node48 and initialize it.
 	static Node48 &New(ART &art, Node &node);
-	//! Free the node (and its subtree)
+	//! Free the node and its children.
 	static void Free(ART &art, Node &node);
 
-	//! Initializes all the fields of the node while growing a Node16 to a Node48
-	static Node48 &GrowNode16(ART &art, Node &node48, Node &node16);
-	//! Initializes all fields of the node while shrinking a Node256 to a Node48
-	static Node48 &ShrinkNode256(ART &art, Node &node48, Node &node256);
-
-	//! Initializes a merge by incrementing the buffer IDs of the node
-	void InitializeMerge(ART &art, const ARTFlags &flags);
-
-	//! Insert a child node at byte
+	//! Insert a child at byte.
 	static void InsertChild(ART &art, Node &node, const uint8_t byte, const Node child);
-	//! Delete the child node at byte
+	//! Delete the child at byte.
 	static void DeleteChild(ART &art, Node &node, const uint8_t byte);
+	//! Replace the child at byte.
+	void ReplaceChild(const uint8_t byte, const Node child);
 
-	//! Replace the child node at byte
-	inline void ReplaceChild(const uint8_t byte, const Node child) {
-		D_ASSERT(child_index[byte] != Node::EMPTY_MARKER);
-		children[child_index[byte]] = child;
+public:
+	template <class F, class NODE>
+	static void Iterator(NODE &n, F &&lambda) {
+		for (idx_t i = 0; i < Node256::CAPACITY; i++) {
+			if (n.child_index[i] != EMPTY_MARKER) {
+				lambda(n.children[n.child_index[i]]);
+			}
+		}
 	}
 
-	//! Get the (immutable) child for the respective byte in the node
-	optional_ptr<const Node> GetChild(const uint8_t byte) const;
-	//! Get the child for the respective byte in the node
-	optional_ptr<Node> GetChildMutable(const uint8_t byte);
-	//! Get the first (immutable) child that is greater or equal to the specific byte
-	optional_ptr<const Node> GetNextChild(uint8_t &byte) const;
-	//! Get the first child that is greater or equal to the specific byte
-	optional_ptr<Node> GetNextChildMutable(uint8_t &byte);
+	template <class NODE>
+	static unsafe_optional_ptr<Node> GetChild(NODE &n, const uint8_t byte) {
+		if (n.child_index[byte] != Node48::EMPTY_MARKER) {
+			return &n.children[n.child_index[byte]];
+		}
+		return nullptr;
+	}
 
-	//! Vacuum the children of the node
-	void Vacuum(ART &art, const ARTFlags &flags);
+	template <class NODE>
+	static unsafe_optional_ptr<Node> GetNextChild(NODE &n, uint8_t &byte) {
+		for (idx_t i = byte; i < Node256::CAPACITY; i++) {
+			if (n.child_index[i] != EMPTY_MARKER) {
+				byte = UnsafeNumericCast<uint8_t>(i);
+				return &n.children[n.child_index[i]];
+			}
+		}
+		return nullptr;
+	}
+
+private:
+	static Node48 &GrowNode16(ART &art, Node &node48, Node &node16);
+	static Node48 &ShrinkNode256(ART &art, Node &node48, Node &node256);
 };
 } // namespace duckdb