duckdb 0.7.2-dev2552.0 → 0.7.2-dev2675.0

package/src/duckdb/src/include/duckdb/execution/index/art/iterator.hpp

@@ -8,6 +8,7 @@
 #pragma once
 
 #include "duckdb/common/stack.hpp"
+#include "duckdb/execution/index/art/art_key.hpp"
 #include "duckdb/execution/index/art/leaf.hpp"
 #include "duckdb/execution/index/art/node.hpp"
 
@@ -16,57 +17,62 @@ namespace duckdb {
 struct IteratorEntry {
 	IteratorEntry() {
 	}
-	IteratorEntry(Node *node, idx_t pos) : node(node), pos(pos) {
+	IteratorEntry(Node node, uint8_t byte) : node(node), byte(byte) {
 	}
 
-	Node *node = nullptr;
-	idx_t pos = 0;
+	Node node;
+	uint8_t byte = 0;
 };
 
 //! Keeps track of the current key in the iterator
 class IteratorCurrentKey {
 public:
-	//! Push Byte
-	void Push(uint8_t key);
-	//! Pops n elements
-	void Pop(idx_t n);
+	//! Push byte into current key
+	void Push(const uint8_t key);
+	//! Pops n elements from the key
+	void Pop(const idx_t n);
+
 	//! Subscript operator
 	uint8_t &operator[](idx_t idx);
-	bool operator>(const Key &k) const;
-	bool operator>=(const Key &k) const;
-	bool operator==(const Key &k) const;
+	//! Greater than operator
+	bool operator>(const ARTKey &k) const;
+	//! Greater than or equal to operator
+	bool operator>=(const ARTKey &k) const;
+	//! Equal to operator
+	bool operator==(const ARTKey &k) const;
 
 private:
 	//! The current key position
 	idx_t cur_key_pos = 0;
-	//! The current key of the Leaf Node
+	//! The current key corresponding to the current leaf
 	vector<uint8_t> key;
 };
 
 class Iterator {
 public:
-	//! Current Key
+	//! All information about the current key
 	IteratorCurrentKey cur_key;
-	//! Pointer to the ART tree we are iterating
+	//! Pointer to the ART
 	ART *art = nullptr;
 
 	//! Scan the tree
-	bool Scan(Key &bound, idx_t max_count, vector<row_t> &result_ids, bool is_inclusive);
-	//! Finds minimum value of the tree
+	bool Scan(const ARTKey &key, const idx_t &max_count, vector<row_t> &result_ids, const bool &is_inclusive);
+	//! Finds the minimum value of the tree
 	void FindMinimum(Node &node);
-	//! Goes to lower bound
-	bool LowerBound(Node *node, Key &key, bool inclusive);
+	//! Goes to the lower bound of the tree
+	bool LowerBound(Node node, const ARTKey &key, const bool &is_inclusive);
 
 private:
 	//! Stack of iterator entries
 	stack<IteratorEntry> nodes;
 	//! Last visited leaf
 	Leaf *last_leaf = nullptr;
+
 	//! Go to the next node
 	bool Next();
-	//! Push part of the key to cur_key
-	void PushKey(Node *node, uint16_t pos);
-	//! Pop node
+	//! Push part of the key to the current key
+	void PushKey(const Node &node, const uint8_t byte);
+	//! Pop node from the stack of iterator entries
 	void PopNode();
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp

@@ -8,61 +8,85 @@
 
 #pragma once
 
+#include "duckdb/execution/index/art/art.hpp"
+#include "duckdb/execution/index/art/fixed_size_allocator.hpp"
 #include "duckdb/execution/index/art/node.hpp"
-#include "duckdb/storage/meta_block_reader.hpp"
+#include "duckdb/execution/index/art/prefix.hpp"
 
 namespace duckdb {
 
-class Leaf : public Node {
-public:
-	explicit Leaf();
-	Leaf(Key &value, uint32_t depth, row_t row_id);
-	Leaf(Key &value, uint32_t depth, row_t *row_ids, idx_t num_elements);
-	Leaf(row_t *row_ids, idx_t num_elements, Prefix &prefix);
-	Leaf(row_t row_id, Prefix &prefix);
-	~Leaf();
+// classes
+class Node;
+class ARTKey;
+class MetaBlockWriter;
+class MetaBlockReader;
+
+// structs
+struct BlockPointer;
 
-	//! Get the row ID at idx
-	row_t GetRowId(idx_t idx);
-	//! Get the maximum capacity of the leaf, must not match with its count
-	idx_t GetCapacity() const;
-	//! Returns whether a leaf holds exactly one inlined row ID or multiple row IDs
-	bool IsInlined() const;
-	//! Returns a pointer to all row IDs of the leaf
-	row_t *GetRowIds();
+class Leaf {
+public:
+	//! Number of row IDs
+	uint32_t count;
+	//! Compressed path (prefix)
+	Prefix prefix;
+	union {
+		//! The pointer to the head of the list of leaf segments
+		Node ptr;
+		//! Inlined row ID
+		row_t inlined;
+	} row_ids;
 
 public:
-	static Leaf *New();
-	static Leaf *New(Key &value, uint32_t depth, row_t row_id);
-	static Leaf *New(Key &value, uint32_t depth, row_t *row_ids, idx_t num_elements);
-	static Leaf *New(row_t *row_ids, idx_t num_elements, Prefix &prefix);
-	static Leaf *New(row_t row_id, Prefix &prefix);
+	//! Get a new leaf node, might cause a new buffer allocation, and initializes a leaf holding one
+	//! row ID and a prefix starting at depth
+	static Leaf &New(ART &art, Node &node, const ARTKey &key, const uint32_t depth, const row_t row_id);
+	//! Get a new leaf node, might cause a new buffer allocation, and initializes a leaf holding
+	//! n_row_ids row IDs and a prefix starting at depth
+	static Leaf &New(ART &art, Node &node, const ARTKey &key, const uint32_t depth, const row_t *row_ids,
+	                 const idx_t count);
+	//! Free the leaf
+	static void Free(ART &art, Node &node);
+	//! Get a reference to the leaf
+	static inline Leaf &Get(const ART &art, const Node ptr) {
+		return *Node::GetAllocator(art, NType::LEAF).Get<Leaf>(ptr);
+	}
+
+	//! Initializes a merge by incrementing the buffer IDs of the leaf segments
+	void InitializeMerge(const ART &art, const idx_t buffer_count);
+	//! Merge leaves
+	void Merge(ART &art, Node &other);
 
-	//! Returns the memory size of the leaf
-	idx_t MemorySize(ART &art, const bool &recurse) override;
 	//! Insert a row ID into a leaf
-	void Insert(ART &art, row_t row_id);
+	void Insert(ART &art, const row_t row_id);
 	//! Remove a row ID from a leaf
-	void Remove(ART &art, row_t row_id);
+	void Remove(ART &art, const row_t row_id);
+
+	//! Returns whether this leaf is inlined
+	inline bool IsInlined() const {
+		return count <= 1;
+	}
+	//! Get the row ID at the position
+	row_t GetRowId(const ART &art, const idx_t position) const;
+	//! Returns the position of a row ID, and an invalid index, if the leaf does not contain the row ID,
+	//! and sets the ptr to point to the segment containing the row ID
+	uint32_t FindRowId(const ART &art, Node &ptr, const row_t row_id) const;
 
 	//! Returns the string representation of a leaf
-	static string ToString(Node *node);
-	//! Merge two NLeaf nodes
-	static void Merge(ART &art, Node *&l_node, Node *&r_node);
+	string ToString(const ART &art) const;
 
-	//! Serialize a leaf
-	BlockPointer Serialize(duckdb::MetaBlockWriter &writer);
-	//! Deserialize a leaf
-	void Deserialize(ART &art, duckdb::MetaBlockReader &reader);
+	//! Serialize this leaf
+	BlockPointer Serialize(const ART &art, MetaBlockWriter &writer) const;
+	//! Deserialize this leaf
+	void Deserialize(ART &art, MetaBlockReader &reader);
 
-private:
-	union {
-		row_t inlined;
-		row_t *ptr;
-	} rowids;
+	//! Vacuum the leaf segments of a leaf, if not inlined
+	void Vacuum(ART &art);
 
 private:
-	row_t *Resize(row_t *current_row_ids, uint32_t current_count, idx_t new_capacity);
+	//! Moves the inlined row ID onto a leaf segment, does not change the size
+	//! so this will be a (temporarily) invalid leaf
+	void MoveInlinedToSegment(ART &art);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/index/art/leaf_segment.hpp

@@ -0,0 +1,36 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/index/art/leaf_segment.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+#pragma once
+
+#include "duckdb/execution/index/art/art.hpp"
+#include "duckdb/execution/index/art/node.hpp"
+
+namespace duckdb {
+
+class LeafSegment {
+public:
+	//! The row IDs stored in this segment
+	row_t row_ids[Node::LEAF_SEGMENT_SIZE];
+	//! The pointer of the next segment, if the row IDs exceeds this segment
+	Node next;
+
+public:
+	//! Get a new leaf segment node, might cause a new buffer allocation, and initialize it
+	static LeafSegment &New(ART &art, Node &node);
+	//! Get a reference to the leaf segment
+	static inline LeafSegment &Get(const ART &art, const Node ptr) {
+		return *Node::GetAllocator(art, NType::LEAF_SEGMENT).Get<LeafSegment>(ptr);
+	}
+
+	//! Append a row ID to the current segment, or create a new segment containing that row ID
+	LeafSegment &Append(ART &art, uint32_t &count, const row_t row_id);
+	//! Get the tail of a list of segments
+	LeafSegment &GetTail(const ART &art);
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp

@@ -1,138 +1,120 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/execution/index/art/node.hpp
+// duckdb/execution/index/art/art_node.hpp
 //
 //
 //===----------------------------------------------------------------------===//
 
 #pragma once
 
-#include "duckdb/common/common.hpp"
-#include "duckdb/execution/index/art/art_key.hpp"
-#include "duckdb/execution/index/art/prefix.hpp"
-#include "duckdb/storage/index.hpp"
-#include "duckdb/storage/meta_block_reader.hpp"
-#include "duckdb/storage/meta_block_writer.hpp"
-#include "duckdb/common/allocator.hpp"
+#include "duckdb/execution/index/art/fixed_size_allocator.hpp"
+#include "duckdb/execution/index/art/swizzleable_pointer.hpp"
 
 namespace duckdb {
-enum class NodeType : uint8_t { NLeaf = 0, N4 = 1, N16 = 2, N48 = 3, N256 = 4 };
-class ART;
-class Node;
-
-// Note: SwizzleablePointer assumes top 33 bits of the block_id are 0. Use a different
-// pointer implementation if that does not hold.
-class SwizzleablePointer;
-using ARTPointer = SwizzleablePointer;
-
-struct InternalType {
-	explicit InternalType(Node *n);
-
-	void Set(uint8_t *key_p, uint16_t key_size_p, ARTPointer *children_p, uint16_t children_size_p);
-	uint8_t *key;
-	uint16_t key_size;
-	ARTPointer *children;
-	uint16_t children_size;
-};
-
-struct MergeInfo {
-	MergeInfo(ART *l_art, ART *r_art, ART *root_l_art, ART *root_r_art, Node *&l_node, Node *&r_node)
-	    : l_art(l_art), r_art(r_art), root_l_art(root_l_art), root_r_art(root_r_art), l_node(l_node), r_node(r_node) {};
-	ART *l_art;
-	ART *r_art;
-	ART *root_l_art;
-	ART *root_r_art;
-	Node *&l_node;
-	Node *&r_node;
-};
 
-struct ParentsOfNodes {
-	ParentsOfNodes(Node *&l_parent, idx_t l_pos, Node *&r_parent, idx_t r_pos)
-	    : l_parent(l_parent), l_pos(l_pos), r_parent(r_parent), r_pos(r_pos) {};
-	Node *&l_parent;
-	idx_t l_pos;
-	Node *&r_parent;
-	idx_t r_pos;
+// classes
+enum class NType : uint8_t {
+	PREFIX_SEGMENT = 1,
+	LEAF_SEGMENT = 2,
+	LEAF = 3,
+	NODE_4 = 4,
+	NODE_16 = 5,
+	NODE_48 = 6,
+	NODE_256 = 7
 };
-
-class Node {
+class ART;
+class Node;
+class Prefix;
+class MetaBlockReader;
+class MetaBlockWriter;
+
+// structs
+struct BlockPointer;
+struct ARTFlags;
+
+//! The ARTNode is the swizzleable pointer class of the ART index.
+//! If the ARTNode pointer is not swizzled, then the leftmost byte identifies the NType.
+//! The remaining bytes are the position in the respective ART buffer.
+class Node : public SwizzleablePointer {
 public:
-	static const uint8_t EMPTY_MARKER = 48;
-	static const uint8_t NODE_48_SHRINK_THRESHOLD = 12;
-	static const uint8_t NODE_256_SHRINK_THRESHOLD = 36;
+	// constants (this allows testing performance with different ART node sizes)
+
+	//! Node prefixes (NOTE: this should always hold: PREFIX_SEGMENT_SIZE >= PREFIX_INLINE_BYTES)
+	static constexpr uint32_t PREFIX_INLINE_BYTES = 8;
+	static constexpr uint32_t PREFIX_SEGMENT_SIZE = 32;
+	//! Node thresholds
+	static constexpr uint8_t NODE_48_SHRINK_THRESHOLD = 12;
+	static constexpr uint8_t NODE_256_SHRINK_THRESHOLD = 36;
+	//! Node sizes
+	static constexpr uint8_t NODE_4_CAPACITY = 4;
+	static constexpr uint8_t NODE_16_CAPACITY = 16;
+	static constexpr uint8_t NODE_48_CAPACITY = 48;
+	static constexpr uint16_t NODE_256_CAPACITY = 256;
+	//! Other constants
+	static constexpr uint8_t EMPTY_MARKER = 48;
+	static constexpr uint32_t LEAF_SEGMENT_SIZE = 8;
 
 public:
-	explicit Node(NodeType type);
-	virtual ~Node() {
+	//! Constructs an empty ARTNode
+	Node();
+	//! Constructs a swizzled pointer from a block ID and an offset
+	explicit Node(MetaBlockReader &reader);
+	//! Get a new pointer to a node, might cause a new buffer allocation, and initialize it
+	static void New(ART &art, Node &node, const NType type);
+	//! Free the node (and its subtree)
+	static void Free(ART &art, Node &node);
+
+	//! Retrieve the node type from the leftmost byte
+	inline NType DecodeARTNodeType() const {
+		return NType(type);
 	}
 
-	//! Number of non-null children
-	uint16_t count;
-	//! Node type
-	NodeType type;
-	//! Compressed path (prefix)
-	Prefix prefix;
-
-	static void Delete(Node *node);
-	//! Returns the memory size of the node
-	virtual idx_t MemorySize(ART &art, const bool &recurse);
-	//! Get the position of a child corresponding exactly to the specific byte, returns DConstants::INVALID_INDEX if not
-	//! exists
-	virtual idx_t GetChildPos(uint8_t k) {
-		return DConstants::INVALID_INDEX;
-	}
-	//! Get the position of the first child that is greater or equal to the specific byte, or DConstants::INVALID_INDEX
-	//! if there are no children matching the criteria
-	virtual idx_t GetChildGreaterEqual(uint8_t k, bool &equal) {
-		throw InternalException("Unimplemented GetChildGreaterEqual for ART node");
+	//! Set the pointer
+	inline void SetPtr(const SwizzleablePointer ptr) {
+		offset = ptr.offset;
+		buffer_id = ptr.buffer_id;
 	}
-	//! Get the position of the minimum element in the node
-	virtual idx_t GetMin();
-	//! Get the next position in the node, or DConstants::INVALID_INDEX if there is no next position. if pos ==
-	//! DConstants::INVALID_INDEX, then the first valid position in the node is returned
-	virtual idx_t GetNextPos(idx_t pos) {
-		return DConstants::INVALID_INDEX;
-	}
-	//! Get the next position and byte in the node, or DConstants::INVALID_INDEX if there is no next position. if pos ==
-	//! DConstants::INVALID_INDEX, then the first valid position in the node is returned
-	virtual idx_t GetNextPosAndByte(idx_t pos, uint8_t &byte) {
-		return DConstants::INVALID_INDEX;
-	}
-	//! Get the child at the specified position in the node. pos should be between [0, count). Throws an assertion if
-	//! the element is not found
-	virtual Node *GetChild(ART &art, idx_t pos);
-	//! Replaces the pointer to a child node
-	virtual void ReplaceChildPointer(idx_t pos, Node *node);
-	//! Returns whether the child at pos is in memory
-	virtual bool ChildIsInMemory(idx_t pos);
-
-	//! Insert a new child node at key_byte into the node
-	static void InsertChild(ART &art, Node *&node, uint8_t key_byte, Node *new_child);
-	//! Erase child node entry from node
-	static void EraseChild(ART &art, Node *&node, idx_t pos);
-	//! Get the corresponding node type for the provided size
-	static NodeType GetTypeBySize(idx_t size);
-	//! Create a new node of the specified type
-	static void New(const NodeType &type, Node *&node);
-
-	//! Returns the string representation of a node
-	string ToString(ART &art);
-	//! Serialize this node
-	BlockPointer Serialize(ART &art, duckdb::MetaBlockWriter &writer);
-	//! Returns the memory size of the node
-	idx_t RecursiveMemorySize(ART &art);
-
-	//! Deserialize this node
-	static Node *Deserialize(ART &art, idx_t block_id, idx_t offset);
-	//! Merge two ART
-	static bool MergeARTs(ART *l_art, ART *r_art);
-
-private:
-	//! Serialize internal nodes
-	BlockPointer SerializeInternal(ART &art, duckdb::MetaBlockWriter &writer, InternalType &internal_type);
-	//! Deserialize internal nodes
-	void DeserializeInternal(ART &art, duckdb::MetaBlockReader &reader);
+
+	//! Replace the child node at the respective byte
+	void ReplaceChild(const ART &art, const uint8_t byte, const Node child);
+	//! Insert the child node at byte
+	static void InsertChild(ART &art, Node &node, const uint8_t byte, const Node child);
+	//! Delete the child node at the respective byte
+	static void DeleteChild(ART &art, Node &node, const uint8_t byte);
+
+	//! Get the child for the respective byte in the node
+	optional_ptr<Node> GetChild(ART &art, const uint8_t byte) const;
+	//! Get the first child that is greater or equal to the specific byte
+	optional_ptr<Node> GetNextChild(ART &art, uint8_t &byte) const;
+
+	//! Serialize the node
+	BlockPointer Serialize(ART &art, MetaBlockWriter &writer);
+	//! Deserialize the node
+	void Deserialize(ART &art);
+
+	//! Returns the string representation of the node
+	string ToString(ART &art) const;
+	//! Returns the capacity of the node
+	idx_t GetCapacity() const;
+	//! Returns a pointer to the prefix of the node
+	Prefix &GetPrefix(ART &art);
+	//! Returns the matching node type for a given count
+	static NType GetARTNodeTypeByCount(const idx_t count);
+	//! Get references to the different allocators
+	static FixedSizeAllocator &GetAllocator(const ART &art, NType type);
+
+	//! Initializes a merge by fully deserializing the subtree of the node and incrementing its buffer IDs
+	void InitializeMerge(ART &art, const ARTFlags &flags);
+	//! Merge another node into this node
+	bool Merge(ART &art, Node &other);
+	//! Merge two nodes by first resolving their prefixes
+	bool ResolvePrefixes(ART &art, Node &other);
+	//! Merge two nodes that have no prefix or the same prefix
+	bool MergeInternal(ART &art, Node &other);
+
+	//! Vacuum all nodes that exceed their respective vacuum thresholds
+	static void Vacuum(ART &art, Node &node, const ARTFlags &flags);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/index/art/node16.hpp

@@ -7,50 +7,62 @@
 //===----------------------------------------------------------------------===//
 
 #pragma once
+
+#include "duckdb/execution/index/art/art.hpp"
+#include "duckdb/execution/index/art/fixed_size_allocator.hpp"
 #include "duckdb/execution/index/art/node.hpp"
-#include "duckdb/execution/index/art/swizzleable_pointer.hpp"
+#include "duckdb/execution/index/art/prefix.hpp"
 
 namespace duckdb {
 
-class Node16 : public Node {
+//! Node16 holds up to 16 ARTNode children sorted by their key byte
+class Node16 {
 public:
-	//! Empty Node16
-	explicit Node16();
+	//! Number of non-null children
+	uint8_t count;
+	//! Compressed path (prefix)
+	Prefix prefix;
 	//! Array containing all partial key bytes
-	uint8_t key[16];
-	//! ART pointers to the child nodes
-	ARTPointer children[16];
+	uint8_t key[Node::NODE_16_CAPACITY];
+	//! ART node pointers to the child nodes
+	Node children[Node::NODE_16_CAPACITY];
 
 public:
-	static Node16 *New();
-	//! Returns the memory size of the Node16
-	idx_t MemorySize(ART &art, const bool &recurse) override;
-	//! Get position of a specific byte, returns DConstants::INVALID_INDEX if not exists
-	idx_t GetChildPos(uint8_t k) override;
-	//! Get the position of the first child that is greater or equal to the specific byte, or DConstants::INVALID_INDEX
-	//! if there are no children matching the criteria
-	idx_t GetChildGreaterEqual(uint8_t k, bool &equal) override;
-	//! Get the position of the minimum element in the node
-	idx_t GetMin() override;
-	//! Get the next position in the node, or DConstants::INVALID_INDEX if there is no next position
-	idx_t GetNextPos(idx_t pos) override;
-	//! Get the next position in the node, or DConstants::INVALID_INDEX if there is no next position
-	idx_t GetNextPosAndByte(idx_t pos, uint8_t &byte) override;
-	//! Get Node16 child
-	Node *GetChild(ART &art, idx_t pos) override;
-	//! Replace child pointer
-	void ReplaceChildPointer(idx_t pos, Node *node) override;
-	//! Returns whether the child at pos is in memory
-	bool ChildIsInMemory(idx_t pos) override;
-
-	//! Insert a new child node at key_byte into the Node16
-	static void InsertChild(ART &art, Node *&node, uint8_t key_byte, Node *new_child);
-	//! Erase the child at pos and (if necessary) shrink to Node4
-	static void EraseChild(ART &art, Node *&node, idx_t pos);
-
-	//! Returns the size (maximum capacity) of the Node16
-	static constexpr idx_t GetSize() {
-		return 16;
+	//! Get a new Node16 node, might cause a new buffer allocation, and initialize it
+	static Node16 &New(ART &art, Node &node);
+	//! Free the node (and its subtree)
+	static void Free(ART &art, Node &node);
+	//! Get a reference to the node
+	static inline Node16 &Get(const ART &art, const Node ptr) {
+		return *Node::GetAllocator(art, NType::NODE_16).Get<Node16>(ptr);
 	}
+	//! Initializes all the fields of the node while growing a Node4 to a Node16
+	static Node16 &GrowNode4(ART &art, Node &node16, Node &node4);
+	//! Initializes all fields of the node while shrinking a Node48 to a Node16
+	static Node16 &ShrinkNode48(ART &art, Node &node16, Node &node48);
+
+	//! Initializes a merge by incrementing the buffer IDs of the node
+	void InitializeMerge(ART &art, const ARTFlags &flags);
+
+	//! Insert a child node at byte
+	static void InsertChild(ART &art, Node &node, const uint8_t byte, const Node child);
+	//! Delete the child node at the respective byte
+	static void DeleteChild(ART &art, Node &node, const uint8_t byte);
+
+	//! Replace the child node at the respective byte
+	void ReplaceChild(const uint8_t byte, const Node child);
+
+	//! Get the child for the respective byte in the node
+	optional_ptr<Node> GetChild(const uint8_t byte);
+	//! Get the first child that is greater or equal to the specific byte
+	optional_ptr<Node> GetNextChild(uint8_t &byte);
+
+	//! Serialize an ART node
+	BlockPointer Serialize(ART &art, MetaBlockWriter &writer);
+	//! Deserialize this node
+	void Deserialize(ART &art, MetaBlockReader &reader);
+
+	//! Vacuum the children of the node
+	void Vacuum(ART &art, const ARTFlags &flags);
 };
 } // namespace duckdb