duckdb 0.8.2-dev1791.0 → 0.8.2-dev1859.0

package/src/duckdb/src/include/duckdb/execution/index/art/fixed_size_allocator.hpp

@@ -8,16 +8,14 @@
 
 #pragma once
 
-#include "duckdb/common/unordered_set.hpp"
-#include "duckdb/common/constants.hpp"
-#include "duckdb/common/vector.hpp"
-#include "duckdb/common/assert.hpp"
 #include "duckdb/common/types/validity_mask.hpp"
+#include "duckdb/common/unordered_set.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
-#include "duckdb/execution/index/art/swizzleable_pointer.hpp"
+#include "duckdb/execution/index/art/node.hpp"
 
 namespace duckdb {
 
+// structs
 struct BufferEntry {
 	BufferEntry(const data_ptr_t &ptr, const idx_t &allocation_count) : ptr(ptr), allocation_count(allocation_count) {
 	}
@@ -26,7 +24,7 @@ struct BufferEntry {
 };
 
 //! The FixedSizeAllocator provides pointers to fixed-size sections of pre-allocated memory buffers.
-//! The pointers are SwizzleablePointers, and the leftmost byte (swizzle flag and type) must always be zero.
+//! The pointers are Node pointers, and the leftmost byte (serialize flag and type) must always be zero.
 class FixedSizeAllocator {
 public:
 	//! Fixed size of the buffers
@@ -65,17 +63,17 @@ public:
 	Allocator &allocator;
 
 public:
-	//! Get a new pointer to data, might cause a new buffer allocation
-	SwizzleablePointer New();
-	//! Free the data of the pointer
-	void Free(const SwizzleablePointer ptr);
-	//! Get the data of the pointer
+	//! Get a new Node pointer to data, might cause a new buffer allocation
+	Node New();
+	//! Free the data of the Node pointer
+	void Free(const Node ptr);
+	//! Get the data of the Node pointer
 	template <class T>
-	inline T *Get(const SwizzleablePointer ptr) const {
+	inline T *Get(const Node ptr) const {
 		return (T *)Get(ptr);
 	}
 
-	//! Resets the allocator, which e.g. becomes necessary during DELETE FROM table
+	//! Resets the allocator, e.g., becomes necessary during DELETE FROM table
 	void Reset();
 
 	//! Returns the allocated memory size in bytes
@@ -83,32 +81,32 @@ public:
 		return buffers.size() * BUFFER_ALLOC_SIZE;
 	}
 
-	//! Merge another FixedSizeAllocator with this allocator. Both must have the same allocation size
+	//! Merge another FixedSizeAllocator into this allocator. Both must have the same allocation size
 	void Merge(FixedSizeAllocator &other);
 
 	//! Initialize a vacuum operation, and return true, if the allocator needs a vacuum
 	bool InitializeVacuum();
 	//! Finalize a vacuum operation by freeing all buffers exceeding the min_vacuum_buffer_id
 	void FinalizeVacuum();
-	//! Returns true, if a pointer qualifies for a vacuum operation, and false otherwise
-	inline bool NeedsVacuum(const SwizzleablePointer ptr) const {
-		if (ptr.buffer_id >= min_vacuum_buffer_id) {
+	//! Returns true, if a Node pointer qualifies for a vacuum operation, and false otherwise
+	inline bool NeedsVacuum(const Node ptr) const {
+		if (ptr.GetBufferId() >= min_vacuum_buffer_id) {
 			return true;
 		}
 		return false;
 	}
-	//! Vacuums a pointer
-	SwizzleablePointer VacuumPointer(const SwizzleablePointer ptr);
+	//! Vacuums a Node pointer
+	Node VacuumPointer(const Node ptr);
 
 	//! Verify that the allocation counts match the existing positions on the buffers
 	void Verify() const;
 
 private:
-	//! Returns the data_ptr_t of a pointer
-	inline data_ptr_t Get(const SwizzleablePointer ptr) const {
-		D_ASSERT(ptr.buffer_id < buffers.size());
-		D_ASSERT(ptr.offset < allocations_per_buffer);
-		return buffers[ptr.buffer_id].ptr + ptr.offset * allocation_size + allocation_offset;
+	//! Returns the data_ptr_t of a Node pointer
+	inline data_ptr_t Get(const Node ptr) const {
+		D_ASSERT(ptr.GetBufferId() < buffers.size());
+		D_ASSERT(ptr.GetOffset() < allocations_per_buffer);
+		return buffers[ptr.GetBufferId()].ptr + ptr.GetOffset() * allocation_size + allocation_offset;
 	}
 	//! Returns the first free offset in a bitmask
 	uint32_t GetOffset(ValidityMask &mask, const idx_t allocation_count);

package/src/duckdb/src/include/duckdb/execution/index/art/iterator.hpp

@@ -70,8 +70,8 @@ public:
 private:
 	//! Stack of nodes from the root to the currently active node
 	stack<IteratorEntry> nodes;
-	//! Last visited leaf
-	optional_ptr<Leaf> last_leaf = nullptr;
+	//! Last visited leaf node
+	Node last_leaf = Node();
 
 	//! Goes to the next leaf in the ART and sets it as last_leaf,
 	//! returns false if there is no next leaf

package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp

@@ -21,64 +21,67 @@ class MetaBlockReader;
 // structs
 struct BlockPointer;
 
+//! The LEAF is a special node type that contains a count, up to LEAF_SIZE row IDs,
+//! and a Node pointer. If this pointer is set, then it must point to another LEAF,
+//! creating a chain of leaf nodes storing row IDs.
+//! This class also contains functionality for nodes of type LEAF_INLINED, in which case we store the
+//! row ID directly in the node pointer.
 class Leaf {
 public:
-	//! Number of row IDs
-	uint32_t count;
-	union {
-		//! The pointer to the head of the list of leaf segments
-		Node ptr;
-		//! Inlined row ID
-		row_t inlined;
-	} row_ids;
+	//! The number of row IDs in this leaf
+	uint8_t count;
+	//! Up to LEAF_SIZE row IDs
+	row_t row_ids[Node::LEAF_SIZE];
+	//! A pointer to the next LEAF node
+	Node ptr;
 
 public:
-	//! Get a new leaf node, might cause a new buffer allocation, and initializes a leaf holding one row ID
-	static Leaf &New(ART &art, Node &node, const row_t row_id);
-	//! Get a new leaf node, might cause a new buffer allocation, and initializes a leaf holding n_row_ids row IDs
-	static Leaf &New(ART &art, Node &node, const row_t *row_ids, const idx_t count);
-	//! Free the leaf
+	//! Inline a row ID into a node pointer
+	static void New(Node &node, const row_t row_id);
+	//! Get a new chain of leaf nodes, might cause new buffer allocations,
+	//! with the node parameter holding the tail of the chain
+	static void New(ART &art, reference<Node> &node, const row_t *row_ids, idx_t count);
+	//! Free the leaf (chain)
 	static void Free(ART &art, Node &node);
 	//! Get a reference to the leaf
 	static inline Leaf &Get(const ART &art, const Node ptr) {
+		D_ASSERT(!ptr.IsSerialized());
 		return *Node::GetAllocator(art, NType::LEAF).Get<Leaf>(ptr);
 	}
 
-	//! Initializes a merge by incrementing the buffer IDs of the leaf segments
-	void InitializeMerge(const ART &art, const idx_t buffer_count);
-	//! Merge leaves
-	void Merge(ART &art, Node &other);
+	//! Initializes a merge by incrementing the buffer IDs of the leaf
+	static void InitializeMerge(ART &art, Node &node, const ARTFlags &flags);
+	//! Merge leaves and free all copied leaf nodes
+	static void Merge(ART &art, Node &l_node, Node &r_node);
 
 	//! Insert a row ID into a leaf
-	void Insert(ART &art, const row_t row_id);
-	//! Remove a row ID from a leaf
-	void Remove(ART &art, const row_t row_id);
+	static void Insert(ART &art, Node &node, const row_t row_id);
+	//! Remove a row ID from a leaf. Returns true, if the leaf is empty after the removal
+	static bool Remove(ART &art, reference<Node> &node, const row_t row_id);
 
-	//! Returns whether this leaf is inlined
-	inline bool IsInlined() const {
-		return count <= 1;
-	}
-	//! Get the row ID at the position
-	row_t GetRowId(const ART &art, const idx_t position) const;
-	//! Returns the position of a row ID, and an invalid index, if the leaf does not contain the row ID,
-	//! and sets the ptr to point to the segment containing the row ID
-	uint32_t FindRowId(const ART &art, Node &ptr, const row_t row_id) const;
+	//! Get the total count of row IDs in the chain of leaves
+	static idx_t TotalCount(ART &art, Node &node);
+	//! Fill the result_ids vector with the row IDs of this leaf chain, if the total count does not exceed max_count
+	static bool GetRowIds(ART &art, Node &node, vector<row_t> &result_ids, idx_t max_count);
+	//! Returns whether the leaf contains the row ID
+	static bool ContainsRowId(ART &art, Node &node, const row_t row_id);
 
-	//! Returns the string representation of the node, or only traverses and verifies the node and its subtree
-	string VerifyAndToString(const ART &art, const bool only_verify) const;
+	//! Returns the string representation of the leaf, or only traverses and verifies the leaf
+	static string VerifyAndToString(ART &art, Node &node);
 
-	//! Serialize this leaf
-	BlockPointer Serialize(const ART &art, MetaBlockWriter &writer) const;
-	//! Deserialize this leaf
-	void Deserialize(ART &art, MetaBlockReader &reader);
+	//! Serialize the leaf
+	static BlockPointer Serialize(ART &art, Node &node, MetaBlockWriter &writer);
+	//! Deserialize the leaf
+	static void Deserialize(ART &art, Node &node, MetaBlockReader &reader);
 
-	//! Vacuum the leaf segments of a leaf, if not inlined
-	void Vacuum(ART &art);
+	//! Vacuum the leaf
+	static void Vacuum(ART &art, Node &node);
 
 private:
-	//! Moves the inlined row ID onto a leaf segment, does not change the size
-	//! so this will be a (temporarily) invalid leaf
-	void MoveInlinedToSegment(ART &art);
+	//! Moves the inlined row ID onto a leaf
+	static void MoveInlinedToLeaf(ART &art, Node &node);
+	//! Appends the row ID to this leaf, or creates a subsequent leaf, if this node is full
+	Leaf &Append(ART &art, const row_t row_id);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp

@@ -1,30 +1,32 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/execution/index/art/art_node.hpp
+// duckdb/execution/index/art/node.hpp
 //
 //
 //===----------------------------------------------------------------------===//
 
 #pragma once
 
-#include "duckdb/execution/index/art/fixed_size_allocator.hpp"
-#include "duckdb/execution/index/art/swizzleable_pointer.hpp"
+#include "duckdb/common/assert.hpp"
+#include "duckdb/common/optional_ptr.hpp"
+#include "duckdb/common/to_string.hpp"
+#include "duckdb/common/typedefs.hpp"
 
 namespace duckdb {
 
 // classes
 enum class NType : uint8_t {
 	PREFIX = 1,
-	LEAF_SEGMENT = 2,
-	LEAF = 3,
-	NODE_4 = 4,
-	NODE_16 = 5,
-	NODE_48 = 6,
-	NODE_256 = 7
+	LEAF = 2,
+	NODE_4 = 3,
+	NODE_16 = 4,
+	NODE_48 = 5,
+	NODE_256 = 6,
+	LEAF_INLINED = 7,
 };
+class FixedSizeAllocator;
 class ART;
-class Node;
 class Prefix;
 class MetaBlockReader;
 class MetaBlockWriter;
@@ -33,10 +35,10 @@ class MetaBlockWriter;
 struct BlockPointer;
 struct ARTFlags;
 
-//! The ARTNode is the swizzleable pointer class of the ART index.
-//! If the ARTNode pointer is not swizzled, then the leftmost byte identifies the NType.
-//! The remaining bytes are the position in the respective ART buffer.
-class Node : public SwizzleablePointer {
+//! The Node is the pointer class of the ART index.
+//! If the node is serialized, then the pointer points to a storage address (and has no type),
+//! otherwise the pointer has a type and stores other information (e.g., a buffer location or a row ID).
+class Node {
 public:
 	//! Node thresholds
 	static constexpr uint8_t NODE_48_SHRINK_THRESHOLD = 12;
@@ -46,42 +48,40 @@ public:
 	static constexpr uint8_t NODE_16_CAPACITY = 16;
 	static constexpr uint8_t NODE_48_CAPACITY = 48;
 	static constexpr uint16_t NODE_256_CAPACITY = 256;
+	//! Bit-shifting
+	static constexpr uint64_t SHIFT_OFFSET = 32;
+	static constexpr uint64_t SHIFT_TYPE = 56;
+	static constexpr uint64_t SHIFT_SERIALIZED_FLAG = 63;
+	//! AND operations
+	static constexpr uint64_t AND_OFFSET = 0x0000000000FFFFFF;
+	static constexpr uint64_t AND_BUFFER_ID = 0x00000000FFFFFFFF;
+	static constexpr uint64_t AND_IS_SET = 0xFF00000000000000;
+	static constexpr uint64_t AND_RESET = 0x00FFFFFFFFFFFFFF;
+	//! OR operations
+	static constexpr uint64_t SET_SERIALIZED_FLAG = 0x8000000000000000;
 	//! Other constants
 	static constexpr uint8_t EMPTY_MARKER = 48;
-	static constexpr uint32_t LEAF_SEGMENT_SIZE = 8;
+	static constexpr uint8_t LEAF_SIZE = 4;
 	static constexpr uint8_t PREFIX_SIZE = 15;
 
 public:
-	//! Constructs an empty ARTNode
-	Node();
-	//! Constructs a swizzled pointer from a block ID and an offset
+	//! Constructors
+
+	//! Constructs an empty Node
+	Node() : data(0) {};
+	//! Constructs a serialized Node pointer from a block ID and an offset
 	explicit Node(MetaBlockReader &reader);
+	//! Constructs an in-memory Node from a buffer ID and an offset
+	Node(const uint32_t buffer_id, const uint32_t offset) : data(0) {
+		SetPtr(buffer_id, offset);
+	};
+
+public:
 	//! Get a new pointer to a node, might cause a new buffer allocation, and initialize it
 	static void New(ART &art, Node &node, const NType type);
 	//! Free the node (and its subtree)
 	static void Free(ART &art, Node &node);
 
-	inline bool operator==(const Node &node) const {
-		return swizzle_flag == node.swizzle_flag && type == node.type && offset == node.offset &&
-		       buffer_id == node.buffer_id;
-	}
-
-	//! Retrieve the node type from the leftmost byte
-	inline NType DecodeARTNodeType() const {
-		D_ASSERT(!IsSwizzled());
-		D_ASSERT(type >= (uint8_t)NType::PREFIX);
-		D_ASSERT(type <= (uint8_t)NType::NODE_256);
-		return NType(type);
-	}
-
-	//! Set the pointer
-	inline void SetPtr(const SwizzleablePointer ptr) {
-		swizzle_flag = ptr.swizzle_flag;
-		type = ptr.type;
-		offset = ptr.offset;
-		buffer_id = ptr.buffer_id;
-	}
-
 	//! Replace the child node at the respective byte
 	void ReplaceChild(const ART &art, const uint8_t byte, const Node child);
 	//! Insert the child node at byte
@@ -118,7 +118,86 @@ public:
 	bool MergeInternal(ART &art, Node &other);
 
 	//! Vacuum all nodes that exceed their respective vacuum thresholds
-	static void Vacuum(ART &art, Node &node, const ARTFlags &flags);
+	void Vacuum(ART &art, const ARTFlags &flags);
+
+	// Getters and Setters
+
+	//! Returns whether the node is serialized or not (zero bit)
+	inline bool IsSerialized() const {
+		return data >> Node::SHIFT_SERIALIZED_FLAG;
+	}
+	//! Get the type (1st to 7th bit)
+	inline NType GetType() const {
+		D_ASSERT(!IsSerialized());
+		auto type = data >> Node::SHIFT_TYPE;
+		D_ASSERT(type >= (uint8_t)NType::PREFIX);
+		D_ASSERT(type <= (uint8_t)NType::LEAF_INLINED);
+		return NType(type);
+	}
+	//! Get the offset (8th to 23rd bit)
+	inline idx_t GetOffset() const {
+		auto offset = data >> Node::SHIFT_OFFSET;
+		return offset & Node::AND_OFFSET;
+	}
+	//! Get the block/buffer ID (24th to 63rd bit)
+	inline idx_t GetBufferId() const {
+		return data & Node::AND_BUFFER_ID;
+	}
+	//! Get the row ID (8th to 63rd bit)
+	inline row_t GetRowId() const {
+		return data & Node::AND_RESET;
+	}
+
+	//! Set the serialized flag (zero bit)
+	inline void SetSerialized() {
+		data &= Node::AND_RESET;
+		data |= Node::SET_SERIALIZED_FLAG;
+	}
+	//! Set the type (1st to 7th bit)
+	inline void SetType(const uint8_t type) {
+		D_ASSERT(!IsSerialized());
+		data += (uint64_t)type << Node::SHIFT_TYPE;
+	}
+	//! Set the block/buffer ID (24th to 63rd bit) and offset (8th to 23rd bit)
+	inline void SetPtr(const uint32_t buffer_id, const uint32_t offset) {
+		D_ASSERT(!(data & Node::AND_RESET));
+		auto shifted_offset = ((uint64_t)offset) << Node::SHIFT_OFFSET;
+		data += shifted_offset;
+		data += buffer_id;
+	}
+	//! Set the row ID (8th to 63rd bit)
+	inline void SetRowId(const row_t row_id) {
+		D_ASSERT(!(data & Node::AND_RESET));
+		data += row_id;
+	}
+
+	//! Returns true, if neither the serialized flag is set nor the type
+	inline bool IsSet() const {
+		return data & Node::AND_IS_SET;
+	}
+	//! Reset the Node pointer by setting the node info to zero
+	inline void Reset() {
+		data = 0;
+	}
+
+	//! Comparison operator
+	inline bool operator==(const Node &node) const {
+		return data == node.data;
+	}
+
+private:
+	//! Data holds all the information contained in a Node pointer
+	//! [0: serialized flag, 1 - 7: type,
+	//! 8 - 23: offset, 24 - 63: buffer/block ID OR
+	//! 8 - 63: row ID]
+	//! NOTE: a Node pointer can be either serialized OR have a type
+	//! NOTE: we do not use bit fields because when using bit fields Windows compiles
+	//! the Node class into 16 bytes instead of the intended 8 bytes, doubling the
+	//! space requirements
+	//! https://learn.microsoft.com/en-us/cpp/cpp/cpp-bit-fields?view=msvc-170
+	uint64_t data;
 };
 
+static_assert(sizeof(Node) == sizeof(uint64_t), "Invalid size for Node type.");
+
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/index/art/node16.hpp

@@ -31,6 +31,7 @@ public:
 	static void Free(ART &art, Node &node);
 	//! Get a reference to the node
 	static inline Node16 &Get(const ART &art, const Node ptr) {
+		D_ASSERT(!ptr.IsSerialized());
 		return *Node::GetAllocator(art, NType::NODE_16).Get<Node16>(ptr);
 	}
 	//! Initializes all the fields of the node while growing a Node4 to a Node16

package/src/duckdb/src/include/duckdb/execution/index/art/node256.hpp

@@ -29,6 +29,7 @@ public:
 	static void Free(ART &art, Node &node);
 	//! Get a reference to the node
 	static inline Node256 &Get(const ART &art, const Node ptr) {
+		D_ASSERT(!ptr.IsSerialized());
 		return *Node::GetAllocator(art, NType::NODE_256).Get<Node256>(ptr);
 	}
 	//! Initializes all the fields of the node while growing a Node48 to a Node256

package/src/duckdb/src/include/duckdb/execution/index/art/node4.hpp

@@ -31,6 +31,7 @@ public:
 	static void Free(ART &art, Node &node);
 	//! Get a reference to the node
 	static inline Node4 &Get(const ART &art, const Node ptr) {
+		D_ASSERT(!ptr.IsSerialized());
 		return *Node::GetAllocator(art, NType::NODE_4).Get<Node4>(ptr);
 	}
 	//! Initializes all fields of the node while shrinking a Node16 to a Node4

package/src/duckdb/src/include/duckdb/execution/index/art/node48.hpp

@@ -32,6 +32,7 @@ public:
 	static void Free(ART &art, Node &node);
 	//! Get a reference to the node
 	static inline Node48 &Get(const ART &art, const Node ptr) {
+		D_ASSERT(!ptr.IsSerialized());
 		return *Node::GetAllocator(art, NType::NODE_48).Get<Node48>(ptr);
 	}
 	//! Initializes all the fields of the node while growing a Node16 to a Node48

package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp

@@ -8,6 +8,7 @@
 #pragma once
 
 #include "duckdb/execution/index/art/art.hpp"
+#include "duckdb/execution/index/art/fixed_size_allocator.hpp"
 #include "duckdb/execution/index/art/node.hpp"
 
 namespace duckdb {
@@ -37,6 +38,7 @@ public:
 	static void Free(ART &art, Node &node);
 	//! Get a reference to the prefix
 	static inline Prefix &Get(const ART &art, const Node ptr) {
+		D_ASSERT(!ptr.IsSerialized());
 		return *Node::GetAllocator(art, NType::PREFIX).Get<Prefix>(ptr);
 	}
 
@@ -71,7 +73,7 @@ public:
 	static void Split(ART &art, reference<Node> &prefix_node, Node &child_node, idx_t position);
 
 	//! Returns the string representation of the node, or only traverses and verifies the node and its subtree
-	string VerifyAndToString(ART &art, const bool only_verify);
+	static string VerifyAndToString(ART &art, Node &node, const bool only_verify);
 
 	//! Serialize this node
 	BlockPointer Serialize(ART &art, MetaBlockWriter &writer);
@@ -80,7 +82,7 @@ public:
 
 	//! Vacuum the child of the node
 	inline void Vacuum(ART &art, const ARTFlags &flags) {
-		Node::Vacuum(art, ptr, flags);
+		ptr.Vacuum(art, flags);
 	}
 
 private: