duckdb 0.7.2-dev2507.0 → 0.7.2-dev2675.0

package/src/duckdb/src/execution/index/art/prefix.cpp

@@ -1,191 +1,469 @@
 #include "duckdb/execution/index/art/prefix.hpp"
 
 #include "duckdb/execution/index/art/art.hpp"
+#include "duckdb/execution/index/art/art_key.hpp"
+#include "duckdb/execution/index/art/node.hpp"
+#include "duckdb/execution/index/art/prefix_segment.hpp"
+#include "duckdb/storage/meta_block_reader.hpp"
+#include "duckdb/storage/meta_block_writer.hpp"
 
 namespace duckdb {
 
-bool Prefix::IsInlined() const {
-	return size <= PREFIX_INLINE_BYTES;
+void Prefix::Free(ART &art) {
+
+	if (IsInlined()) {
+		return Initialize();
+	}
+
+	// delete all prefix segments
+	auto ptr = data.ptr;
+	while (ptr.IsSet()) {
+		auto next_ptr = PrefixSegment::Get(art, ptr).next;
+		Node::Free(art, ptr);
+		ptr = next_ptr;
+	}
+
+	Initialize();
 }
 
-uint8_t *Prefix::GetPrefixData() {
-	return IsInlined() ? &value.inlined[0] : value.ptr;
+void Prefix::Initialize(ART &art, const ARTKey &key, const uint32_t depth, const uint32_t count_p) {
+
+	// prefix can be inlined
+	if (count_p <= Node::PREFIX_INLINE_BYTES) {
+		memcpy(data.inlined, key.data + depth, count_p);
+		count = count_p;
+		return;
+	}
+
+	// prefix cannot be inlined, copy to segment(s)
+	count = 0;
+	reference<PrefixSegment> segment(PrefixSegment::New(art, data.ptr));
+	for (idx_t i = 0; i < count_p; i++) {
+		segment = segment.get().Append(art, count, key.data[depth + i]);
+	}
+	D_ASSERT(count == count_p);
 }
 
-const uint8_t *Prefix::GetPrefixData() const {
-	return IsInlined() ? &value.inlined[0] : value.ptr;
+void Prefix::Initialize(ART &art, const Prefix &other, const uint32_t count_p) {
+
+	D_ASSERT(count_p <= other.count);
+
+	// copy inlined data
+	if (other.IsInlined()) {
+		memcpy(data.inlined, other.data.inlined, count_p);
+		count = count_p;
+		return;
+	}
+
+	// initialize the count and get the first segment
+	count = 0;
+	reference<PrefixSegment> segment(PrefixSegment::New(art, data.ptr));
+
+	// iterate the segments of the other prefix and copy their data
+	auto other_ptr = other.data.ptr;
+	auto remaining = count_p;
+
+	while (remaining != 0) {
+		D_ASSERT(other_ptr.IsSet());
+		auto &other_segment = PrefixSegment::Get(art, other_ptr);
+		auto copy_count = MinValue(Node::PREFIX_SEGMENT_SIZE, remaining);
+
+		// copy the data
+		for (idx_t i = 0; i < copy_count; i++) {
+			segment = segment.get().Append(art, count, other_segment.bytes[i]);
+		}
+
+		// adjust the loop variables
+		other_ptr = other_segment.next;
+		remaining -= copy_count;
+	}
+	D_ASSERT(count == count_p);
 }
 
-uint8_t *Prefix::AllocatePrefix(uint32_t size) {
-	Destroy();
+void Prefix::InitializeMerge(ART &art, const idx_t buffer_count) {
 
-	this->size = size;
-	uint8_t *prefix;
 	if (IsInlined()) {
-		prefix = &value.inlined[0];
-	} else {
-		// allocate new prefix
-		value.ptr = AllocateArray<uint8_t>(size);
-		prefix = value.ptr;
+		return;
 	}
-	return prefix;
-}
 
-Prefix::Prefix() : size(0) {
+	reference<PrefixSegment> segment(PrefixSegment::Get(art, data.ptr));
+	data.ptr.buffer_id += buffer_count;
+
+	auto ptr = segment.get().next;
+	while (ptr.IsSet()) {
+		segment.get().next.buffer_id += buffer_count;
+		segment = PrefixSegment::Get(art, ptr);
+		ptr = segment.get().next;
+	}
 }
 
-Prefix::Prefix(Key &key, uint32_t depth, uint32_t size) : size(0) {
-	auto prefix = AllocatePrefix(size);
+void Prefix::Append(ART &art, const Prefix &other) {
 
-	// copy key to prefix
-	idx_t prefix_idx = 0;
-	for (idx_t i = depth; i < size + depth; i++) {
-		prefix[prefix_idx++] = key.data[i];
+	// result fits into inlined data, i.e., both prefixes are also inlined
+	if (count + other.count <= Node::PREFIX_INLINE_BYTES) {
+		memcpy(data.inlined + count, other.data.inlined, other.count);
+		count += other.count;
+		return;
 	}
-}
 
-Prefix::Prefix(Prefix &other_prefix, uint32_t size) : size(0) {
-	auto prefix = AllocatePrefix(size);
+	// this prefix is inlined, but will no longer be after appending the other prefix,
+	// move the inlined bytes to the first prefix segment
+	if (IsInlined()) {
+		MoveInlinedToSegment(art);
+	}
 
-	// copy key to Prefix
-	auto other_data = other_prefix.GetPrefixData();
-	for (idx_t i = 0; i < size; i++) {
-		prefix[i] = other_data[i];
+	// get the tail of the segments of this prefix
+	reference<PrefixSegment> segment(PrefixSegment::Get(art, data.ptr).GetTail(art));
+
+	// the other prefix is inlined
+	if (other.IsInlined()) {
+		for (idx_t i = 0; i < other.count; i++) {
+			segment = segment.get().Append(art, count, other.data.inlined[i]);
+		}
+		return;
 	}
-}
 
-Prefix::~Prefix() {
-	Destroy();
-}
+	// iterate all segments of the other prefix and copy their data
+	auto other_ptr = other.data.ptr;
+	auto remaining = other.count;
 
-idx_t Prefix::MemorySize() {
-	return sizeof(*this) + sizeof(uint8_t) * size;
-}
+	while (other_ptr.IsSet()) {
+		auto &other_segment = PrefixSegment::Get(art, other_ptr);
+		auto copy_count = MinValue(Node::PREFIX_SEGMENT_SIZE, remaining);
+
+		// copy the data
+		for (idx_t i = 0; i < copy_count; i++) {
+			segment = segment.get().Append(art, count, other_segment.bytes[i]);
+		}
 
-void Prefix::Destroy() {
-	if (!IsInlined()) {
-		DeleteArray<uint8_t>(value.ptr, size);
-		size = 0;
+		// adjust the loop variables
+		other_ptr = other_segment.next;
+		remaining -= copy_count;
 	}
+	D_ASSERT(remaining == 0);
 }
 
-uint8_t &Prefix::operator[](idx_t idx) {
-	D_ASSERT(idx < Size());
-	return GetPrefixData()[idx];
+void Prefix::Concatenate(ART &art, const uint8_t byte, const Prefix &other) {
+
+	auto new_size = count + 1 + other.count;
+
+	// overwrite into this prefix (both are inlined)
+	if (new_size <= Node::PREFIX_INLINE_BYTES) {
+		// move this prefix backwards
+		memmove(data.inlined + other.count + 1, data.inlined, count);
+		// copy byte
+		data.inlined[other.count] = byte;
+		// copy the other prefix into this prefix
+		memcpy(data.inlined, other.data.inlined, other.count);
+		count = new_size;
+		return;
+	}
+
+	auto this_count = count;
+	auto this_data = data;
+	Initialize();
+
+	// append the other prefix
+	Append(art, other);
+
+	if (IsInlined()) {
+		// move to a segment
+		reference<PrefixSegment> segment(MoveInlinedToSegment(art));
+		// append the byte
+		segment = segment.get().Append(art, count, byte);
+		// append this prefix
+		for (idx_t i = 0; i < this_count; i++) {
+			segment = segment.get().Append(art, count, this_data.inlined[i]);
+		}
+		return;
+	}
+
+	// get the tail
+	reference<PrefixSegment> segment(PrefixSegment::Get(art, data.ptr).GetTail(art));
+	// append the byte
+	segment = segment.get().Append(art, count, byte);
+
+	// iterate all segments of this prefix, copy their data, and free them
+	auto this_ptr = this_data.ptr;
+	auto remaining = this_count;
+
+	while (this_ptr.IsSet()) {
+		auto &this_segment = PrefixSegment::Get(art, this_ptr);
+		auto copy_count = MinValue(Node::PREFIX_SEGMENT_SIZE, remaining);
+
+		// copy the data
+		for (idx_t i = 0; i < copy_count; i++) {
+			segment = segment.get().Append(art, count, this_segment.bytes[i]);
+		}
+
+		// adjust the loop variables
+		Node::Free(art, this_ptr);
+		this_ptr = this_segment.next;
+		remaining -= copy_count;
+	}
+	D_ASSERT(remaining == 0);
 }
 
-Prefix &Prefix::operator=(const Prefix &src) {
-	auto prefix = AllocatePrefix(src.size);
+uint8_t Prefix::Reduce(ART &art, const idx_t reduce_count) {
+
+	auto new_count = count - reduce_count - 1;
+	auto new_first_byte = GetByte(art, reduce_count);
+
+	// prefix is now empty
+	if (new_count == 0) {
+		Free(art);
+		return new_first_byte;
+	}
+
+	// was inlined, just move bytes
+	if (IsInlined()) {
+		memmove(data.inlined, data.inlined + reduce_count + 1, new_count);
+		count = new_count;
+		return new_first_byte;
+	}
+
+	count = 0;
+	auto start = reduce_count + 1;
+	auto offset = start % Node::PREFIX_SEGMENT_SIZE;
+	auto remaining = new_count;
+
+	// get the source segment, i.e., the segment that contains the byte at start
+	reference<PrefixSegment> src_segment(PrefixSegment::Get(art, data.ptr));
+	for (idx_t i = 0; i < start / Node::PREFIX_SEGMENT_SIZE; i++) {
+		D_ASSERT(src_segment.get().next.IsSet());
+		src_segment = PrefixSegment::Get(art, src_segment.get().next);
+	}
+
+	// iterate all segments starting at the source segment and shift their data
+	reference<PrefixSegment> dst_segment(PrefixSegment::Get(art, data.ptr));
+	while (true) {
+		auto copy_count = MinValue(Node::PREFIX_SEGMENT_SIZE - offset, remaining);
+
+		// copy the data
+		for (idx_t i = offset; i < offset + copy_count; i++) {
+			dst_segment = dst_segment.get().Append(art, count, src_segment.get().bytes[i]);
+		}
 
-	// copy prefix
-	auto src_prefix = src.GetPrefixData();
-	for (idx_t i = 0; i < src.size; i++) {
-		prefix[i] = src_prefix[i];
+		// adjust the loop variables
+		offset = 0;
+		remaining -= copy_count;
+		if (remaining == 0) {
+			break;
+		}
+		D_ASSERT(src_segment.get().next.IsSet());
+		src_segment = PrefixSegment::Get(art, src_segment.get().next);
+	}
+
+	// possibly inline the data
+	if (IsInlined()) {
+		MoveSegmentToInlined(art);
 	}
-	size = src.size;
-	return *this;
+
+	return new_first_byte;
 }
 
-Prefix &Prefix::operator=(Prefix &&other) noexcept {
-	std::swap(size, other.size);
-	std::swap(value, other.value);
-	return *this;
+uint8_t Prefix::GetByte(const ART &art, const idx_t position) const {
+
+	D_ASSERT(position < count);
+	if (IsInlined()) {
+		return data.inlined[position];
+	}
+
+	// get the correct segment
+	reference<PrefixSegment> segment(PrefixSegment::Get(art, data.ptr));
+	for (idx_t i = 0; i < position / Node::PREFIX_SEGMENT_SIZE; i++) {
+		D_ASSERT(segment.get().next.IsSet());
+		segment = PrefixSegment::Get(art, segment.get().next);
+	}
+
+	return segment.get().bytes[position % Node::PREFIX_SEGMENT_SIZE];
 }
 
-void Prefix::Overwrite(uint32_t new_size, uint8_t *data) {
-	if (new_size <= PREFIX_INLINE_BYTES) {
-		// new entry would be inlined
-		// inline the data and destroy the pointer
-		auto prefix = AllocatePrefix(new_size);
-		for (idx_t i = 0; i < new_size; i++) {
-			prefix[i] = data[i];
+uint32_t Prefix::KeyMismatchPosition(const ART &art, const ARTKey &key, const uint32_t depth) const {
+
+	if (IsInlined()) {
+		for (idx_t mismatch_position = 0; mismatch_position < count; mismatch_position++) {
+			D_ASSERT(depth + mismatch_position < key.len);
+			if (key[depth + mismatch_position] != data.inlined[mismatch_position]) {
+				return mismatch_position;
+			}
+		}
+		return count;
+	}
+
+	uint32_t mismatch_position = 0;
+	auto ptr = data.ptr;
+
+	while (mismatch_position != count) {
+		D_ASSERT(depth + mismatch_position < key.len);
+		D_ASSERT(ptr.IsSet());
+
+		auto &segment = PrefixSegment::Get(art, ptr);
+		auto compare_count = MinValue(Node::PREFIX_SEGMENT_SIZE, count - mismatch_position);
+
+		// compare bytes
+		for (uint32_t i = 0; i < compare_count; i++) {
+			if (key[depth + mismatch_position] != segment.bytes[i]) {
+				return mismatch_position;
+			}
+			mismatch_position++;
 		}
-		DeleteArray<uint8_t>(data, new_size);
 
-	} else {
-		// new entry would not be inlined
-		// take over the data directly
-		Destroy();
-		size = new_size;
-		value.ptr = data;
+		// adjust loop variables
+		ptr = segment.next;
 	}
+	return count;
 }
 
-void Prefix::Concatenate(ART &art, uint8_t key, Prefix &other) {
-	auto new_size = size + 1 + other.size;
-	art.IncreaseMemorySize((new_size - size) * sizeof(uint8_t));
-	// have to allocate space in our prefix array
-	auto new_prefix = AllocateArray<uint8_t>(new_size);
-	idx_t new_prefix_idx = 0;
+uint32_t Prefix::MismatchPosition(const ART &art, const Prefix &other) const {
+
+	D_ASSERT(count <= other.count);
 
-	// 1) add the to-be deleted node's prefix
-	for (uint32_t i = 0; i < other.size; i++) {
-		new_prefix[new_prefix_idx++] = other[i];
+	// case 1: both prefixes are inlined
+	if (IsInlined() && other.IsInlined()) {
+		for (uint32_t i = 0; i < count; i++) {
+			if (data.inlined[i] != other.data.inlined[i]) {
+				return i;
+			}
+		}
+		return count;
 	}
 
-	// 2) now move the current partial key byte as part of the prefix
-	new_prefix[new_prefix_idx++] = key;
+	// case 2: only this prefix is inlined
+	if (IsInlined()) {
+		// we only need the first segment of the other prefix
+		auto &segment = PrefixSegment::Get(art, other.data.ptr);
+		for (uint32_t i = 0; i < count; i++) {
+			if (data.inlined[i] != segment.bytes[i]) {
+				return i;
+			}
+		}
+		return count;
+	}
 
-	// 3) move the existing prefix (if any)
-	auto prefix = GetPrefixData();
-	for (uint32_t i = 0; i < size; i++) {
-		new_prefix[new_prefix_idx++] = prefix[i];
+	// case 3: both prefixes are not inlined
+	auto ptr = data.ptr;
+	auto other_ptr = other.data.ptr;
+
+	// iterate segments and compare bytes
+	uint32_t mismatch_position = 0;
+	while (ptr.IsSet()) {
+		D_ASSERT(other_ptr.IsSet());
+		auto &segment = PrefixSegment::Get(art, ptr);
+		auto &other_segment = PrefixSegment::Get(art, other_ptr);
+
+		// compare bytes
+		auto compare_count = MinValue(Node::PREFIX_SEGMENT_SIZE, count - mismatch_position);
+		for (uint32_t i = 0; i < compare_count; i++) {
+			if (segment.bytes[i] != other_segment.bytes[i]) {
+				return mismatch_position;
+			}
+			mismatch_position++;
+		}
+
+		// adjust loop variables
+		ptr = segment.next;
+		other_ptr = other_segment.next;
 	}
-	Overwrite(new_size, new_prefix);
+	return count;
 }
 
-uint8_t Prefix::Reduce(ART &art, uint32_t n) {
-	auto new_size = size - n - 1;
-	art.DecreaseMemorySize((size - new_size) * sizeof(uint8_t));
-	auto prefix = GetPrefixData();
-	auto partial_key = prefix[n];
+void Prefix::Serialize(const ART &art, MetaBlockWriter &writer) const {
+
+	writer.Write(count);
 
-	if (new_size == 0) {
-		Destroy();
-		size = 0;
-		return partial_key;
+	// write inlined data
+	if (IsInlined()) {
+		writer.WriteData(data.inlined, count);
+		return;
 	}
-	auto new_prefix = AllocateArray<uint8_t>(new_size);
-	for (idx_t i = 0; i < new_size; i++) {
-		new_prefix[i] = prefix[i + n + 1];
+
+	D_ASSERT(data.ptr.IsSet());
+	auto ptr = data.ptr;
+	auto remaining = count;
+
+	// iterate all prefix segments and write their bytes
+	while (ptr.IsSet()) {
+		auto &segment = PrefixSegment::Get(art, ptr);
+		auto copy_count = MinValue(Node::PREFIX_SEGMENT_SIZE, remaining);
+
+		// write the bytes
+		writer.WriteData(segment.bytes, copy_count);
+
+		// adjust loop variables
+		remaining -= copy_count;
+		ptr = segment.next;
 	}
-	Overwrite(new_size, new_prefix);
-	return partial_key;
+	D_ASSERT(remaining == 0);
 }
 
-void Prefix::Serialize(duckdb::MetaBlockWriter &writer) {
-	writer.Write(size);
-	auto prefix = GetPrefixData();
-	writer.WriteData(prefix, size);
-}
+void Prefix::Deserialize(ART &art, MetaBlockReader &reader) {
 
-void Prefix::Deserialize(duckdb::MetaBlockReader &reader) {
-	auto prefix_size = reader.Read<uint32_t>();
-	auto prefix = AllocatePrefix(prefix_size);
-	this->size = prefix_size;
-	reader.ReadData(prefix, size);
-}
+	auto count_p = reader.Read<uint32_t>();
 
-uint32_t Prefix::KeyMismatchPosition(Key &key, uint32_t depth) {
-	uint64_t pos;
-	auto prefix = GetPrefixData();
-	for (pos = 0; pos < size; pos++) {
-		if (key[depth + pos] != prefix[pos]) {
-			return pos;
-		}
+	// copy into inlined data
+	if (count_p <= Node::PREFIX_INLINE_BYTES) {
+		reader.ReadData(data.inlined, count_p);
+		count = count_p;
+		return;
 	}
-	return pos;
+
+	// copy into segments
+	count = 0;
+	reference<PrefixSegment> segment(PrefixSegment::New(art, data.ptr));
+	for (idx_t i = 0; i < count_p; i++) {
+		segment = segment.get().Append(art, count, reader.Read<uint8_t>());
+	}
+	D_ASSERT(count_p == count);
 }
 
-uint32_t Prefix::MismatchPosition(Prefix &other) {
-	auto prefix = GetPrefixData();
-	auto other_data = other.GetPrefixData();
-	for (idx_t i = 0; i < size; i++) {
-		if (prefix[i] != other_data[i]) {
-			return i;
+void Prefix::Vacuum(ART &art) {
+
+	if (IsInlined()) {
+		return;
+	}
+
+	// first pointer has special treatment because we don't obtain it from a prefix segment
+	auto &allocator = Node::GetAllocator(art, NType::PREFIX_SEGMENT);
+	if (allocator.NeedsVacuum(data.ptr)) {
+		data.ptr.SetPtr(allocator.VacuumPointer(data.ptr));
+	}
+
+	auto ptr = data.ptr;
+	while (ptr.IsSet()) {
+		auto &segment = PrefixSegment::Get(art, ptr);
+		ptr = segment.next;
+		if (ptr.IsSet() && allocator.NeedsVacuum(ptr)) {
+			segment.next.SetPtr(allocator.VacuumPointer(ptr));
+			ptr = segment.next;
 		}
 	}
-	return size;
+}
+
+PrefixSegment &Prefix::MoveInlinedToSegment(ART &art) {
+
+	D_ASSERT(IsInlined());
+
+	Node ptr;
+	auto &segment = PrefixSegment::New(art, ptr);
+
+	// move data
+	D_ASSERT(Node::PREFIX_SEGMENT_SIZE >= Node::PREFIX_INLINE_BYTES);
+	memcpy(segment.bytes, data.inlined, count);
+	data.ptr = ptr;
+	return segment;
+}
+
+void Prefix::MoveSegmentToInlined(ART &art) {
+
+	D_ASSERT(IsInlined());
+	D_ASSERT(data.ptr.IsSet());
+
+	auto ptr = data.ptr;
+	auto &segment = PrefixSegment::Get(art, data.ptr);
+
+	memcpy(data.inlined, segment.bytes, count);
+	Node::Free(art, ptr);
 }
 
 } // namespace duckdb

package/src/duckdb/src/execution/index/art/prefix_segment.cpp

@@ -0,0 +1,42 @@
+#include "duckdb/execution/index/art/prefix_segment.hpp"
+
+#include "duckdb/execution/index/art/art.hpp"
+#include "duckdb/execution/index/art/node.hpp"
+
+namespace duckdb {
+
+PrefixSegment &PrefixSegment::New(ART &art, Node &node) {
+
+	node.SetPtr(Node::GetAllocator(art, NType::PREFIX_SEGMENT).New());
+	node.type = (uint8_t)NType::PREFIX_SEGMENT;
+
+	auto &segment = PrefixSegment::Get(art, node);
+	segment.next.Reset();
+	return segment;
+}
+
+PrefixSegment &PrefixSegment::Append(ART &art, uint32_t &count, const uint8_t byte) {
+
+	reference<PrefixSegment> segment(*this);
+	auto position = count % Node::PREFIX_SEGMENT_SIZE;
+
+	// we need a new segment
+	if (position == 0 && count != 0) {
+		segment = PrefixSegment::New(art, next);
+	}
+
+	segment.get().bytes[position] = byte;
+	count++;
+	return segment.get();
+}
+
+PrefixSegment &PrefixSegment::GetTail(const ART &art) {
+
+	reference<PrefixSegment> segment(*this);
+	while (segment.get().next.IsSet()) {
+		segment = PrefixSegment::Get(art, segment.get().next);
+	}
+	return segment.get();
+}
+
+} // namespace duckdb