duckdb 0.7.2-dev2552.0 → 0.7.2-dev2699.0

package/src/duckdb/src/execution/index/art/art_key.cpp

@@ -1,29 +1,27 @@
 #include "duckdb/execution/index/art/art_key.hpp"
 
-#include "duckdb/execution/index/art/art.hpp"
-
 namespace duckdb {
 
-Key::Key() : len(0) {
+ARTKey::ARTKey() : len(0) {
 }
 
-Key::Key(data_ptr_t data, idx_t len) : len(len), data(data) {
+ARTKey::ARTKey(const data_ptr_t &data, const uint32_t &len) : len(len), data(data) {
 }
 
-Key::Key(ArenaAllocator &allocator, idx_t len) : len(len) {
+ARTKey::ARTKey(ArenaAllocator &allocator, const uint32_t &len) : len(len) {
 	data = allocator.Allocate(len);
 }
 
 template <>
-Key Key::CreateKey(ArenaAllocator &allocator, const LogicalType &type, string_t value) {
-	idx_t len = value.GetSize() + 1;
+ARTKey ARTKey::CreateARTKey(ArenaAllocator &allocator, const LogicalType &type, string_t value) {
+	uint32_t len = value.GetSize() + 1;
 	auto data = allocator.Allocate(len);
 	memcpy(data, value.GetData(), len - 1);
 
 	// FIXME: rethink this
 	if (type == LogicalType::BLOB || type == LogicalType::VARCHAR) {
 		// indexes cannot contain BLOBs (or BLOBs cast to VARCHARs) that contain null-terminated bytes
-		for (idx_t i = 0; i < len - 1; i++) {
+		for (uint32_t i = 0; i < len - 1; i++) {
 			if (data[i] == '\0') {
 				throw NotImplementedException("Indexes cannot contain BLOBs that contain null-terminated bytes.");
 			}
@@ -31,16 +29,16 @@ Key Key::CreateKey(ArenaAllocator &allocator, const LogicalType &type, string_t
 	}
 
 	data[len - 1] = '\0';
-	return Key(data, len);
+	return ARTKey(data, len);
 }
 
 template <>
-Key Key::CreateKey(ArenaAllocator &allocator, const LogicalType &type, const char *value) {
-	return Key::CreateKey(allocator, type, string_t(value, strlen(value)));
+ARTKey ARTKey::CreateARTKey(ArenaAllocator &allocator, const LogicalType &type, const char *value) {
+	return ARTKey::CreateARTKey(allocator, type, string_t(value, strlen(value)));
 }
 
 template <>
-void Key::CreateKey(ArenaAllocator &allocator, const LogicalType &type, Key &key, string_t value) {
+void ARTKey::CreateARTKey(ArenaAllocator &allocator, const LogicalType &type, ARTKey &key, string_t value) {
 	key.len = value.GetSize() + 1;
 	key.data = allocator.Allocate(key.len);
 	memcpy(key.data, value.GetData(), key.len - 1);
@@ -48,7 +46,7 @@ void Key::CreateKey(ArenaAllocator &allocator, const LogicalType &type, Key &key
 	// FIXME: rethink this
 	if (type == LogicalType::BLOB || type == LogicalType::VARCHAR) {
 		// indexes cannot contain BLOBs (or BLOBs cast to VARCHARs) that contain null-terminated bytes
-		for (idx_t i = 0; i < key.len - 1; i++) {
+		for (uint32_t i = 0; i < key.len - 1; i++) {
 			if (key.data[i] == '\0') {
 				throw NotImplementedException("Indexes cannot contain BLOBs that contain null-terminated bytes.");
 			}
@@ -59,12 +57,12 @@ void Key::CreateKey(ArenaAllocator &allocator, const LogicalType &type, Key &key
 }
 
 template <>
-void Key::CreateKey(ArenaAllocator &allocator, const LogicalType &type, Key &key, const char *value) {
-	Key::CreateKey(allocator, type, key, string_t(value, strlen(value)));
+void ARTKey::CreateARTKey(ArenaAllocator &allocator, const LogicalType &type, ARTKey &key, const char *value) {
+	ARTKey::CreateARTKey(allocator, type, key, string_t(value, strlen(value)));
 }
 
-bool Key::operator>(const Key &k) const {
-	for (idx_t i = 0; i < MinValue<idx_t>(len, k.len); i++) {
+bool ARTKey::operator>(const ARTKey &k) const {
+	for (uint32_t i = 0; i < MinValue<uint32_t>(len, k.len); i++) {
 		if (data[i] > k.data[i]) {
 			return true;
 		} else if (data[i] < k.data[i]) {
@@ -74,8 +72,8 @@ bool Key::operator>(const Key &k) const {
 	return len > k.len;
 }
 
-bool Key::operator<(const Key &k) const {
-	for (idx_t i = 0; i < MinValue<idx_t>(len, k.len); i++) {
+bool ARTKey::operator<(const ARTKey &k) const {
+	for (uint32_t i = 0; i < MinValue<uint32_t>(len, k.len); i++) {
 		if (data[i] < k.data[i]) {
 			return true;
 		} else if (data[i] > k.data[i]) {
@@ -85,8 +83,8 @@ bool Key::operator<(const Key &k) const {
 	return len < k.len;
 }
 
-bool Key::operator>=(const Key &k) const {
-	for (idx_t i = 0; i < MinValue<idx_t>(len, k.len); i++) {
+bool ARTKey::operator>=(const ARTKey &k) const {
+	for (uint32_t i = 0; i < MinValue<uint32_t>(len, k.len); i++) {
 		if (data[i] > k.data[i]) {
 			return true;
 		} else if (data[i] < k.data[i]) {
@@ -96,11 +94,11 @@ bool Key::operator>=(const Key &k) const {
 	return len >= k.len;
 }
 
-bool Key::operator==(const Key &k) const {
+bool ARTKey::operator==(const ARTKey &k) const {
 	if (len != k.len) {
 		return false;
 	}
-	for (idx_t i = 0; i < len; i++) {
+	for (uint32_t i = 0; i < len; i++) {
 		if (data[i] != k.data[i]) {
 			return false;
 		}
@@ -108,15 +106,7 @@ bool Key::operator==(const Key &k) const {
 	return true;
 }
 
-bool Key::ByteMatches(Key &other, idx_t &depth) {
-	return data[depth] == other[depth];
-}
-
-bool Key::Empty() {
-	return len == 0;
-}
-
-void Key::ConcatenateKey(ArenaAllocator &allocator, Key &other_key) {
+void ARTKey::ConcatenateARTKey(ArenaAllocator &allocator, ARTKey &other_key) {
 
 	auto compound_data = allocator.Allocate(len + other_key.len);
 	memcpy(compound_data, data, len);

package/src/duckdb/src/execution/index/art/fixed_size_allocator.cpp

@@ -0,0 +1,224 @@
+#include "duckdb/execution/index/art/fixed_size_allocator.hpp"
+
+#include "duckdb/common/allocator.hpp"
+#include "duckdb/common/exception.hpp"
+#include "duckdb/common/helper.hpp"
+
+namespace duckdb {
+
+constexpr idx_t FixedSizeAllocator::BASE[];
+constexpr uint8_t FixedSizeAllocator::SHIFT[];
+
+FixedSizeAllocator::FixedSizeAllocator(const idx_t allocation_size, Allocator &allocator)
+    : allocation_size(allocation_size), total_allocations(0), allocator(allocator) {
+
+	// calculate how many allocations fit into one buffer
+
+	idx_t bits_per_value = sizeof(validity_t) * 8;
+	idx_t curr_alloc_size = 0;
+
+	bitmask_count = 0;
+	allocations_per_buffer = 0;
+
+	while (curr_alloc_size < BUFFER_ALLOC_SIZE) {
+		if (!bitmask_count || (bitmask_count * bits_per_value) % allocations_per_buffer == 0) {
+			bitmask_count++;
+			curr_alloc_size += sizeof(validity_t);
+		}
+
+		auto remaining_alloc_size = BUFFER_ALLOC_SIZE - curr_alloc_size;
+		auto remaining_allocations = MinValue(remaining_alloc_size / allocation_size, bits_per_value);
+
+		if (remaining_allocations == 0) {
+			break;
+		}
+
+		allocations_per_buffer += remaining_allocations;
+		curr_alloc_size += remaining_allocations * allocation_size;
+	}
+
+	allocation_offset = bitmask_count * sizeof(validity_t);
+}
+
+FixedSizeAllocator::~FixedSizeAllocator() {
+	for (auto &buffer : buffers) {
+		allocator.FreeData(buffer.ptr, BUFFER_ALLOC_SIZE);
+	}
+}
+
+SwizzleablePointer FixedSizeAllocator::New() {
+
+	// no more free pointers
+	if (buffers_with_free_space.empty()) {
+
+		// add a new buffer
+		idx_t buffer_id = buffers.size();
+		D_ASSERT(buffer_id <= (uint32_t)DConstants::INVALID_INDEX);
+		auto buffer = allocator.AllocateData(BUFFER_ALLOC_SIZE);
+		buffers.emplace_back(buffer, 0);
+		buffers_with_free_space.insert(buffer_id);
+
+		// set the bitmask
+		ValidityMask mask((validity_t *)buffer);
+		mask.SetAllValid(allocations_per_buffer);
+	}
+
+	// return a pointer
+	D_ASSERT(!buffers_with_free_space.empty());
+	auto buffer_id = (uint32_t)*buffers_with_free_space.begin();
+
+	auto bitmask_ptr = (validity_t *)buffers[buffer_id].ptr;
+	ValidityMask mask(bitmask_ptr);
+	auto offset = GetOffset(mask, buffers[buffer_id].allocation_count);
+
+	buffers[buffer_id].allocation_count++;
+	total_allocations++;
+	if (buffers[buffer_id].allocation_count == allocations_per_buffer) {
+		buffers_with_free_space.erase(buffer_id);
+	}
+
+	return SwizzleablePointer(offset, buffer_id);
+}
+
+void FixedSizeAllocator::Free(const SwizzleablePointer ptr) {
+
+	auto bitmask_ptr = (validity_t *)buffers[ptr.buffer_id].ptr;
+	ValidityMask mask(bitmask_ptr);
+	D_ASSERT(!mask.RowIsValid(ptr.offset));
+	mask.SetValid(ptr.offset);
+	buffers_with_free_space.insert(ptr.buffer_id);
+
+	D_ASSERT(total_allocations > 0);
+	D_ASSERT(buffers[ptr.buffer_id].allocation_count > 0);
+	buffers[ptr.buffer_id].allocation_count--;
+	total_allocations--;
+}
+
+void FixedSizeAllocator::Reset() {
+
+	for (auto &buffer : buffers) {
+		allocator.FreeData(buffer.ptr, BUFFER_ALLOC_SIZE);
+	}
+	buffers.clear();
+	buffers_with_free_space.clear();
+	total_allocations = 0;
+}
+
+void FixedSizeAllocator::Merge(FixedSizeAllocator &other) {
+
+	D_ASSERT(allocation_size == other.allocation_size);
+
+	// remember the buffer count and merge the buffers
+	idx_t buffer_count = buffers.size();
+	for (auto &buffer : other.buffers) {
+		buffers.push_back(buffer);
+	}
+	other.buffers.clear();
+
+	// merge the vectors containing all buffers with free space
+	for (auto &buffer_id : other.buffers_with_free_space) {
+		buffers_with_free_space.insert(buffer_id + buffer_count);
+	}
+	other.buffers_with_free_space.clear();
+
+	// add the total allocations
+	total_allocations += other.total_allocations;
+}
+
+bool FixedSizeAllocator::InitializeVacuum() {
+
+	auto total_available_allocations = allocations_per_buffer * buffers.size();
+	auto total_free_positions = total_available_allocations - total_allocations;
+
+	// vacuum_count buffers can be freed
+	auto vacuum_count = total_free_positions / allocations_per_buffer / 2;
+
+	// calculate the vacuum threshold adaptively
+	idx_t memory_usage = GetMemoryUsage();
+	idx_t excess_memory_usage = vacuum_count * BUFFER_ALLOC_SIZE;
+	auto excess_percentage = (double)excess_memory_usage / (double)memory_usage;
+	auto threshold = (double)VACUUM_THRESHOLD / 100.0;
+	if (excess_percentage < threshold) {
+		return false;
+	}
+
+	min_vacuum_buffer_id = buffers.size() - vacuum_count;
+
+	// remove all invalid buffers from the available buffer list to ensure that we do not reuse them
+	auto it = buffers_with_free_space.begin();
+	while (it != buffers_with_free_space.end()) {
+		if (*it >= min_vacuum_buffer_id) {
+			it = buffers_with_free_space.erase(it);
+		} else {
+			it++;
+		}
+	}
+
+	return true;
+}
+
+void FixedSizeAllocator::FinalizeVacuum() {
+
+	// free all (now unused) buffers
+	while (min_vacuum_buffer_id < buffers.size()) {
+		allocator.FreeData(buffers.back().ptr, BUFFER_ALLOC_SIZE);
+		buffers.pop_back();
+	}
+}
+
+SwizzleablePointer FixedSizeAllocator::VacuumPointer(const SwizzleablePointer ptr) {
+
+	// we do not need to adjust the bitmask of the old buffer, because we will free the entire
+	// buffer after the vacuum operation
+
+	auto new_ptr = New();
+	memcpy(Get(new_ptr), Get(ptr), allocation_size);
+	return new_ptr;
+}
+
+uint32_t FixedSizeAllocator::GetOffset(ValidityMask &mask, const idx_t allocation_count) {
+
+	auto data = mask.GetData();
+
+	// fills up a buffer sequentially before searching for free bits
+	if (mask.RowIsValid(allocation_count)) {
+		mask.SetInvalid(allocation_count);
+		return allocation_count;
+	}
+
+	// get an entry with free bits
+	for (idx_t entry_idx = 0; entry_idx < bitmask_count; entry_idx++) {
+		if (data[entry_idx] != 0) {
+
+			// find the position of the free bit
+			auto entry = data[entry_idx];
+			idx_t first_valid_bit = 0;
+
+			// this loop finds the position of the rightmost set bit in entry and stores it
+			// in first_valid_bit
+			for (idx_t i = 0; i < 6; i++) {
+				// set the left half of the bits of this level to zero and test if the entry is still not zero
+				if (entry & BASE[i]) {
+					// first valid bit is in the rightmost s[i] bits
+					// permanently set the left half of the bits to zero
+					entry &= BASE[i];
+				} else {
+					// first valid bit is in the leftmost s[i] bits
+					// shift by s[i] for the next iteration and add s[i] to the position of the rightmost set bit
+					entry >>= SHIFT[i];
+					first_valid_bit += SHIFT[i];
+				}
+			}
+			D_ASSERT(entry);
+
+			auto prev_bits = entry_idx * sizeof(validity_t) * 8;
+			D_ASSERT(mask.RowIsValid(prev_bits + first_valid_bit));
+			mask.SetInvalid(prev_bits + first_valid_bit);
+			return (prev_bits + first_valid_bit);
+		}
+	}
+
+	throw InternalException("Invalid bitmask of FixedSizeAllocator");
+}
+
+} // namespace duckdb