duckdb 0.8.2-dev87.0 → 0.9.0

package/src/duckdb/src/execution/index/fixed_size_allocator.cpp

@@ -0,0 +1,323 @@
+#include "duckdb/execution/index/fixed_size_allocator.hpp"
+
+#include "duckdb/storage/metadata/metadata_reader.hpp"
+
+namespace duckdb {
+
+FixedSizeAllocator::FixedSizeAllocator(const idx_t segment_size, BlockManager &block_manager)
+    : block_manager(block_manager), buffer_manager(block_manager.buffer_manager),
+      metadata_manager(block_manager.GetMetadataManager()), segment_size(segment_size), total_segment_count(0) {
+
+	if (segment_size > Storage::BLOCK_SIZE - sizeof(validity_t)) {
+		throw InternalException("The maximum segment size of fixed-size allocators is " +
+		                        to_string(Storage::BLOCK_SIZE - sizeof(validity_t)));
+	}
+
+	// calculate how many segments fit into one buffer (available_segments_per_buffer)
+
+	idx_t bits_per_value = sizeof(validity_t) * 8;
+	idx_t byte_count = 0;
+
+	bitmask_count = 0;
+	available_segments_per_buffer = 0;
+
+	while (byte_count < Storage::BLOCK_SIZE) {
+		if (!bitmask_count || (bitmask_count * bits_per_value) % available_segments_per_buffer == 0) {
+			// we need to add another validity_t value to the bitmask, to allow storing another
+			// bits_per_value segments on a buffer
+			bitmask_count++;
+			byte_count += sizeof(validity_t);
+		}
+
+		auto remaining_bytes = Storage::BLOCK_SIZE - byte_count;
+		auto remaining_segments = MinValue(remaining_bytes / segment_size, bits_per_value);
+
+		if (remaining_segments == 0) {
+			break;
+		}
+
+		available_segments_per_buffer += remaining_segments;
+		byte_count += remaining_segments * segment_size;
+	}
+
+	bitmask_offset = bitmask_count * sizeof(validity_t);
+}
+
+IndexPointer FixedSizeAllocator::New() {
+
+	// no more segments available
+	if (buffers_with_free_space.empty()) {
+
+		// add a new buffer
+		auto buffer_id = GetAvailableBufferId();
+		FixedSizeBuffer new_buffer(block_manager);
+		buffers.insert(make_pair(buffer_id, std::move(new_buffer)));
+		buffers_with_free_space.insert(buffer_id);
+
+		// set the bitmask
+		D_ASSERT(buffers.find(buffer_id) != buffers.end());
+		auto &buffer = buffers.find(buffer_id)->second;
+		ValidityMask mask(reinterpret_cast<validity_t *>(buffer.Get()));
+
+		// zero-initialize the bitmask to avoid leaking memory to disk
+		auto data = mask.GetData();
+		for (idx_t i = 0; i < bitmask_count; i++) {
+			data[i] = 0;
+		}
+
+		// initializing the bitmask of the new buffer
+		mask.SetAllValid(available_segments_per_buffer);
+	}
+
+	// return a pointer to a free segment
+	D_ASSERT(!buffers_with_free_space.empty());
+	auto buffer_id = uint32_t(*buffers_with_free_space.begin());
+
+	D_ASSERT(buffers.find(buffer_id) != buffers.end());
+	auto &buffer = buffers.find(buffer_id)->second;
+	auto offset = buffer.GetOffset(bitmask_count);
+
+	total_segment_count++;
+	buffer.segment_count++;
+	if (buffer.segment_count == available_segments_per_buffer) {
+		buffers_with_free_space.erase(buffer_id);
+	}
+
+	// zero-initialize that segment
+	auto buffer_ptr = buffer.Get();
+	auto offset_in_buffer = buffer_ptr + offset * segment_size + bitmask_offset;
+	memset(offset_in_buffer, 0, segment_size);
+
+	return IndexPointer(buffer_id, offset);
+}
+
+void FixedSizeAllocator::Free(const IndexPointer ptr) {
+
+	auto buffer_id = ptr.GetBufferId();
+	auto offset = ptr.GetOffset();
+
+	D_ASSERT(buffers.find(buffer_id) != buffers.end());
+	auto &buffer = buffers.find(buffer_id)->second;
+
+	auto bitmask_ptr = reinterpret_cast<validity_t *>(buffer.Get());
+	ValidityMask mask(bitmask_ptr);
+	D_ASSERT(!mask.RowIsValid(offset));
+	mask.SetValid(offset);
+
+	D_ASSERT(total_segment_count > 0);
+	D_ASSERT(buffer.segment_count > 0);
+
+	// adjust the allocator fields
+	buffers_with_free_space.insert(buffer_id);
+	total_segment_count--;
+	buffer.segment_count--;
+}
+
+void FixedSizeAllocator::Reset() {
+	for (auto &buffer : buffers) {
+		buffer.second.Destroy();
+	}
+	buffers.clear();
+	buffers_with_free_space.clear();
+	total_segment_count = 0;
+}
+
+idx_t FixedSizeAllocator::GetMemoryUsage() const {
+	idx_t memory_usage = 0;
+	for (auto &buffer : buffers) {
+		if (buffer.second.InMemory()) {
+			memory_usage += Storage::BLOCK_SIZE;
+		}
+	}
+	return memory_usage;
+}
+
+idx_t FixedSizeAllocator::GetUpperBoundBufferId() const {
+	idx_t upper_bound_id = 0;
+	for (auto &buffer : buffers) {
+		if (buffer.first >= upper_bound_id) {
+			upper_bound_id = buffer.first + 1;
+		}
+	}
+	return upper_bound_id;
+}
+
+void FixedSizeAllocator::Merge(FixedSizeAllocator &other) {
+
+	D_ASSERT(segment_size == other.segment_size);
+
+	// remember the buffer count and merge the buffers
+	idx_t upper_bound_id = GetUpperBoundBufferId();
+	for (auto &buffer : other.buffers) {
+		buffers.insert(make_pair(buffer.first + upper_bound_id, std::move(buffer.second)));
+	}
+	other.buffers.clear();
+
+	// merge the buffers with free spaces
+	for (auto &buffer_id : other.buffers_with_free_space) {
+		buffers_with_free_space.insert(buffer_id + upper_bound_id);
+	}
+	other.buffers_with_free_space.clear();
+
+	// add the total allocations
+	total_segment_count += other.total_segment_count;
+}
+
+bool FixedSizeAllocator::InitializeVacuum() {
+
+	// NOTE: we do not vacuum buffers that are not in memory. We might consider changing this
+	// in the future, although buffers on disk should almost never be eligible for a vacuum
+
+	if (total_segment_count == 0) {
+		Reset();
+		return false;
+	}
+
+	// remove all empty buffers
+	auto buffer_it = buffers.begin();
+	while (buffer_it != buffers.end()) {
+		if (!buffer_it->second.segment_count) {
+			buffers_with_free_space.erase(buffer_it->first);
+			buffer_it->second.Destroy();
+			buffer_it = buffers.erase(buffer_it);
+		} else {
+			buffer_it++;
+		}
+	}
+
+	// determine if a vacuum is necessary
+	multimap<idx_t, idx_t> temporary_vacuum_buffers;
+	D_ASSERT(vacuum_buffers.empty());
+	idx_t available_segments_in_memory = 0;
+
+	for (auto &buffer : buffers) {
+		buffer.second.vacuum = false;
+		if (buffer.second.InMemory()) {
+			auto available_segments_in_buffer = available_segments_per_buffer - buffer.second.segment_count;
+			available_segments_in_memory += available_segments_in_buffer;
+			temporary_vacuum_buffers.emplace(available_segments_in_buffer, buffer.first);
+		}
+	}
+
+	// no buffers in memory
+	if (temporary_vacuum_buffers.empty()) {
+		return false;
+	}
+
+	auto excess_buffer_count = available_segments_in_memory / available_segments_per_buffer;
+
+	// calculate the vacuum threshold adaptively
+	D_ASSERT(excess_buffer_count < temporary_vacuum_buffers.size());
+	idx_t memory_usage = GetMemoryUsage();
+	idx_t excess_memory_usage = excess_buffer_count * Storage::BLOCK_SIZE;
+	auto excess_percentage = double(excess_memory_usage) / double(memory_usage);
+	auto threshold = double(VACUUM_THRESHOLD) / 100.0;
+	if (excess_percentage < threshold) {
+		return false;
+	}
+
+	D_ASSERT(excess_buffer_count <= temporary_vacuum_buffers.size());
+	D_ASSERT(temporary_vacuum_buffers.size() <= buffers.size());
+
+	// erasing from a multimap, we vacuum the buffers with the most free spaces (least full)
+	while (temporary_vacuum_buffers.size() != excess_buffer_count) {
+		temporary_vacuum_buffers.erase(temporary_vacuum_buffers.begin());
+	}
+
+	// adjust the buffers, and erase all to-be-vacuumed buffers from the available buffer list
+	for (auto &vacuum_buffer : temporary_vacuum_buffers) {
+		auto buffer_id = vacuum_buffer.second;
+		D_ASSERT(buffers.find(buffer_id) != buffers.end());
+		buffers.find(buffer_id)->second.vacuum = true;
+		buffers_with_free_space.erase(buffer_id);
+	}
+
+	for (auto &vacuum_buffer : temporary_vacuum_buffers) {
+		vacuum_buffers.insert(vacuum_buffer.second);
+	}
+
+	return true;
+}
+
+void FixedSizeAllocator::FinalizeVacuum() {
+
+	for (auto &buffer_id : vacuum_buffers) {
+		D_ASSERT(buffers.find(buffer_id) != buffers.end());
+		auto &buffer = buffers.find(buffer_id)->second;
+		D_ASSERT(buffer.InMemory());
+		buffer.Destroy();
+		buffers.erase(buffer_id);
+	}
+	vacuum_buffers.clear();
+}
+
+IndexPointer FixedSizeAllocator::VacuumPointer(const IndexPointer ptr) {
+
+	// we do not need to adjust the bitmask of the old buffer, because we will free the entire
+	// buffer after the vacuum operation
+
+	auto new_ptr = New();
+	// new increases the allocation count, we need to counter that here
+	total_segment_count--;
+
+	memcpy(Get(new_ptr), Get(ptr), segment_size);
+	return new_ptr;
+}
+
+BlockPointer FixedSizeAllocator::Serialize(PartialBlockManager &partial_block_manager, MetadataWriter &writer) {
+
+	for (auto &buffer : buffers) {
+		buffer.second.Serialize(partial_block_manager, available_segments_per_buffer, segment_size, bitmask_offset);
+	}
+
+	auto block_pointer = writer.GetBlockPointer();
+	writer.Write(segment_size);
+	writer.Write(static_cast<idx_t>(buffers.size()));
+	writer.Write(static_cast<idx_t>(buffers_with_free_space.size()));
+
+	for (auto &buffer : buffers) {
+		writer.Write(buffer.first);
+		writer.Write(buffer.second.block_pointer);
+		writer.Write(buffer.second.segment_count);
+		writer.Write(buffer.second.allocation_size);
+	}
+	for (auto &buffer_id : buffers_with_free_space) {
+		writer.Write(buffer_id);
+	}
+
+	return block_pointer;
+}
+
+void FixedSizeAllocator::Deserialize(const BlockPointer &block_pointer) {
+
+	MetadataReader reader(metadata_manager, block_pointer);
+	segment_size = reader.Read<idx_t>();
+	auto buffer_count = reader.Read<idx_t>();
+	auto buffers_with_free_space_count = reader.Read<idx_t>();
+
+	total_segment_count = 0;
+
+	for (idx_t i = 0; i < buffer_count; i++) {
+		auto buffer_id = reader.Read<idx_t>();
+		auto buffer_block_pointer = reader.Read<BlockPointer>();
+		auto segment_count = reader.Read<idx_t>();
+		auto allocation_size = reader.Read<idx_t>();
+		FixedSizeBuffer new_buffer(block_manager, segment_count, allocation_size, buffer_block_pointer);
+		buffers.insert(make_pair(buffer_id, std::move(new_buffer)));
+		total_segment_count += segment_count;
+	}
+	for (idx_t i = 0; i < buffers_with_free_space_count; i++) {
+		buffers_with_free_space.insert(reader.Read<idx_t>());
+	}
+}
+
+idx_t FixedSizeAllocator::GetAvailableBufferId() const {
+	idx_t buffer_id = buffers.size();
+	while (buffers.find(buffer_id) != buffers.end()) {
+		D_ASSERT(buffer_id > 0);
+		buffer_id--;
+	}
+	return buffer_id;
+}
+
+} // namespace duckdb

package/src/duckdb/src/execution/index/fixed_size_buffer.cpp

@@ -0,0 +1,285 @@
+#include "duckdb/execution/index/fixed_size_buffer.hpp"
+
+#include "duckdb/storage/block_manager.hpp"
+#include "duckdb/storage/buffer_manager.hpp"
+
+namespace duckdb {
+
+//===--------------------------------------------------------------------===//
+// PartialBlockForIndex
+//===--------------------------------------------------------------------===//
+
+PartialBlockForIndex::PartialBlockForIndex(PartialBlockState state, BlockManager &block_manager,
+                                           const shared_ptr<BlockHandle> &block_handle)
+    : PartialBlock(state, block_manager, block_handle) {
+}
+
+void PartialBlockForIndex::Flush(const idx_t free_space_left) {
+	FlushInternal(free_space_left);
+	block_handle = block_manager.ConvertToPersistent(state.block_id, std::move(block_handle));
+	Clear();
+}
+
+void PartialBlockForIndex::Merge(PartialBlock &other, idx_t offset, idx_t other_size) {
+	throw InternalException("no merge for PartialBlockForIndex");
+}
+
+void PartialBlockForIndex::Clear() {
+	block_handle.reset();
+}
+
+//===--------------------------------------------------------------------===//
+// FixedSizeBuffer
+//===--------------------------------------------------------------------===//
+
+constexpr idx_t FixedSizeBuffer::BASE[];
+constexpr uint8_t FixedSizeBuffer::SHIFT[];
+
+FixedSizeBuffer::FixedSizeBuffer(BlockManager &block_manager)
+    : block_manager(block_manager), segment_count(0), allocation_size(0), dirty(false), vacuum(false), block_pointer(),
+      block_handle(nullptr) {
+
+	auto &buffer_manager = block_manager.buffer_manager;
+	buffer_handle = buffer_manager.Allocate(Storage::BLOCK_SIZE, false, &block_handle);
+}
+
+FixedSizeBuffer::FixedSizeBuffer(BlockManager &block_manager, const idx_t segment_count, const idx_t allocation_size,
+                                 const BlockPointer &block_pointer)
+    : block_manager(block_manager), segment_count(segment_count), allocation_size(allocation_size), dirty(false),
+      vacuum(false), block_pointer(block_pointer) {
+
+	D_ASSERT(block_pointer.IsValid());
+	block_handle = block_manager.RegisterBlock(block_pointer.block_id);
+	D_ASSERT(block_handle->BlockId() < MAXIMUM_BLOCK);
+}
+
+void FixedSizeBuffer::Destroy() {
+	if (InMemory()) {
+		// we can have multiple readers on a pinned block, and unpinning the buffer handle
+		// decrements the reader count on the underlying block handle (Destroy() unpins)
+		buffer_handle.Destroy();
+	}
+	if (OnDisk()) {
+		// marking a block as modified decreases the reference count of multi-use blocks
+		block_manager.MarkBlockAsModified(block_pointer.block_id);
+	}
+}
+
+void FixedSizeBuffer::Serialize(PartialBlockManager &partial_block_manager, const idx_t available_segments,
+                                const idx_t segment_size, const idx_t bitmask_offset) {
+
+	// we do not serialize a block that is already on disk and not in memory
+	if (!InMemory()) {
+		if (!OnDisk() || dirty) {
+			throw InternalException("invalid or missing buffer in FixedSizeAllocator");
+		}
+		return;
+	}
+
+	// we do not serialize a block that is already on disk and not dirty
+	if (!dirty && OnDisk()) {
+		return;
+	}
+
+	if (dirty) {
+		// the allocation possibly changed
+		auto max_offset = GetMaxOffset(available_segments);
+		allocation_size = max_offset * segment_size + bitmask_offset;
+	}
+
+	// the buffer is in memory, so we copied it onto a new buffer when pinning
+	D_ASSERT(InMemory() && !OnDisk());
+
+	// now we write the changes, first get a partial block allocation
+	PartialBlockAllocation allocation = partial_block_manager.GetBlockAllocation(allocation_size);
+	block_pointer.block_id = allocation.state.block_id;
+	block_pointer.offset = allocation.state.offset;
+
+	auto &buffer_manager = block_manager.buffer_manager;
+
+	if (allocation.partial_block) {
+		// copy to an existing partial block
+		D_ASSERT(block_pointer.offset > 0);
+		auto &p_block_for_index = allocation.partial_block->Cast<PartialBlockForIndex>();
+		auto dst_handle = buffer_manager.Pin(p_block_for_index.block_handle);
+		memcpy(dst_handle.Ptr() + block_pointer.offset, buffer_handle.Ptr(), allocation_size);
+		SetUninitializedRegions(p_block_for_index, segment_size, block_pointer.offset, bitmask_offset);
+
+	} else {
+		// create a new block that can potentially be used as a partial block
+		D_ASSERT(block_handle);
+		D_ASSERT(!block_pointer.offset);
+		auto p_block_for_index = make_uniq<PartialBlockForIndex>(allocation.state, block_manager, block_handle);
+		SetUninitializedRegions(*p_block_for_index, segment_size, block_pointer.offset, bitmask_offset);
+		allocation.partial_block = std::move(p_block_for_index);
+	}
+
+	partial_block_manager.RegisterPartialBlock(std::move(allocation));
+
+	// resetting this buffer
+	buffer_handle.Destroy();
+	block_handle = block_manager.RegisterBlock(block_pointer.block_id);
+	D_ASSERT(block_handle->BlockId() < MAXIMUM_BLOCK);
+
+	// we persist any changes, so the buffer is no longer dirty
+	dirty = false;
+}
+
+void FixedSizeBuffer::Pin() {
+
+	auto &buffer_manager = block_manager.buffer_manager;
+	D_ASSERT(block_pointer.IsValid());
+	D_ASSERT(block_handle && block_handle->BlockId() < MAXIMUM_BLOCK);
+	D_ASSERT(!dirty);
+
+	buffer_handle = buffer_manager.Pin(block_handle);
+
+	// we need to copy the (partial) data into a new (not yet disk-backed) buffer handle
+	shared_ptr<BlockHandle> new_block_handle;
+	auto new_buffer_handle = buffer_manager.Allocate(Storage::BLOCK_SIZE, false, &new_block_handle);
+
+	memcpy(new_buffer_handle.Ptr(), buffer_handle.Ptr() + block_pointer.offset, allocation_size);
+
+	Destroy();
+	buffer_handle = std::move(new_buffer_handle);
+	block_handle = new_block_handle;
+	block_pointer = BlockPointer();
+}
+
+uint32_t FixedSizeBuffer::GetOffset(const idx_t bitmask_count) {
+
+	// get the bitmask data
+	auto bitmask_ptr = reinterpret_cast<validity_t *>(Get());
+	ValidityMask mask(bitmask_ptr);
+	auto data = mask.GetData();
+
+	// fills up a buffer sequentially before searching for free bits
+	if (mask.RowIsValid(segment_count)) {
+		mask.SetInvalid(segment_count);
+		return segment_count;
+	}
+
+	for (idx_t entry_idx = 0; entry_idx < bitmask_count; entry_idx++) {
+		// get an entry with free bits
+		if (data[entry_idx] == 0) {
+			continue;
+		}
+
+		// find the position of the free bit
+		auto entry = data[entry_idx];
+		idx_t first_valid_bit = 0;
+
+		// this loop finds the position of the rightmost set bit in entry and stores it
+		// in first_valid_bit
+		for (idx_t i = 0; i < 6; i++) {
+			// set the left half of the bits of this level to zero and test if the entry is still not zero
+			if (entry & BASE[i]) {
+				// first valid bit is in the rightmost s[i] bits
+				// permanently set the left half of the bits to zero
+				entry &= BASE[i];
+			} else {
+				// first valid bit is in the leftmost s[i] bits
+				// shift by s[i] for the next iteration and add s[i] to the position of the rightmost set bit
+				entry >>= SHIFT[i];
+				first_valid_bit += SHIFT[i];
+			}
+		}
+		D_ASSERT(entry);
+
+		auto prev_bits = entry_idx * sizeof(validity_t) * 8;
+		D_ASSERT(mask.RowIsValid(prev_bits + first_valid_bit));
+		mask.SetInvalid(prev_bits + first_valid_bit);
+		return (prev_bits + first_valid_bit);
+	}
+
+	throw InternalException("Invalid bitmask for FixedSizeAllocator");
+}
+
+uint32_t FixedSizeBuffer::GetMaxOffset(const idx_t available_segments) {
+
+	// this function calls Get() on the buffer
+	D_ASSERT(InMemory());
+
+	// finds the maximum zero bit in a bitmask, and adds one to it,
+	// so that max_offset * segment_size = allocated_size of this bitmask's buffer
+	idx_t entry_size = sizeof(validity_t) * 8;
+	idx_t bitmask_count = available_segments / entry_size;
+	if (available_segments % entry_size != 0) {
+		bitmask_count++;
+	}
+	uint32_t max_offset = bitmask_count * sizeof(validity_t) * 8;
+	auto bits_in_last_entry = available_segments % (sizeof(validity_t) * 8);
+
+	// get the bitmask data
+	auto bitmask_ptr = reinterpret_cast<validity_t *>(Get());
+	const ValidityMask mask(bitmask_ptr);
+	const auto data = mask.GetData();
+
+	D_ASSERT(bitmask_count > 0);
+	for (idx_t i = bitmask_count; i > 0; i--) {
+
+		auto entry = data[i - 1];
+
+		// set all bits after bits_in_last_entry
+		if (i == bitmask_count) {
+			entry |= ~idx_t(0) << bits_in_last_entry;
+		}
+
+		if (entry == ~idx_t(0)) {
+			max_offset -= sizeof(validity_t) * 8;
+			continue;
+		}
+
+		// invert data[entry_idx]
+		auto entry_inv = ~entry;
+		idx_t first_valid_bit = 0;
+
+		// then find the position of the LEFTMOST set bit
+		for (idx_t level = 0; level < 6; level++) {
+
+			// set the right half of the bits of this level to zero and test if the entry is still not zero
+			if (entry_inv & ~BASE[level]) {
+				// first valid bit is in the leftmost s[level] bits
+				// shift by s[level] for the next iteration and add s[level] to the position of the leftmost set bit
+				entry_inv >>= SHIFT[level];
+				first_valid_bit += SHIFT[level];
+			} else {
+				// first valid bit is in the rightmost s[level] bits
+				// permanently set the left half of the bits to zero
+				entry_inv &= BASE[level];
+			}
+		}
+		D_ASSERT(entry_inv);
+		max_offset -= sizeof(validity_t) * 8 - first_valid_bit;
+		D_ASSERT(!mask.RowIsValid(max_offset));
+		return max_offset + 1;
+	}
+
+	// there are no allocations in this buffer
+	throw InternalException("tried to serialize empty buffer");
+}
+
+void FixedSizeBuffer::SetUninitializedRegions(PartialBlockForIndex &p_block_for_index, const idx_t segment_size,
+                                              const idx_t offset, const idx_t bitmask_offset) {
+
+	// this function calls Get() on the buffer
+	D_ASSERT(InMemory());
+
+	auto bitmask_ptr = reinterpret_cast<validity_t *>(Get());
+	ValidityMask mask(bitmask_ptr);
+
+	idx_t i = 0;
+	idx_t max_offset = offset + allocation_size;
+	idx_t current_offset = offset + bitmask_offset;
+	while (current_offset < max_offset) {
+
+		if (mask.RowIsValid(i)) {
+			D_ASSERT(current_offset + segment_size <= max_offset);
+			p_block_for_index.AddUninitializedRegion(current_offset, current_offset + segment_size);
+		}
+		current_offset += segment_size;
+		i++;
+	}
+}
+
+} // namespace duckdb