duckdb 0.8.2-dev4025.0 → 0.8.2-dev4126.0

package/src/duckdb/src/storage/metadata/metadata_manager.cpp

@@ -13,7 +13,7 @@ MetadataManager::~MetadataManager() {
 
 MetadataHandle MetadataManager::AllocateHandle() {
 	// check if there is any free space left in an existing block
-	// if not allocate a new bloc
+	// if not allocate a new block
 	block_id_t free_block = INVALID_BLOCK;
 	for (auto &kv : blocks) {
 		auto &block = kv.second;
@@ -230,27 +230,27 @@ void MetadataBlock::FreeBlocksFromInteger(idx_t free_list) {
 }
 
 void MetadataManager::MarkBlocksAsModified() {
-	if (!modified_blocks.empty()) {
-		// for any blocks that were modified in the last checkpoint - set them to free blocks currently
-		for (auto &kv : modified_blocks) {
-			auto block_id = kv.first;
-			idx_t modified_list = kv.second;
-			auto entry = blocks.find(block_id);
-			D_ASSERT(entry != blocks.end());
-			auto &block = entry->second;
-			idx_t current_free_blocks = block.FreeBlocksToInteger();
-			// merge the current set of free blocks with the modified blocks
-			idx_t new_free_blocks = current_free_blocks | modified_list;
-			//			if (new_free_blocks == NumericLimits<idx_t>::Maximum()) {
-			//				// if new free_blocks is all blocks - mark entire block as modified
-			//				blocks.erase(entry);
-			//				block_manager.MarkBlockAsModified(block_id);
-			//			} else {
-			// set the new set of free blocks
-			block.FreeBlocksFromInteger(new_free_blocks);
-			//			}
-		}
+
+	// for any blocks that were modified in the last checkpoint - set them to free blocks currently
+	for (auto &kv : modified_blocks) {
+		auto block_id = kv.first;
+		idx_t modified_list = kv.second;
+		auto entry = blocks.find(block_id);
+		D_ASSERT(entry != blocks.end());
+		auto &block = entry->second;
+		idx_t current_free_blocks = block.FreeBlocksToInteger();
+		// merge the current set of free blocks with the modified blocks
+		idx_t new_free_blocks = current_free_blocks | modified_list;
+		//			if (new_free_blocks == NumericLimits<idx_t>::Maximum()) {
+		//				// if new free_blocks is all blocks - mark entire block as modified
+		//				blocks.erase(entry);
+		//				block_manager.MarkBlockAsModified(block_id);
+		//			} else {
+		// set the new set of free blocks
+		block.FreeBlocksFromInteger(new_free_blocks);
+		//			}
 	}
+
 	modified_blocks.clear();
 	for (auto &kv : blocks) {
 		auto &block = kv.second;

package/src/duckdb/src/storage/standard_buffer_manager.cpp

@@ -77,14 +77,6 @@ idx_t StandardBufferManager::GetMaxMemory() const {
 	return buffer_pool.GetMaxMemory();
 }
 
-// POTENTIALLY PROBLEMATIC
-// void StandardBufferManager::IncreaseUsedMemory(idx_t size, bool unsafe) {
-//	if (!unsafe && buffer_pool.GetUsedMemory() + size > buffer_pool.GetMaxMemory()) {
-//		throw OutOfMemoryException("Failed to allocate data of size %lld%s", size, InMemoryWarning());
-//	}
-//	buffer_pool.IncreaseUsedMemory(size);
-//}
-
 template <typename... ARGS>
 TempBufferPoolReservation StandardBufferManager::EvictBlocksOrThrow(idx_t memory_delta, unique_ptr<FileBuffer> *buffer,
                                                                     ARGS... args) {

package/src/duckdb/src/storage/storage_info.cpp

@@ -2,7 +2,7 @@
 
 namespace duckdb {
 
-const uint64_t VERSION_NUMBER = 57;
+const uint64_t VERSION_NUMBER = 58;
 
 struct StorageVersionInfo {
 	const char *version_name;

package/src/duckdb/src/storage/table_index_list.cpp

@@ -54,7 +54,7 @@ void TableIndexList::VerifyForeignKey(const vector<PhysicalIndex> &fk_keys, Data
 	                   ? ForeignKeyType::FK_TYPE_PRIMARY_KEY_TABLE
 	                   : ForeignKeyType::FK_TYPE_FOREIGN_KEY_TABLE;
 
-	// check whether or not the chunk can be inserted or deleted into the referenced table' storage
+	// check whether the chunk can be inserted or deleted into the referenced table storage
 	auto index = FindForeignKeyIndex(fk_keys, fk_type);
 	if (!index) {
 		throw InternalException("Internal Foreign Key error: could not find index to verify...");

package/src/duckdb/src/transaction/commit_state.cpp

@@ -1,5 +1,6 @@
 #include "duckdb/transaction/commit_state.hpp"
 
+#include "duckdb/catalog/catalog_entry/duck_index_entry.hpp"
 #include "duckdb/catalog/catalog_entry/duck_table_entry.hpp"
 #include "duckdb/catalog/catalog_entry/type_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_set.hpp"
@@ -124,9 +125,12 @@ void CommitState::WriteCatalogEntry(CatalogEntry &entry, data_ptr_t dataptr) {
 		case CatalogType::TYPE_ENTRY:
 			log->WriteDropType(entry.Cast<TypeCatalogEntry>());
 			break;
-		case CatalogType::INDEX_ENTRY:
+		case CatalogType::INDEX_ENTRY: {
+			auto &index_entry = entry.Cast<DuckIndexEntry>();
+			index_entry.CommitDrop();
 			log->WriteDropIndex(entry.Cast<IndexCatalogEntry>());
 			break;
+		}
 		case CatalogType::PREPARED_STATEMENT:
 		case CatalogType::SCALAR_FUNCTION_ENTRY:
 			// do nothing, indexes/prepared statements/functions aren't persisted to disk

package/src/duckdb/ub_src_core_functions_scalar_list.cpp

@@ -10,5 +10,11 @@
 
 #include "src/core_functions/scalar/list/list_sort.cpp"
 
+#include "src/core_functions/scalar/list/list_distance.cpp"
+
+#include "src/core_functions/scalar/list/list_cosine_similarity.cpp"
+
+#include "src/core_functions/scalar/list/list_inner_product.cpp"
+
 #include "src/core_functions/scalar/list/range.cpp"
 

package/src/duckdb/ub_src_execution_index.cpp

@@ -0,0 +1,4 @@
+#include "src/execution/index/fixed_size_allocator.cpp"
+
+#include "src/execution/index/fixed_size_buffer.cpp"
+

package/src/duckdb/ub_src_execution_index_art.cpp

@@ -2,8 +2,6 @@
 
 #include "src/execution/index/art/node.cpp"
 
-#include "src/execution/index/art/fixed_size_allocator.cpp"
-
 #include "src/execution/index/art/iterator.cpp"
 
 #include "src/execution/index/art/leaf.cpp"

package/src/duckdb/src/execution/index/art/fixed_size_allocator.cpp

@@ -1,238 +0,0 @@
-#include "duckdb/execution/index/art/fixed_size_allocator.hpp"
-
-namespace duckdb {
-
-constexpr idx_t FixedSizeAllocator::BASE[];
-constexpr uint8_t FixedSizeAllocator::SHIFT[];
-
-FixedSizeAllocator::FixedSizeAllocator(const idx_t allocation_size, Allocator &allocator)
-    : allocation_size(allocation_size), total_allocations(0), allocator(allocator) {
-
-	// calculate how many allocations fit into one buffer
-
-	idx_t bits_per_value = sizeof(validity_t) * 8;
-	idx_t curr_alloc_size = 0;
-
-	bitmask_count = 0;
-	allocations_per_buffer = 0;
-
-	while (curr_alloc_size < BUFFER_ALLOC_SIZE) {
-		if (!bitmask_count || (bitmask_count * bits_per_value) % allocations_per_buffer == 0) {
-			bitmask_count++;
-			curr_alloc_size += sizeof(validity_t);
-		}
-
-		auto remaining_alloc_size = BUFFER_ALLOC_SIZE - curr_alloc_size;
-		auto remaining_allocations = MinValue(remaining_alloc_size / allocation_size, bits_per_value);
-
-		if (remaining_allocations == 0) {
-			break;
-		}
-
-		allocations_per_buffer += remaining_allocations;
-		curr_alloc_size += remaining_allocations * allocation_size;
-	}
-
-	allocation_offset = bitmask_count * sizeof(validity_t);
-}
-
-FixedSizeAllocator::~FixedSizeAllocator() {
-	for (auto &buffer : buffers) {
-		allocator.FreeData(buffer.ptr, BUFFER_ALLOC_SIZE);
-	}
-}
-
-Node FixedSizeAllocator::New() {
-
-	// no more free pointers
-	if (buffers_with_free_space.empty()) {
-
-		// add a new buffer
-		idx_t buffer_id = buffers.size();
-		D_ASSERT(buffer_id <= (uint32_t)DConstants::INVALID_INDEX);
-		auto buffer = allocator.AllocateData(BUFFER_ALLOC_SIZE);
-		buffers.emplace_back(buffer, 0);
-		buffers_with_free_space.insert(buffer_id);
-
-		// set the bitmask
-		ValidityMask mask(reinterpret_cast<validity_t *>(buffer));
-		mask.SetAllValid(allocations_per_buffer);
-	}
-
-	// return a pointer
-	D_ASSERT(!buffers_with_free_space.empty());
-	auto buffer_id = (uint32_t)*buffers_with_free_space.begin();
-
-	auto bitmask_ptr = reinterpret_cast<validity_t *>(buffers[buffer_id].ptr);
-	ValidityMask mask(bitmask_ptr);
-	auto offset = GetOffset(mask, buffers[buffer_id].allocation_count);
-
-	buffers[buffer_id].allocation_count++;
-	total_allocations++;
-	if (buffers[buffer_id].allocation_count == allocations_per_buffer) {
-		buffers_with_free_space.erase(buffer_id);
-	}
-
-	return Node(buffer_id, offset);
-}
-
-void FixedSizeAllocator::Free(const Node ptr) {
-	auto bitmask_ptr = reinterpret_cast<validity_t *>(buffers[ptr.GetBufferId()].ptr);
-	ValidityMask mask(bitmask_ptr);
-	D_ASSERT(!mask.RowIsValid(ptr.GetOffset()));
-	mask.SetValid(ptr.GetOffset());
-	buffers_with_free_space.insert(ptr.GetBufferId());
-
-	D_ASSERT(total_allocations > 0);
-	D_ASSERT(buffers[ptr.GetBufferId()].allocation_count > 0);
-	buffers[ptr.GetBufferId()].allocation_count--;
-	total_allocations--;
-}
-
-void FixedSizeAllocator::Reset() {
-
-	for (auto &buffer : buffers) {
-		allocator.FreeData(buffer.ptr, BUFFER_ALLOC_SIZE);
-	}
-	buffers.clear();
-	buffers_with_free_space.clear();
-	total_allocations = 0;
-}
-
-void FixedSizeAllocator::Merge(FixedSizeAllocator &other) {
-
-	D_ASSERT(allocation_size == other.allocation_size);
-
-	// remember the buffer count and merge the buffers
-	idx_t buffer_count = buffers.size();
-	for (auto &buffer : other.buffers) {
-		buffers.push_back(buffer);
-	}
-	other.buffers.clear();
-
-	// merge the buffers with free spaces
-	for (auto &buffer_id : other.buffers_with_free_space) {
-		buffers_with_free_space.insert(buffer_id + buffer_count);
-	}
-	other.buffers_with_free_space.clear();
-
-	// add the total allocations
-	total_allocations += other.total_allocations;
-}
-
-bool FixedSizeAllocator::InitializeVacuum() {
-
-	if (total_allocations == 0) {
-		Reset();
-		return false;
-	}
-
-	auto total_available_allocations = allocations_per_buffer * buffers.size();
-	D_ASSERT(total_available_allocations >= total_allocations);
-	auto total_free_positions = total_available_allocations - total_allocations;
-
-	// vacuum_count buffers can be freed
-	auto vacuum_count = total_free_positions / allocations_per_buffer;
-
-	// calculate the vacuum threshold adaptively
-	D_ASSERT(vacuum_count < buffers.size());
-	idx_t memory_usage = GetMemoryUsage();
-	idx_t excess_memory_usage = vacuum_count * BUFFER_ALLOC_SIZE;
-	auto excess_percentage = (double)excess_memory_usage / (double)memory_usage;
-	auto threshold = (double)VACUUM_THRESHOLD / 100.0;
-	if (excess_percentage < threshold) {
-		return false;
-	}
-
-	min_vacuum_buffer_id = buffers.size() - vacuum_count;
-
-	// remove all invalid buffers from the available buffer list to ensure that we do not reuse them
-	auto it = buffers_with_free_space.begin();
-	while (it != buffers_with_free_space.end()) {
-		if (*it >= min_vacuum_buffer_id) {
-			it = buffers_with_free_space.erase(it);
-		} else {
-			it++;
-		}
-	}
-
-	return true;
-}
-
-void FixedSizeAllocator::FinalizeVacuum() {
-
-	// free all (now unused) buffers
-	while (min_vacuum_buffer_id < buffers.size()) {
-		allocator.FreeData(buffers.back().ptr, BUFFER_ALLOC_SIZE);
-		buffers.pop_back();
-	}
-}
-
-Node FixedSizeAllocator::VacuumPointer(const Node ptr) {
-
-	// we do not need to adjust the bitmask of the old buffer, because we will free the entire
-	// buffer after the vacuum operation
-
-	auto new_ptr = New();
-
-	// new increases the allocation count
-	total_allocations--;
-
-	memcpy(Get(new_ptr), Get(ptr), allocation_size);
-	return new_ptr;
-}
-
-void FixedSizeAllocator::Verify() const {
-#ifdef DEBUG
-	auto total_available_allocations = allocations_per_buffer * buffers.size();
-	D_ASSERT(total_available_allocations >= total_allocations);
-	D_ASSERT(buffers.size() >= buffers_with_free_space.size());
-#endif
-}
-
-uint32_t FixedSizeAllocator::GetOffset(ValidityMask &mask, const idx_t allocation_count) {
-
-	auto data = mask.GetData();
-
-	// fills up a buffer sequentially before searching for free bits
-	if (mask.RowIsValid(allocation_count)) {
-		mask.SetInvalid(allocation_count);
-		return allocation_count;
-	}
-
-	// get an entry with free bits
-	for (idx_t entry_idx = 0; entry_idx < bitmask_count; entry_idx++) {
-		if (data[entry_idx] != 0) {
-
-			// find the position of the free bit
-			auto entry = data[entry_idx];
-			idx_t first_valid_bit = 0;
-
-			// this loop finds the position of the rightmost set bit in entry and stores it
-			// in first_valid_bit
-			for (idx_t i = 0; i < 6; i++) {
-				// set the left half of the bits of this level to zero and test if the entry is still not zero
-				if (entry & BASE[i]) {
-					// first valid bit is in the rightmost s[i] bits
-					// permanently set the left half of the bits to zero
-					entry &= BASE[i];
-				} else {
-					// first valid bit is in the leftmost s[i] bits
-					// shift by s[i] for the next iteration and add s[i] to the position of the rightmost set bit
-					entry >>= SHIFT[i];
-					first_valid_bit += SHIFT[i];
-				}
-			}
-			D_ASSERT(entry);
-
-			auto prev_bits = entry_idx * sizeof(validity_t) * 8;
-			D_ASSERT(mask.RowIsValid(prev_bits + first_valid_bit));
-			mask.SetInvalid(prev_bits + first_valid_bit);
-			return (prev_bits + first_valid_bit);
-		}
-	}
-
-	throw InternalException("Invalid bitmask of FixedSizeAllocator");
-}
-
-} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/index/art/fixed_size_allocator.hpp

@@ -1,115 +0,0 @@
-//===----------------------------------------------------------------------===//
-//                         DuckDB
-//
-// duckdb/execution/index/art/fixed_size_allocator.hpp
-//
-//
-//===----------------------------------------------------------------------===//
-
-#pragma once
-
-#include "duckdb/common/types/validity_mask.hpp"
-#include "duckdb/common/unordered_set.hpp"
-#include "duckdb/storage/buffer_manager.hpp"
-#include "duckdb/execution/index/art/node.hpp"
-
-namespace duckdb {
-
-// structs
-struct BufferEntry {
-	BufferEntry(const data_ptr_t &ptr, const idx_t &allocation_count) : ptr(ptr), allocation_count(allocation_count) {
-	}
-	data_ptr_t ptr;
-	idx_t allocation_count;
-};
-
-//! The FixedSizeAllocator provides pointers to fixed-size sections of pre-allocated memory buffers.
-//! The pointers are Node pointers, and the leftmost byte (serialize flag and type) must always be zero.
-class FixedSizeAllocator {
-public:
-	//! Fixed size of the buffers
-	static constexpr idx_t BUFFER_ALLOC_SIZE = Storage::BLOCK_ALLOC_SIZE;
-	//! We can vacuum 10% or more of the total memory usage of the allocator
-	static constexpr uint8_t VACUUM_THRESHOLD = 10;
-
-	//! Constants for fast offset calculations in the bitmask
-	static constexpr idx_t BASE[] = {0x00000000FFFFFFFF, 0x0000FFFF, 0x00FF, 0x0F, 0x3, 0x1};
-	static constexpr uint8_t SHIFT[] = {32, 16, 8, 4, 2, 1};
-
-public:
-	explicit FixedSizeAllocator(const idx_t allocation_size, Allocator &allocator);
-	~FixedSizeAllocator();
-
-	//! Allocation size of one element in a buffer
-	idx_t allocation_size;
-	//! Total number of allocations
-	idx_t total_allocations;
-	//! Number of validity_t values in the bitmask
-	idx_t bitmask_count;
-	//! First starting byte of the payload
-	idx_t allocation_offset;
-	//! Number of possible allocations per buffer
-	idx_t allocations_per_buffer;
-
-	//! Buffers containing the data
-	vector<BufferEntry> buffers;
-	//! Buffers with free space
-	unordered_set<idx_t> buffers_with_free_space;
-
-	//! Minimum buffer ID of buffers that can be vacuumed
-	idx_t min_vacuum_buffer_id;
-
-	//! Buffer manager of the database instance
-	Allocator &allocator;
-
-public:
-	//! Get a new Node pointer to data, might cause a new buffer allocation
-	Node New();
-	//! Free the data of the Node pointer
-	void Free(const Node ptr);
-	//! Get the data of the Node pointer
-	template <class T>
-	inline T *Get(const Node ptr) const {
-		return (T *)Get(ptr);
-	}
-
-	//! Resets the allocator, e.g., becomes necessary during DELETE FROM table
-	void Reset();
-
-	//! Returns the allocated memory size in bytes
-	inline idx_t GetMemoryUsage() const {
-		return buffers.size() * BUFFER_ALLOC_SIZE;
-	}
-
-	//! Merge another FixedSizeAllocator into this allocator. Both must have the same allocation size
-	void Merge(FixedSizeAllocator &other);
-
-	//! Initialize a vacuum operation, and return true, if the allocator needs a vacuum
-	bool InitializeVacuum();
-	//! Finalize a vacuum operation by freeing all buffers exceeding the min_vacuum_buffer_id
-	void FinalizeVacuum();
-	//! Returns true, if a Node pointer qualifies for a vacuum operation, and false otherwise
-	inline bool NeedsVacuum(const Node ptr) const {
-		if (ptr.GetBufferId() >= min_vacuum_buffer_id) {
-			return true;
-		}
-		return false;
-	}
-	//! Vacuums a Node pointer
-	Node VacuumPointer(const Node ptr);
-
-	//! Verify that the allocation counts match the existing positions on the buffers
-	void Verify() const;
-
-private:
-	//! Returns the data_ptr_t of a Node pointer
-	inline data_ptr_t Get(const Node ptr) const {
-		D_ASSERT(ptr.GetBufferId() < buffers.size());
-		D_ASSERT(ptr.GetOffset() < allocations_per_buffer);
-		return buffers[ptr.GetBufferId()].ptr + ptr.GetOffset() * allocation_size + allocation_offset;
-	}
-	//! Returns the first free offset in a bitmask
-	uint32_t GetOffset(ValidityMask &mask, const idx_t allocation_count);
-};
-
-} // namespace duckdb