duckdb 0.8.2-dev3989.0 → 0.8.2-dev4126.0

package/src/duckdb/src/storage/checkpoint_manager.cpp

@@ -262,7 +262,8 @@ void CheckpointWriter::WriteSchema(SchemaCatalogEntry &schema) {
 void CheckpointReader::ReadSchema(ClientContext &context, MetadataReader &reader) {
 	// read the schema and create it in the catalog
 	auto info = CatalogEntry::Deserialize(reader);
-	// we set create conflict to ignore to ignore the failure of recreating the main schema
+
+	// we set create conflict to IGNORE_ON_CONFLICT, so that we can ignore a failure when recreating the main schema
 	info->on_conflict = OnCreateConflict::IGNORE_ON_CONFLICT;
 	catalog.CreateSchema(context, info->Cast<CreateSchemaInfo>());
 
@@ -336,14 +337,12 @@ void CheckpointReader::ReadSequence(ClientContext &context, MetadataReader &read
 // Indexes
 //===--------------------------------------------------------------------===//
 void CheckpointWriter::WriteIndex(IndexCatalogEntry &index_catalog) {
-	// The index data should already have been written as part of WriteTableData.
-	// Here, we need only serialize the pointer to that data.
-	auto root_offset = index_catalog.index->GetSerializedDataPointer();
+	// we write the index data in WriteTableData
+	// here, we only write the root pointer
+	const auto root_block_pointer = index_catalog.index->GetRootBlockPointer();
 	auto &metadata_writer = GetMetadataWriter();
 	index_catalog.Serialize(metadata_writer);
-	// Serialize the Block id and offset of root node
-	metadata_writer.Write(root_offset.block_id);
-	metadata_writer.Write(root_offset.offset);
+	metadata_writer.Write(root_block_pointer);
 }
 
 void CheckpointReader::ReadIndex(ClientContext &context, MetadataReader &reader) {
@@ -358,10 +357,8 @@ void CheckpointReader::ReadIndex(ClientContext &context, MetadataReader &reader)
 	auto &index_catalog = schema_catalog.CreateIndex(context, index_info, table_catalog)->Cast<DuckIndexEntry>();
 	index_catalog.info = table_catalog.GetStorage().info;
 
-	// we deserialize the index lazily, i.e., we do not need to load any node information
-	// except the root block id and offset
-	auto root_block_id = reader.Read<block_id_t>();
-	auto root_offset = reader.Read<uint32_t>();
+	// we deserialize the index lazily, i.e., we only load the root block pointer
+	const auto index_block_pointer = reader.Read<BlockPointer>();
 
 	// obtain the expressions of the ART from the index metadata
 	vector<unique_ptr<Expression>> unbound_expressions;
@@ -401,9 +398,8 @@ void CheckpointReader::ReadIndex(ClientContext &context, MetadataReader &reader)
 	switch (index_info.index_type) {
 	case IndexType::ART: {
 		auto &storage = table_catalog.GetStorage();
-		auto art =
-		    make_uniq<ART>(index_info.column_ids, TableIOManager::Get(storage), std::move(unbound_expressions),
-		                   index_info.constraint_type, storage.db, nullptr, BlockPointer(root_block_id, root_offset));
+		auto art = make_uniq<ART>(index_info.column_ids, TableIOManager::Get(storage), std::move(unbound_expressions),
+		                          index_info.constraint_type, storage.db, nullptr, index_block_pointer);
 		index_catalog.index = art.get();
 		storage.info->indexes.AddIndex(std::move(art));
 		break;
@@ -450,7 +446,7 @@ void CheckpointReader::ReadTableMacro(ClientContext &context, MetadataReader &re
 // Table Metadata
 //===--------------------------------------------------------------------===//
 void CheckpointWriter::WriteTable(TableCatalogEntry &table) {
-	// write the table meta data
+	// write the table metadata
 	table.Serialize(GetMetadataWriter());
 	// now we need to write the table data.
 	if (auto writer = GetTableDataWriter(table)) {
@@ -483,12 +479,11 @@ void CheckpointReader::ReadTableData(ClientContext &context, MetadataReader &rea
 	data_reader.ReadTableData();
 	bound_info.data->total_rows = reader.Read<idx_t>();
 
-	// Get any indexes block info
-	idx_t num_indexes = reader.Read<idx_t>();
-	for (idx_t i = 0; i < num_indexes; i++) {
-		auto idx_block_id = reader.Read<block_id_t>();
-		auto idx_offset = reader.Read<uint32_t>();
-		bound_info.indexes.emplace_back(idx_block_id, idx_offset);
+	// get the root block pointers of each index
+	idx_t index_count = reader.Read<idx_t>();
+	for (idx_t i = 0; i < index_count; i++) {
+		const auto index_pointer = reader.Read<BlockPointer>();
+		bound_info.indexes.emplace_back(index_pointer);
 	}
 }
 

package/src/duckdb/src/storage/data_table.cpp

@@ -429,13 +429,13 @@ void DataTable::VerifyForeignKeyConstraint(const BoundForeignKeyConstraint &bfk,
 	data_table.info->indexes.VerifyForeignKey(*dst_keys_ptr, dst_chunk, regular_conflicts);
 	regular_conflicts.Finalize();
 	auto &regular_matches = regular_conflicts.Conflicts();
-	// check whether or not the chunk can be inserted or deleted into the referenced table' transaction local storage
-	auto &local_storage = LocalStorage::Get(context, db);
 
+	// check if we can insert the chunk into the reference table's local storage
+	auto &local_storage = LocalStorage::Get(context, db);
 	bool error = IsForeignKeyConstraintError(is_append, count, regular_matches);
 	bool transaction_error = false;
-
 	bool transaction_check = local_storage.Find(data_table);
+
 	if (transaction_check) {
 		auto &transact_index = local_storage.GetIndexes(data_table);
 		transact_index.VerifyForeignKey(*dst_keys_ptr, dst_chunk, transaction_conflicts);

package/src/duckdb/src/storage/index.cpp

@@ -12,7 +12,7 @@ Index::Index(AttachedDatabase &db, IndexType type, TableIOManager &table_io_mana
              IndexConstraintType constraint_type_p)
 
     : type(type), table_io_manager(table_io_manager), column_ids(column_ids_p), constraint_type(constraint_type_p),
-      db(db), buffer_manager(BufferManager::GetBufferManager(db)) {
+      db(db) {
 
 	for (auto &expr : unbound_expressions) {
 		types.push_back(expr->return_type.InternalType());
@@ -39,6 +39,12 @@ PreservedError Index::Append(DataChunk &entries, Vector &row_identifiers) {
 	return Append(state, entries, row_identifiers);
 }
 
+void Index::CommitDrop() {
+	IndexLock index_lock;
+	InitializeLock(index_lock);
+	CommitDrop(index_lock);
+}
+
 void Index::Delete(DataChunk &entries, Vector &row_identifiers) {
 	IndexLock state;
 	InitializeLock(state);

package/src/duckdb/src/storage/metadata/metadata_manager.cpp

@@ -13,7 +13,7 @@ MetadataManager::~MetadataManager() {
 
 MetadataHandle MetadataManager::AllocateHandle() {
 	// check if there is any free space left in an existing block
-	// if not allocate a new bloc
+	// if not allocate a new block
 	block_id_t free_block = INVALID_BLOCK;
 	for (auto &kv : blocks) {
 		auto &block = kv.second;
@@ -230,27 +230,27 @@ void MetadataBlock::FreeBlocksFromInteger(idx_t free_list) {
 }
 
 void MetadataManager::MarkBlocksAsModified() {
-	if (!modified_blocks.empty()) {
-		// for any blocks that were modified in the last checkpoint - set them to free blocks currently
-		for (auto &kv : modified_blocks) {
-			auto block_id = kv.first;
-			idx_t modified_list = kv.second;
-			auto entry = blocks.find(block_id);
-			D_ASSERT(entry != blocks.end());
-			auto &block = entry->second;
-			idx_t current_free_blocks = block.FreeBlocksToInteger();
-			// merge the current set of free blocks with the modified blocks
-			idx_t new_free_blocks = current_free_blocks | modified_list;
-			//			if (new_free_blocks == NumericLimits<idx_t>::Maximum()) {
-			//				// if new free_blocks is all blocks - mark entire block as modified
-			//				blocks.erase(entry);
-			//				block_manager.MarkBlockAsModified(block_id);
-			//			} else {
-			// set the new set of free blocks
-			block.FreeBlocksFromInteger(new_free_blocks);
-			//			}
-		}
+
+	// for any blocks that were modified in the last checkpoint - set them to free blocks currently
+	for (auto &kv : modified_blocks) {
+		auto block_id = kv.first;
+		idx_t modified_list = kv.second;
+		auto entry = blocks.find(block_id);
+		D_ASSERT(entry != blocks.end());
+		auto &block = entry->second;
+		idx_t current_free_blocks = block.FreeBlocksToInteger();
+		// merge the current set of free blocks with the modified blocks
+		idx_t new_free_blocks = current_free_blocks | modified_list;
+		//			if (new_free_blocks == NumericLimits<idx_t>::Maximum()) {
+		//				// if new free_blocks is all blocks - mark entire block as modified
+		//				blocks.erase(entry);
+		//				block_manager.MarkBlockAsModified(block_id);
+		//			} else {
+		// set the new set of free blocks
+		block.FreeBlocksFromInteger(new_free_blocks);
+		//			}
 	}
+
 	modified_blocks.clear();
 	for (auto &kv : blocks) {
 		auto &block = kv.second;

package/src/duckdb/src/storage/standard_buffer_manager.cpp

@@ -3,11 +3,11 @@
 #include "duckdb/common/allocator.hpp"
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/set.hpp"
-#include "duckdb/storage/in_memory_block_manager.hpp"
-#include "duckdb/storage/storage_manager.hpp"
 #include "duckdb/main/attached_database.hpp"
 #include "duckdb/main/database.hpp"
 #include "duckdb/storage/buffer/buffer_pool.hpp"
+#include "duckdb/storage/in_memory_block_manager.hpp"
+#include "duckdb/storage/storage_manager.hpp"
 
 namespace duckdb {
 
@@ -77,14 +77,6 @@ idx_t StandardBufferManager::GetMaxMemory() const {
 	return buffer_pool.GetMaxMemory();
 }
 
-// POTENTIALLY PROBLEMATIC
-// void StandardBufferManager::IncreaseUsedMemory(idx_t size, bool unsafe) {
-//	if (!unsafe && buffer_pool.GetUsedMemory() + size > buffer_pool.GetMaxMemory()) {
-//		throw OutOfMemoryException("Failed to allocate data of size %lld%s", size, InMemoryWarning());
-//	}
-//	buffer_pool.IncreaseUsedMemory(size);
-//}
-
 template <typename... ARGS>
 TempBufferPoolReservation StandardBufferManager::EvictBlocksOrThrow(idx_t memory_delta, unique_ptr<FileBuffer> *buffer,
                                                                     ARGS... args) {
@@ -329,12 +321,13 @@ private:
 };
 
 class TemporaryFileHandle {
-	constexpr static idx_t MAX_ALLOWED_INDEX = 4000;
+	constexpr static idx_t MAX_ALLOWED_INDEX_BASE = 4000;
 
 public:
-	TemporaryFileHandle(DatabaseInstance &db, const string &temp_directory, idx_t index)
-	    : db(db), file_index(index), path(FileSystem::GetFileSystem(db).JoinPath(
-	                                     temp_directory, "duckdb_temp_storage-" + to_string(index) + ".tmp")) {
+	TemporaryFileHandle(idx_t temp_file_count, DatabaseInstance &db, const string &temp_directory, idx_t index)
+	    : max_allowed_index((1 << temp_file_count) * MAX_ALLOWED_INDEX_BASE), db(db), file_index(index),
+	      path(FileSystem::GetFileSystem(db).JoinPath(temp_directory,
+	                                                  "duckdb_temp_storage-" + to_string(index) + ".tmp")) {
 	}
 
 public:
@@ -348,7 +341,7 @@ public:
 public:
 	TemporaryFileIndex TryGetBlockIndex() {
 		TemporaryFileLock lock(file_lock);
-		if (index_manager.GetMaxIndex() >= MAX_ALLOWED_INDEX && index_manager.HasFreeBlocks()) {
+		if (index_manager.GetMaxIndex() >= max_allowed_index && index_manager.HasFreeBlocks()) {
 			// file is at capacity
 			return TemporaryFileIndex();
 		}
@@ -426,6 +419,7 @@ private:
 	}
 
 private:
+	const idx_t max_allowed_index;
 	DatabaseInstance &db;
 	unique_ptr<FileHandle> handle;
 	idx_t file_index;
@@ -467,7 +461,7 @@ public:
 			if (!handle) {
 				// no existing handle to write to; we need to create & open a new file
 				auto new_file_index = index_manager.GetNewBlockIndex();
-				auto new_file = make_uniq<TemporaryFileHandle>(db, temp_directory, new_file_index);
+				auto new_file = make_uniq<TemporaryFileHandle>(files.size(), db, temp_directory, new_file_index);
 				handle = new_file.get();
 				files[new_file_index] = std::move(new_file);
 

package/src/duckdb/src/storage/storage_info.cpp

@@ -2,7 +2,7 @@
 
 namespace duckdb {
 
-const uint64_t VERSION_NUMBER = 57;
+const uint64_t VERSION_NUMBER = 58;
 
 struct StorageVersionInfo {
 	const char *version_name;

package/src/duckdb/src/storage/table_index_list.cpp

@@ -54,7 +54,7 @@ void TableIndexList::VerifyForeignKey(const vector<PhysicalIndex> &fk_keys, Data
 	                   ? ForeignKeyType::FK_TYPE_PRIMARY_KEY_TABLE
 	                   : ForeignKeyType::FK_TYPE_FOREIGN_KEY_TABLE;
 
-	// check whether or not the chunk can be inserted or deleted into the referenced table' storage
+	// check whether the chunk can be inserted or deleted into the referenced table storage
 	auto index = FindForeignKeyIndex(fk_keys, fk_type);
 	if (!index) {
 		throw InternalException("Internal Foreign Key error: could not find index to verify...");

package/src/duckdb/src/transaction/commit_state.cpp

@@ -1,5 +1,6 @@
 #include "duckdb/transaction/commit_state.hpp"
 
+#include "duckdb/catalog/catalog_entry/duck_index_entry.hpp"
 #include "duckdb/catalog/catalog_entry/duck_table_entry.hpp"
 #include "duckdb/catalog/catalog_entry/type_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_set.hpp"
@@ -124,9 +125,12 @@ void CommitState::WriteCatalogEntry(CatalogEntry &entry, data_ptr_t dataptr) {
 		case CatalogType::TYPE_ENTRY:
 			log->WriteDropType(entry.Cast<TypeCatalogEntry>());
 			break;
-		case CatalogType::INDEX_ENTRY:
+		case CatalogType::INDEX_ENTRY: {
+			auto &index_entry = entry.Cast<DuckIndexEntry>();
+			index_entry.CommitDrop();
 			log->WriteDropIndex(entry.Cast<IndexCatalogEntry>());
 			break;
+		}
 		case CatalogType::PREPARED_STATEMENT:
 		case CatalogType::SCALAR_FUNCTION_ENTRY:
 			// do nothing, indexes/prepared statements/functions aren't persisted to disk

package/src/duckdb/third_party/mbedtls/include/mbedtls_wrapper.hpp

@@ -17,8 +17,10 @@ public:
 	static std::string ComputeSha256Hash(const std::string& file_content);
 	static bool IsValidSha256Signature(const std::string& pubkey, const std::string& signature, const std::string& sha256_hash);
 	static void Hmac256(const char* key, size_t key_len, const char* message, size_t message_len, char* out);
+	static void ToBase16(char *in, char *out, size_t len);
 
-	static constexpr size_t SHA256_HASH_BYTES = 32;
+	static constexpr size_t SHA256_HASH_LENGTH_BYTES = 32;
+	static constexpr size_t SHA256_HASH_LENGTH_TEXT = 64;
 
 	class SHA256State {
 	public:
@@ -26,6 +28,7 @@ public:
 		~SHA256State();
 		void AddString(const std::string & str);
 		std::string Finalize();
+		void FinishHex(char *out);
 	private:
 		void *sha_context;
 	};

package/src/duckdb/third_party/mbedtls/mbedtls_wrapper.cpp

@@ -38,7 +38,7 @@ void MbedTlsWrapper::ComputeSha256Hash(const char* in, size_t in_len, char* out)
 
 string MbedTlsWrapper::ComputeSha256Hash(const string& file_content) {
 	string hash;
-	hash.resize(MbedTlsWrapper::SHA256_HASH_BYTES);
+	hash.resize(MbedTlsWrapper::SHA256_HASH_LENGTH_BYTES);
 	ComputeSha256Hash(file_content.data(), file_content.size(), (char*)hash.data());
 	return hash;
 }
@@ -83,6 +83,17 @@ void MbedTlsWrapper::Hmac256(const char* key, size_t key_len, const char* messag
 	mbedtls_md_free(&hmac_ctx);
 }
 
+void MbedTlsWrapper::ToBase16(char *in, char *out, size_t len) {
+	static char const HEX_CODES[] = "0123456789abcdef";
+	size_t i, j;
+
+	for (j = i = 0; i < len; i++) {
+		int a = in[i];
+		out[j++] = HEX_CODES[(a >> 4) & 0xf];
+		out[j++] = HEX_CODES[a & 0xf];
+	}
+}
+
 MbedTlsWrapper::SHA256State::SHA256State() : sha_context(new mbedtls_sha256_context()) {
 	mbedtls_sha256_init((mbedtls_sha256_context*)sha_context);
 
@@ -104,7 +115,7 @@ void MbedTlsWrapper::SHA256State::AddString(const std::string & str) {
 
 std::string MbedTlsWrapper::SHA256State::Finalize() {
 	string hash;
-	hash.resize(MbedTlsWrapper::SHA256_HASH_BYTES);
+	hash.resize(MbedTlsWrapper::SHA256_HASH_LENGTH_BYTES);
 
 	if (mbedtls_sha256_finish((mbedtls_sha256_context*)sha_context, (unsigned char*)hash.data())) {
 		throw std::runtime_error("SHA256 Error");
@@ -112,3 +123,14 @@ std::string MbedTlsWrapper::SHA256State::Finalize() {
 
 	return hash;
 }
+
+void MbedTlsWrapper::SHA256State::FinishHex(char *out) {
+	string hash;
+	hash.resize(MbedTlsWrapper::SHA256_HASH_LENGTH_BYTES);
+
+	if (mbedtls_sha256_finish((mbedtls_sha256_context *)sha_context, (unsigned char *)hash.data())) {
+		throw std::runtime_error("SHA256 Error");
+	}
+
+	MbedTlsWrapper::ToBase16(const_cast<char *>(hash.c_str()), out, MbedTlsWrapper::SHA256_HASH_LENGTH_BYTES);
+}

package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp

@@ -348,9 +348,11 @@
 
 #include "extension/icu/third_party/icu/i18n/wintzimpl.cpp"
 
-#include "extension/icu/third_party/icu/i18n/double-conversion-fast-dtoa.cpp"
+#include "extension/icu/third_party/icu/i18n/double-conversion-strtod.cpp"
 
-#include "extension/icu/third_party/icu/i18n/double-conversion-cached-powers.cpp"
+#include "extension/icu/third_party/icu/i18n/double-conversion-double-to-string.cpp"
+
+#include "extension/icu/third_party/icu/i18n/double-conversion-fast-dtoa.cpp"
 
 #include "extension/icu/third_party/icu/i18n/double-conversion-string-to-double.cpp"
 
@@ -358,7 +360,5 @@
 
 #include "extension/icu/third_party/icu/i18n/double-conversion-bignum.cpp"
 
-#include "extension/icu/third_party/icu/i18n/double-conversion-strtod.cpp"
-
-#include "extension/icu/third_party/icu/i18n/double-conversion-double-to-string.cpp"
+#include "extension/icu/third_party/icu/i18n/double-conversion-cached-powers.cpp"
 

package/src/duckdb/ub_src_core_functions_scalar_list.cpp

@@ -10,5 +10,11 @@
 
 #include "src/core_functions/scalar/list/list_sort.cpp"
 
+#include "src/core_functions/scalar/list/list_distance.cpp"
+
+#include "src/core_functions/scalar/list/list_cosine_similarity.cpp"
+
+#include "src/core_functions/scalar/list/list_inner_product.cpp"
+
 #include "src/core_functions/scalar/list/range.cpp"
 

package/src/duckdb/ub_src_core_functions_scalar_string.cpp

@@ -34,6 +34,8 @@
 
 #include "src/core_functions/scalar/string/reverse.cpp"
 
+#include "src/core_functions/scalar/string/sha256.cpp"
+
 #include "src/core_functions/scalar/string/starts_with.cpp"
 
 #include "src/core_functions/scalar/string/string_split.cpp"

package/src/duckdb/ub_src_execution_index.cpp

@@ -0,0 +1,4 @@
+#include "src/execution/index/fixed_size_allocator.cpp"
+
+#include "src/execution/index/fixed_size_buffer.cpp"
+

package/src/duckdb/ub_src_execution_index_art.cpp

@@ -2,8 +2,6 @@
 
 #include "src/execution/index/art/node.cpp"
 
-#include "src/execution/index/art/fixed_size_allocator.cpp"
-
 #include "src/execution/index/art/iterator.cpp"
 
 #include "src/execution/index/art/leaf.cpp"

package/src/duckdb/src/execution/index/art/fixed_size_allocator.cpp

@@ -1,238 +0,0 @@
-#include "duckdb/execution/index/art/fixed_size_allocator.hpp"
-
-namespace duckdb {
-
-constexpr idx_t FixedSizeAllocator::BASE[];
-constexpr uint8_t FixedSizeAllocator::SHIFT[];
-
-FixedSizeAllocator::FixedSizeAllocator(const idx_t allocation_size, Allocator &allocator)
-    : allocation_size(allocation_size), total_allocations(0), allocator(allocator) {
-
-	// calculate how many allocations fit into one buffer
-
-	idx_t bits_per_value = sizeof(validity_t) * 8;
-	idx_t curr_alloc_size = 0;
-
-	bitmask_count = 0;
-	allocations_per_buffer = 0;
-
-	while (curr_alloc_size < BUFFER_ALLOC_SIZE) {
-		if (!bitmask_count || (bitmask_count * bits_per_value) % allocations_per_buffer == 0) {
-			bitmask_count++;
-			curr_alloc_size += sizeof(validity_t);
-		}
-
-		auto remaining_alloc_size = BUFFER_ALLOC_SIZE - curr_alloc_size;
-		auto remaining_allocations = MinValue(remaining_alloc_size / allocation_size, bits_per_value);
-
-		if (remaining_allocations == 0) {
-			break;
-		}
-
-		allocations_per_buffer += remaining_allocations;
-		curr_alloc_size += remaining_allocations * allocation_size;
-	}
-
-	allocation_offset = bitmask_count * sizeof(validity_t);
-}
-
-FixedSizeAllocator::~FixedSizeAllocator() {
-	for (auto &buffer : buffers) {
-		allocator.FreeData(buffer.ptr, BUFFER_ALLOC_SIZE);
-	}
-}
-
-Node FixedSizeAllocator::New() {
-
-	// no more free pointers
-	if (buffers_with_free_space.empty()) {
-
-		// add a new buffer
-		idx_t buffer_id = buffers.size();
-		D_ASSERT(buffer_id <= (uint32_t)DConstants::INVALID_INDEX);
-		auto buffer = allocator.AllocateData(BUFFER_ALLOC_SIZE);
-		buffers.emplace_back(buffer, 0);
-		buffers_with_free_space.insert(buffer_id);
-
-		// set the bitmask
-		ValidityMask mask(reinterpret_cast<validity_t *>(buffer));
-		mask.SetAllValid(allocations_per_buffer);
-	}
-
-	// return a pointer
-	D_ASSERT(!buffers_with_free_space.empty());
-	auto buffer_id = (uint32_t)*buffers_with_free_space.begin();
-
-	auto bitmask_ptr = reinterpret_cast<validity_t *>(buffers[buffer_id].ptr);
-	ValidityMask mask(bitmask_ptr);
-	auto offset = GetOffset(mask, buffers[buffer_id].allocation_count);
-
-	buffers[buffer_id].allocation_count++;
-	total_allocations++;
-	if (buffers[buffer_id].allocation_count == allocations_per_buffer) {
-		buffers_with_free_space.erase(buffer_id);
-	}
-
-	return Node(buffer_id, offset);
-}
-
-void FixedSizeAllocator::Free(const Node ptr) {
-	auto bitmask_ptr = reinterpret_cast<validity_t *>(buffers[ptr.GetBufferId()].ptr);
-	ValidityMask mask(bitmask_ptr);
-	D_ASSERT(!mask.RowIsValid(ptr.GetOffset()));
-	mask.SetValid(ptr.GetOffset());
-	buffers_with_free_space.insert(ptr.GetBufferId());
-
-	D_ASSERT(total_allocations > 0);
-	D_ASSERT(buffers[ptr.GetBufferId()].allocation_count > 0);
-	buffers[ptr.GetBufferId()].allocation_count--;
-	total_allocations--;
-}
-
-void FixedSizeAllocator::Reset() {
-
-	for (auto &buffer : buffers) {
-		allocator.FreeData(buffer.ptr, BUFFER_ALLOC_SIZE);
-	}
-	buffers.clear();
-	buffers_with_free_space.clear();
-	total_allocations = 0;
-}
-
-void FixedSizeAllocator::Merge(FixedSizeAllocator &other) {
-
-	D_ASSERT(allocation_size == other.allocation_size);
-
-	// remember the buffer count and merge the buffers
-	idx_t buffer_count = buffers.size();
-	for (auto &buffer : other.buffers) {
-		buffers.push_back(buffer);
-	}
-	other.buffers.clear();
-
-	// merge the buffers with free spaces
-	for (auto &buffer_id : other.buffers_with_free_space) {
-		buffers_with_free_space.insert(buffer_id + buffer_count);
-	}
-	other.buffers_with_free_space.clear();
-
-	// add the total allocations
-	total_allocations += other.total_allocations;
-}
-
-bool FixedSizeAllocator::InitializeVacuum() {
-
-	if (total_allocations == 0) {
-		Reset();
-		return false;
-	}
-
-	auto total_available_allocations = allocations_per_buffer * buffers.size();
-	D_ASSERT(total_available_allocations >= total_allocations);
-	auto total_free_positions = total_available_allocations - total_allocations;
-
-	// vacuum_count buffers can be freed
-	auto vacuum_count = total_free_positions / allocations_per_buffer;
-
-	// calculate the vacuum threshold adaptively
-	D_ASSERT(vacuum_count < buffers.size());
-	idx_t memory_usage = GetMemoryUsage();
-	idx_t excess_memory_usage = vacuum_count * BUFFER_ALLOC_SIZE;
-	auto excess_percentage = (double)excess_memory_usage / (double)memory_usage;
-	auto threshold = (double)VACUUM_THRESHOLD / 100.0;
-	if (excess_percentage < threshold) {
-		return false;
-	}
-
-	min_vacuum_buffer_id = buffers.size() - vacuum_count;
-
-	// remove all invalid buffers from the available buffer list to ensure that we do not reuse them
-	auto it = buffers_with_free_space.begin();
-	while (it != buffers_with_free_space.end()) {
-		if (*it >= min_vacuum_buffer_id) {
-			it = buffers_with_free_space.erase(it);
-		} else {
-			it++;
-		}
-	}
-
-	return true;
-}
-
-void FixedSizeAllocator::FinalizeVacuum() {
-
-	// free all (now unused) buffers
-	while (min_vacuum_buffer_id < buffers.size()) {
-		allocator.FreeData(buffers.back().ptr, BUFFER_ALLOC_SIZE);
-		buffers.pop_back();
-	}
-}
-
-Node FixedSizeAllocator::VacuumPointer(const Node ptr) {
-
-	// we do not need to adjust the bitmask of the old buffer, because we will free the entire
-	// buffer after the vacuum operation
-
-	auto new_ptr = New();
-
-	// new increases the allocation count
-	total_allocations--;
-
-	memcpy(Get(new_ptr), Get(ptr), allocation_size);
-	return new_ptr;
-}
-
-void FixedSizeAllocator::Verify() const {
-#ifdef DEBUG
-	auto total_available_allocations = allocations_per_buffer * buffers.size();
-	D_ASSERT(total_available_allocations >= total_allocations);
-	D_ASSERT(buffers.size() >= buffers_with_free_space.size());
-#endif
-}
-
-uint32_t FixedSizeAllocator::GetOffset(ValidityMask &mask, const idx_t allocation_count) {
-
-	auto data = mask.GetData();
-
-	// fills up a buffer sequentially before searching for free bits
-	if (mask.RowIsValid(allocation_count)) {
-		mask.SetInvalid(allocation_count);
-		return allocation_count;
-	}
-
-	// get an entry with free bits
-	for (idx_t entry_idx = 0; entry_idx < bitmask_count; entry_idx++) {
-		if (data[entry_idx] != 0) {
-
-			// find the position of the free bit
-			auto entry = data[entry_idx];
-			idx_t first_valid_bit = 0;
-
-			// this loop finds the position of the rightmost set bit in entry and stores it
-			// in first_valid_bit
-			for (idx_t i = 0; i < 6; i++) {
-				// set the left half of the bits of this level to zero and test if the entry is still not zero
-				if (entry & BASE[i]) {
-					// first valid bit is in the rightmost s[i] bits
-					// permanently set the left half of the bits to zero
-					entry &= BASE[i];
-				} else {
-					// first valid bit is in the leftmost s[i] bits
-					// shift by s[i] for the next iteration and add s[i] to the position of the rightmost set bit
-					entry >>= SHIFT[i];
-					first_valid_bit += SHIFT[i];
-				}
-			}
-			D_ASSERT(entry);
-
-			auto prev_bits = entry_idx * sizeof(validity_t) * 8;
-			D_ASSERT(mask.RowIsValid(prev_bits + first_valid_bit));
-			mask.SetInvalid(prev_bits + first_valid_bit);
-			return (prev_bits + first_valid_bit);
-		}
-	}
-
-	throw InternalException("Invalid bitmask of FixedSizeAllocator");
-}
-
-} // namespace duckdb