duckdb 0.9.3-dev2.0 → 0.10.0

package/src/duckdb/src/execution/index/fixed_size_allocator.cpp

@@ -5,8 +5,8 @@
 namespace duckdb {
 
 FixedSizeAllocator::FixedSizeAllocator(const idx_t segment_size, BlockManager &block_manager)
-    : block_manager(block_manager), buffer_manager(block_manager.buffer_manager),
-      metadata_manager(block_manager.GetMetadataManager()), segment_size(segment_size), total_segment_count(0) {
+    : block_manager(block_manager), buffer_manager(block_manager.buffer_manager), segment_size(segment_size),
+      total_segment_count(0) {
 
 	if (segment_size > Storage::BLOCK_SIZE - sizeof(validity_t)) {
 		throw InternalException("The maximum segment size of fixed-size allocators is " +
@@ -122,7 +122,7 @@ void FixedSizeAllocator::Reset() {
 	total_segment_count = 0;
 }
 
-idx_t FixedSizeAllocator::GetMemoryUsage() const {
+idx_t FixedSizeAllocator::GetInMemorySize() const {
 	idx_t memory_usage = 0;
 	for (auto &buffer : buffers) {
 		if (buffer.second.InMemory()) {
@@ -208,7 +208,7 @@ bool FixedSizeAllocator::InitializeVacuum() {
 
 	// calculate the vacuum threshold adaptively
 	D_ASSERT(excess_buffer_count < temporary_vacuum_buffers.size());
-	idx_t memory_usage = GetMemoryUsage();
+	idx_t memory_usage = GetInMemorySize();
 	idx_t excess_memory_usage = excess_buffer_count * Storage::BLOCK_SIZE;
 	auto excess_percentage = double(excess_memory_usage) / double(memory_usage);
 	auto threshold = double(VACUUM_THRESHOLD) / 100.0;
@@ -264,31 +264,66 @@ IndexPointer FixedSizeAllocator::VacuumPointer(const IndexPointer ptr) {
 	return new_ptr;
 }
 
-BlockPointer FixedSizeAllocator::Serialize(PartialBlockManager &partial_block_manager, MetadataWriter &writer) {
+FixedSizeAllocatorInfo FixedSizeAllocator::GetInfo() const {
 
+	FixedSizeAllocatorInfo info;
+	info.segment_size = segment_size;
+
+	for (const auto &buffer : buffers) {
+		info.buffer_ids.push_back(buffer.first);
+		info.block_pointers.push_back(buffer.second.block_pointer);
+		info.segment_counts.push_back(buffer.second.segment_count);
+		info.allocation_sizes.push_back(buffer.second.allocation_size);
+	}
+
+	for (auto &buffer_id : buffers_with_free_space) {
+		info.buffers_with_free_space.push_back(buffer_id);
+	}
+
+	return info;
+}
+
+void FixedSizeAllocator::SerializeBuffers(PartialBlockManager &partial_block_manager) {
 	for (auto &buffer : buffers) {
 		buffer.second.Serialize(partial_block_manager, available_segments_per_buffer, segment_size, bitmask_offset);
 	}
+}
 
-	auto block_pointer = writer.GetBlockPointer();
-	writer.Write(segment_size);
-	writer.Write(static_cast<idx_t>(buffers.size()));
-	writer.Write(static_cast<idx_t>(buffers_with_free_space.size()));
+vector<IndexBufferInfo> FixedSizeAllocator::InitSerializationToWAL() {
 
+	vector<IndexBufferInfo> buffer_infos;
 	for (auto &buffer : buffers) {
-		writer.Write(buffer.first);
-		writer.Write(buffer.second.block_pointer);
-		writer.Write(buffer.second.segment_count);
-		writer.Write(buffer.second.allocation_size);
+		buffer.second.SetAllocationSize(available_segments_per_buffer, segment_size, bitmask_offset);
+		buffer_infos.emplace_back(buffer.second.Get(), buffer.second.allocation_size);
 	}
-	for (auto &buffer_id : buffers_with_free_space) {
-		writer.Write(buffer_id);
+	return buffer_infos;
+}
+
+void FixedSizeAllocator::Init(const FixedSizeAllocatorInfo &info) {
+
+	segment_size = info.segment_size;
+	total_segment_count = 0;
+
+	for (idx_t i = 0; i < info.buffer_ids.size(); i++) {
+
+		// read all FixedSizeBuffer data
+		auto buffer_id = info.buffer_ids[i];
+		auto buffer_block_pointer = info.block_pointers[i];
+		auto segment_count = info.segment_counts[i];
+		auto allocation_size = info.allocation_sizes[i];
+
+		// create the FixedSizeBuffer
+		FixedSizeBuffer new_buffer(block_manager, segment_count, allocation_size, buffer_block_pointer);
+		buffers.insert(make_pair(buffer_id, std::move(new_buffer)));
+		total_segment_count += segment_count;
 	}
 
-	return block_pointer;
+	for (const auto &buffer_id : info.buffers_with_free_space) {
+		buffers_with_free_space.insert(buffer_id);
+	}
 }
 
-void FixedSizeAllocator::Deserialize(const BlockPointer &block_pointer) {
+void FixedSizeAllocator::Deserialize(MetadataManager &metadata_manager, const BlockPointer &block_pointer) {
 
 	MetadataReader reader(metadata_manager, block_pointer);
 	segment_size = reader.Read<idx_t>();

package/src/duckdb/src/execution/index/fixed_size_buffer.cpp

@@ -40,7 +40,7 @@ FixedSizeBuffer::FixedSizeBuffer(BlockManager &block_manager)
       block_handle(nullptr) {
 
 	auto &buffer_manager = block_manager.buffer_manager;
-	buffer_handle = buffer_manager.Allocate(Storage::BLOCK_SIZE, false, &block_handle);
+	buffer_handle = buffer_manager.Allocate(MemoryTag::ART_INDEX, Storage::BLOCK_SIZE, false, &block_handle);
 }
 
 FixedSizeBuffer::FixedSizeBuffer(BlockManager &block_manager, const idx_t segment_count, const idx_t allocation_size,
@@ -81,11 +81,8 @@ void FixedSizeBuffer::Serialize(PartialBlockManager &partial_block_manager, cons
 		return;
 	}
 
-	if (dirty) {
-		// the allocation possibly changed
-		auto max_offset = GetMaxOffset(available_segments);
-		allocation_size = max_offset * segment_size + bitmask_offset;
-	}
+	// the allocation possibly changed
+	SetAllocationSize(available_segments, segment_size, bitmask_offset);
 
 	// the buffer is in memory, so we copied it onto a new buffer when pinning
 	D_ASSERT(InMemory() && !OnDisk());
@@ -126,7 +123,6 @@ void FixedSizeBuffer::Serialize(PartialBlockManager &partial_block_manager, cons
 }
 
 void FixedSizeBuffer::Pin() {
-
 	auto &buffer_manager = block_manager.buffer_manager;
 	D_ASSERT(block_pointer.IsValid());
 	D_ASSERT(block_handle && block_handle->BlockId() < MAXIMUM_BLOCK);
@@ -136,7 +132,8 @@ void FixedSizeBuffer::Pin() {
 
 	// we need to copy the (partial) data into a new (not yet disk-backed) buffer handle
 	shared_ptr<BlockHandle> new_block_handle;
-	auto new_buffer_handle = buffer_manager.Allocate(Storage::BLOCK_SIZE, false, &new_block_handle);
+	auto new_buffer_handle =
+	    buffer_manager.Allocate(MemoryTag::ART_INDEX, Storage::BLOCK_SIZE, false, &new_block_handle);
 
 	memcpy(new_buffer_handle.Ptr(), buffer_handle.Ptr() + block_pointer.offset, allocation_size);
 
@@ -195,6 +192,15 @@ uint32_t FixedSizeBuffer::GetOffset(const idx_t bitmask_count) {
 	throw InternalException("Invalid bitmask for FixedSizeAllocator");
 }
 
+void FixedSizeBuffer::SetAllocationSize(const idx_t available_segments, const idx_t segment_size,
+                                        const idx_t bitmask_offset) {
+
+	if (dirty) {
+		auto max_offset = GetMaxOffset(available_segments);
+		allocation_size = max_offset * segment_size + bitmask_offset;
+	}
+}
+
 uint32_t FixedSizeBuffer::GetMaxOffset(const idx_t available_segments) {
 
 	// this function calls Get() on the buffer

package/src/duckdb/src/execution/index/index_type_set.cpp

@@ -0,0 +1,32 @@
+#include "duckdb/execution/index/index_type.hpp"
+#include "duckdb/execution/index/index_type_set.hpp"
+#include "duckdb/execution/index/art/art.hpp"
+
+namespace duckdb {
+
+IndexTypeSet::IndexTypeSet() {
+	// Register the ART index type
+	IndexType art_index_type;
+	art_index_type.name = ART::TYPE_NAME;
+	art_index_type.create_instance = ART::Create;
+	RegisterIndexType(art_index_type);
+}
+
+optional_ptr<IndexType> IndexTypeSet::FindByName(const string &name) {
+	lock_guard<mutex> g(lock);
+	auto entry = functions.find(name);
+	if (entry == functions.end()) {
+		return nullptr;
+	}
+	return &entry->second;
+}
+
+void IndexTypeSet::RegisterIndexType(const IndexType &index_type) {
+	lock_guard<mutex> g(lock);
+	if (functions.find(index_type.name) != functions.end()) {
+		throw CatalogException("Index type with name \"%s\" already exists!", index_type.name.c_str());
+	}
+	functions[index_type.name] = index_type;
+}
+
+} // namespace duckdb

package/src/duckdb/src/execution/index/unknown_index.cpp

@@ -0,0 +1,65 @@
+#include "duckdb/execution/index/unknown_index.hpp"
+#include "duckdb/parser/parsed_data/create_index_info.hpp"
+
+namespace duckdb {
+
+//-------------------------------------------------------------------------------
+// Unknown index
+//-------------------------------------------------------------------------------
+
+UnknownIndex::UnknownIndex(const string &name, const string &index_type, IndexConstraintType index_constraint_type,
+                           const vector<column_t> &column_ids, TableIOManager &table_io_manager,
+                           const vector<unique_ptr<Expression>> &unbound_expressions, AttachedDatabase &db,
+                           const CreateIndexInfo &create_info_p, IndexStorageInfo storage_info_p)
+    : Index(name, index_type, index_constraint_type, column_ids, table_io_manager, unbound_expressions, db),
+      create_info(create_info_p), storage_info(std::move(storage_info_p)) {
+}
+
+string UnknownIndex::GenerateErrorMessage() const {
+	return StringUtil::Format(
+	    R"(Unknown index type "%s" for index "%s". You probably need to load an extension containing this index type)",
+	    index_type.c_str(), name.c_str());
+}
+
+ErrorData UnknownIndex::Append(IndexLock &, DataChunk &, Vector &) {
+	throw NotImplementedException(GenerateErrorMessage());
+}
+void UnknownIndex::VerifyAppend(DataChunk &) {
+	throw NotImplementedException(GenerateErrorMessage());
+}
+void UnknownIndex::VerifyAppend(DataChunk &, ConflictManager &) {
+	throw NotImplementedException(GenerateErrorMessage());
+}
+void UnknownIndex::CommitDrop(IndexLock &) {
+	throw NotImplementedException(GenerateErrorMessage());
+}
+void UnknownIndex::Delete(IndexLock &, DataChunk &, Vector &) {
+	throw NotImplementedException(GenerateErrorMessage());
+}
+ErrorData UnknownIndex::Insert(IndexLock &, DataChunk &, Vector &) {
+	throw NotImplementedException(GenerateErrorMessage());
+}
+IndexStorageInfo UnknownIndex::GetStorageInfo(bool) {
+	throw NotImplementedException(GenerateErrorMessage());
+}
+bool UnknownIndex::MergeIndexes(IndexLock &, Index &) {
+	throw NotImplementedException(GenerateErrorMessage());
+}
+void UnknownIndex::Vacuum(IndexLock &) {
+	throw NotImplementedException(GenerateErrorMessage());
+}
+idx_t UnknownIndex::GetInMemorySize(IndexLock &) {
+	throw NotImplementedException(GenerateErrorMessage());
+}
+void UnknownIndex::CheckConstraintsForChunk(DataChunk &, ConflictManager &) {
+	throw NotImplementedException(GenerateErrorMessage());
+}
+string UnknownIndex::VerifyAndToString(IndexLock &, bool) {
+	throw NotImplementedException(GenerateErrorMessage());
+}
+
+string UnknownIndex::GetConstraintViolationMessage(VerifyExistenceType, idx_t, DataChunk &) {
+	throw NotImplementedException(GenerateErrorMessage());
+}
+
+} // namespace duckdb