duckdb 0.8.2-dev37.0 → 0.8.2-dev3949.0

package/src/duckdb/src/storage/metadata/metadata_manager.cpp

@@ -0,0 +1,267 @@
+#include "duckdb/storage/metadata/metadata_manager.hpp"
+#include "duckdb/storage/buffer_manager.hpp"
+#include "duckdb/storage/buffer/block_handle.hpp"
+
+namespace duckdb {
+
+MetadataManager::MetadataManager(BlockManager &block_manager, BufferManager &buffer_manager)
+    : block_manager(block_manager), buffer_manager(buffer_manager) {
+}
+
+MetadataManager::~MetadataManager() {
+}
+
+MetadataHandle MetadataManager::AllocateHandle() {
+	// check if there is any free space left in an existing block
+	// if not allocate a new bloc
+	block_id_t free_block = INVALID_BLOCK;
+	for (auto &kv : blocks) {
+		auto &block = kv.second;
+		D_ASSERT(kv.first == block.block_id);
+		if (!block.free_blocks.empty()) {
+			free_block = kv.first;
+			break;
+		}
+	}
+	if (free_block == INVALID_BLOCK) {
+		free_block = AllocateNewBlock();
+	}
+	D_ASSERT(free_block != INVALID_BLOCK);
+
+	// select the first free metadata block we can find
+	MetadataPointer pointer;
+	pointer.block_index = free_block;
+	auto &block = blocks[free_block];
+	D_ASSERT(!block.free_blocks.empty());
+	pointer.index = block.free_blocks.back();
+	// mark the block as used
+	block.free_blocks.pop_back();
+	D_ASSERT(pointer.index < METADATA_BLOCK_COUNT);
+	// pin the block
+	return Pin(pointer);
+}
+
+MetadataHandle MetadataManager::Pin(MetadataPointer pointer) {
+	D_ASSERT(pointer.index < METADATA_BLOCK_COUNT);
+	auto &block = blocks[pointer.block_index];
+
+	MetadataHandle handle;
+	handle.pointer.block_index = pointer.block_index;
+	handle.pointer.index = pointer.index;
+	handle.handle = buffer_manager.Pin(block.block);
+	return handle;
+}
+
+block_id_t MetadataManager::AllocateNewBlock() {
+	auto new_block_id = GetNextBlockId();
+
+	MetadataBlock new_block;
+	buffer_manager.Allocate(Storage::BLOCK_SIZE, false, &new_block.block);
+	new_block.block_id = new_block_id;
+	for (idx_t i = 0; i < METADATA_BLOCK_COUNT; i++) {
+		new_block.free_blocks.push_back(METADATA_BLOCK_COUNT - i - 1);
+	}
+	AddBlock(std::move(new_block));
+	return new_block_id;
+}
+
+void MetadataManager::AddBlock(MetadataBlock new_block, bool if_exists) {
+	if (blocks.find(new_block.block_id) != blocks.end()) {
+		if (if_exists) {
+			return;
+		}
+		throw InternalException("Block id with id %llu already exists", new_block.block_id);
+	}
+	blocks[new_block.block_id] = std::move(new_block);
+}
+
+void MetadataManager::AddAndRegisterBlock(MetadataBlock block) {
+	if (block.block) {
+		throw InternalException("Calling AddAndRegisterBlock on block that already exists");
+	}
+	block.block = block_manager.RegisterBlock(block.block_id);
+	AddBlock(std::move(block), true);
+}
+
+MetaBlockPointer MetadataManager::GetDiskPointer(MetadataPointer pointer, uint32_t offset) {
+	idx_t block_pointer = idx_t(pointer.block_index);
+	block_pointer |= idx_t(pointer.index) << 56ULL;
+	return MetaBlockPointer(block_pointer, offset);
+}
+
+block_id_t MetaBlockPointer::GetBlockId() {
+	return block_id_t(block_pointer & ~(idx_t(0xFF) << 56ULL));
+}
+
+uint32_t MetaBlockPointer::GetBlockIndex() {
+	return block_pointer >> 56ULL;
+}
+
+MetadataPointer MetadataManager::FromDiskPointer(MetaBlockPointer pointer) {
+	auto block_id = pointer.GetBlockId();
+	auto index = pointer.GetBlockIndex();
+	auto entry = blocks.find(block_id);
+	if (entry == blocks.end()) { // LCOV_EXCL_START
+		throw InternalException("Failed to load metadata pointer (id %llu, idx %llu, ptr %llu)\n", block_id, index,
+		                        pointer.block_pointer);
+	} // LCOV_EXCL_STOP
+	MetadataPointer result;
+	result.block_index = block_id;
+	result.index = index;
+	return result;
+}
+
+MetadataPointer MetadataManager::RegisterDiskPointer(MetaBlockPointer pointer) {
+	auto block_id = pointer.GetBlockId();
+	MetadataBlock block;
+	block.block_id = block_id;
+	AddAndRegisterBlock(block);
+	return FromDiskPointer(pointer);
+}
+
+BlockPointer MetadataManager::ToBlockPointer(MetaBlockPointer meta_pointer) {
+	BlockPointer result;
+	result.block_id = meta_pointer.GetBlockId();
+	result.offset = meta_pointer.GetBlockIndex() * MetadataManager::METADATA_BLOCK_SIZE + meta_pointer.offset;
+	D_ASSERT(result.offset < MetadataManager::METADATA_BLOCK_SIZE * MetadataManager::METADATA_BLOCK_COUNT);
+	return result;
+}
+
+MetaBlockPointer MetadataManager::FromBlockPointer(BlockPointer block_pointer) {
+	if (!block_pointer.IsValid()) {
+		return MetaBlockPointer();
+	}
+	idx_t index = block_pointer.offset / MetadataManager::METADATA_BLOCK_SIZE;
+	auto offset = block_pointer.offset % MetadataManager::METADATA_BLOCK_SIZE;
+	D_ASSERT(index < MetadataManager::METADATA_BLOCK_COUNT);
+	D_ASSERT(offset < MetadataManager::METADATA_BLOCK_SIZE);
+	MetaBlockPointer result;
+	result.block_pointer = idx_t(block_pointer.block_id) | index << 56ULL;
+	result.offset = offset;
+	return result;
+}
+
+idx_t MetadataManager::BlockCount() {
+	return blocks.size();
+}
+
+void MetadataManager::Flush() {
+	const idx_t total_metadata_size = MetadataManager::METADATA_BLOCK_SIZE * MetadataManager::METADATA_BLOCK_COUNT;
+	// write the blocks of the metadata manager to disk
+	for (auto &kv : blocks) {
+		auto &block = kv.second;
+		auto handle = buffer_manager.Pin(block.block);
+		// zero-initialize any free blocks
+		for (auto free_block : block.free_blocks) {
+			memset(handle.Ptr() + free_block * MetadataManager::METADATA_BLOCK_SIZE, 0,
+			       MetadataManager::METADATA_BLOCK_SIZE);
+		}
+		// there are a few bytes left-over at the end of the block, zero-initialize them
+		memset(handle.Ptr() + total_metadata_size, 0, Storage::BLOCK_SIZE - total_metadata_size);
+		D_ASSERT(kv.first == block.block_id);
+		if (block.block->BlockId() >= MAXIMUM_BLOCK) {
+			// temporary block - convert to persistent
+			block.block = block_manager.ConvertToPersistent(kv.first, std::move(block.block));
+		} else {
+			// already a persistent block - only need to write it
+			D_ASSERT(block.block->BlockId() == block.block_id);
+			block_manager.Write(handle.GetFileBuffer(), block.block_id);
+		}
+	}
+}
+
+void MetadataManager::Serialize(Serializer &serializer) {
+	serializer.Write<uint64_t>(blocks.size());
+	for (auto &kv : blocks) {
+		kv.second.Serialize(serializer);
+	}
+}
+
+void MetadataManager::Deserialize(Deserializer &source) {
+	auto block_count = source.Read<uint64_t>();
+	for (idx_t i = 0; i < block_count; i++) {
+		auto block = MetadataBlock::Deserialize(source);
+		auto entry = blocks.find(block.block_id);
+		if (entry == blocks.end()) {
+			// block does not exist yet
+			AddAndRegisterBlock(std::move(block));
+		} else {
+			// block was already created - only copy over the free list
+			entry->second.free_blocks = std::move(block.free_blocks);
+		}
+	}
+}
+
+void MetadataBlock::Serialize(Serializer &serializer) {
+	serializer.Write<block_id_t>(block_id);
+	serializer.Write<idx_t>(FreeBlocksToInteger());
+}
+
+MetadataBlock MetadataBlock::Deserialize(Deserializer &source) {
+	MetadataBlock result;
+	result.block_id = source.Read<block_id_t>();
+	auto free_list = source.Read<idx_t>();
+	result.FreeBlocksFromInteger(free_list);
+	return result;
+}
+
+idx_t MetadataBlock::FreeBlocksToInteger() {
+	idx_t result = 0;
+	for (idx_t i = 0; i < free_blocks.size(); i++) {
+		D_ASSERT(free_blocks[i] < idx_t(64));
+		idx_t mask = idx_t(1) << idx_t(free_blocks[i]);
+		result |= mask;
+	}
+	return result;
+}
+
+void MetadataBlock::FreeBlocksFromInteger(idx_t free_list) {
+	free_blocks.clear();
+	if (free_list == 0) {
+		return;
+	}
+	for (idx_t i = 64; i > 0; i--) {
+		auto index = i - 1;
+		idx_t mask = idx_t(1) << index;
+		if (free_list & mask) {
+			free_blocks.push_back(index);
+		}
+	}
+}
+
+void MetadataManager::MarkBlocksAsModified() {
+	if (!modified_blocks.empty()) {
+		// for any blocks that were modified in the last checkpoint - set them to free blocks currently
+		for (auto &kv : modified_blocks) {
+			auto block_id = kv.first;
+			idx_t modified_list = kv.second;
+			auto entry = blocks.find(block_id);
+			D_ASSERT(entry != blocks.end());
+			auto &block = entry->second;
+			idx_t current_free_blocks = block.FreeBlocksToInteger();
+			// merge the current set of free blocks with the modified blocks
+			idx_t new_free_blocks = current_free_blocks | modified_list;
+			//			if (new_free_blocks == NumericLimits<idx_t>::Maximum()) {
+			//				// if new free_blocks is all blocks - mark entire block as modified
+			//				blocks.erase(entry);
+			//				block_manager.MarkBlockAsModified(block_id);
+			//			} else {
+			// set the new set of free blocks
+			block.FreeBlocksFromInteger(new_free_blocks);
+			//			}
+		}
+	}
+	modified_blocks.clear();
+	for (auto &kv : blocks) {
+		auto &block = kv.second;
+		idx_t free_list = block.FreeBlocksToInteger();
+		idx_t occupied_list = ~free_list;
+		modified_blocks[block.block_id] = occupied_list;
+	}
+}
+
+block_id_t MetadataManager::GetNextBlockId() {
+	return block_manager.GetFreeBlockId();
+}
+
+} // namespace duckdb

package/src/duckdb/src/storage/metadata/metadata_reader.cpp

@@ -0,0 +1,80 @@
+#include "duckdb/storage/metadata/metadata_reader.hpp"
+
+namespace duckdb {
+
+MetadataReader::MetadataReader(MetadataManager &manager, MetaBlockPointer pointer, BlockReaderType type)
+    : manager(manager), type(type), next_pointer(FromDiskPointer(pointer)), has_next_block(true), index(0), offset(0),
+      next_offset(pointer.offset), capacity(0) {
+}
+
+MetadataReader::MetadataReader(MetadataManager &manager, BlockPointer pointer)
+    : MetadataReader(manager, MetadataManager::FromBlockPointer(pointer)) {
+}
+
+MetadataPointer MetadataReader::FromDiskPointer(MetaBlockPointer pointer) {
+	if (type == BlockReaderType::EXISTING_BLOCKS) {
+		return manager.FromDiskPointer(pointer);
+	} else {
+		return manager.RegisterDiskPointer(pointer);
+	}
+}
+
+MetadataReader::~MetadataReader() {
+}
+
+void MetadataReader::ReadData(data_ptr_t buffer, idx_t read_size) {
+	while (offset + read_size > capacity) {
+		// cannot read entire entry from block
+		// first read what we can from this block
+		idx_t to_read = capacity - offset;
+		if (to_read > 0) {
+			memcpy(buffer, Ptr(), to_read);
+			read_size -= to_read;
+			buffer += to_read;
+			offset += read_size;
+		}
+		// then move to the next block
+		ReadNextBlock();
+	}
+	// we have enough left in this block to read from the buffer
+	memcpy(buffer, Ptr(), read_size);
+	offset += read_size;
+}
+
+MetaBlockPointer MetadataReader::GetMetaBlockPointer() {
+	return manager.GetDiskPointer(block.pointer, offset);
+}
+
+void MetadataReader::ReadNextBlock() {
+	if (!has_next_block) {
+		throw IOException("No more data remaining in MetadataReader");
+	}
+	block = manager.Pin(next_pointer);
+	index = next_pointer.index;
+
+	idx_t next_block = Load<idx_t>(BasePtr());
+	if (next_block == idx_t(-1)) {
+		has_next_block = false;
+	} else {
+		next_pointer = FromDiskPointer(MetaBlockPointer(next_block, 0));
+	}
+	if (next_offset < sizeof(block_id_t)) {
+		next_offset = sizeof(block_id_t);
+	}
+	if (next_offset > MetadataManager::METADATA_BLOCK_SIZE) {
+		throw InternalException("next_offset cannot be bigger than block size");
+	}
+	offset = next_offset;
+	next_offset = sizeof(block_id_t);
+	capacity = MetadataManager::METADATA_BLOCK_SIZE;
+}
+
+data_ptr_t MetadataReader::BasePtr() {
+	return block.handle.Ptr() + index * MetadataManager::METADATA_BLOCK_SIZE;
+}
+
+data_ptr_t MetadataReader::Ptr() {
+	return BasePtr() + offset;
+}
+
+} // namespace duckdb

package/src/duckdb/src/storage/metadata/metadata_writer.cpp

@@ -0,0 +1,86 @@
+#include "duckdb/storage/metadata/metadata_writer.hpp"
+#include "duckdb/storage/block_manager.hpp"
+
+namespace duckdb {
+
+MetadataWriter::MetadataWriter(MetadataManager &manager) : manager(manager), capacity(0), offset(0) {
+}
+
+MetadataWriter::~MetadataWriter() {
+	// If there's an exception during checkpoint, this can get destroyed without
+	// flushing the data...which is fine, because none of the unwritten data
+	// will be referenced.
+	//
+	// Otherwise, we should have explicitly flushed (and thereby nulled the block).
+	D_ASSERT(!block.handle.IsValid() || Exception::UncaughtException());
+}
+
+BlockPointer MetadataWriter::GetBlockPointer() {
+	return MetadataManager::ToBlockPointer(GetMetaBlockPointer());
+}
+
+MetaBlockPointer MetadataWriter::GetMetaBlockPointer() {
+	if (offset >= capacity) {
+		// at the end of the block - fetch the next block
+		NextBlock();
+		D_ASSERT(capacity > 0);
+	}
+	return manager.GetDiskPointer(block.pointer, offset);
+}
+
+MetadataHandle MetadataWriter::NextHandle() {
+	return manager.AllocateHandle();
+}
+
+void MetadataWriter::NextBlock() {
+	// now we need to get a new block id
+	auto new_handle = NextHandle();
+
+	// write the block id of the new block to the start of the current block
+	if (capacity > 0) {
+		Store<idx_t>(manager.GetDiskPointer(new_handle.pointer).block_pointer, BasePtr());
+	}
+	// now update the block id of the block
+	block = std::move(new_handle);
+	current_pointer = block.pointer;
+	offset = sizeof(idx_t);
+	capacity = MetadataManager::METADATA_BLOCK_SIZE;
+	Store<idx_t>(-1, BasePtr());
+}
+
+void MetadataWriter::WriteData(const_data_ptr_t buffer, idx_t write_size) {
+	while (offset + write_size > capacity) {
+		// we need to make a new block
+		// first copy what we can
+		D_ASSERT(offset <= capacity);
+		idx_t copy_amount = capacity - offset;
+		if (copy_amount > 0) {
+			memcpy(Ptr(), buffer, copy_amount);
+			buffer += copy_amount;
+			offset += copy_amount;
+			write_size -= copy_amount;
+		}
+		// move forward to the next block
+		NextBlock();
+	}
+	memcpy(Ptr(), buffer, write_size);
+	offset += write_size;
+}
+
+void MetadataWriter::Flush() {
+	if (offset < capacity) {
+		// clear remaining bytes of block (if any)
+		memset(Ptr(), 0, capacity - offset);
+	}
+	block.handle.Destroy();
+}
+
+data_ptr_t MetadataWriter::BasePtr() {
+	return block.handle.Ptr() + current_pointer.index * MetadataManager::METADATA_BLOCK_SIZE;
+}
+
+data_ptr_t MetadataWriter::Ptr() {
+	return BasePtr() + offset;
+}
+
+} // namespace duckdb

package/src/duckdb/src/storage/serialization/serialize_constraint.cpp

@@ -0,0 +1,98 @@
+//===----------------------------------------------------------------------===//
+// This file is automatically generated by scripts/generate_serialization.py
+// Do not edit this file manually, your changes will be overwritten
+//===----------------------------------------------------------------------===//
+
+#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
+#include "duckdb/parser/constraints/list.hpp"
+
+namespace duckdb {
+
+void Constraint::FormatSerialize(FormatSerializer &serializer) const {
+	serializer.WriteProperty(100, "type", type);
+}
+
+unique_ptr<Constraint> Constraint::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto type = deserializer.ReadProperty<ConstraintType>(100, "type");
+	unique_ptr<Constraint> result;
+	switch (type) {
+	case ConstraintType::CHECK:
+		result = CheckConstraint::FormatDeserialize(deserializer);
+		break;
+	case ConstraintType::FOREIGN_KEY:
+		result = ForeignKeyConstraint::FormatDeserialize(deserializer);
+		break;
+	case ConstraintType::NOT_NULL:
+		result = NotNullConstraint::FormatDeserialize(deserializer);
+		break;
+	case ConstraintType::UNIQUE:
+		result = UniqueConstraint::FormatDeserialize(deserializer);
+		break;
+	default:
+		throw SerializationException("Unsupported type for deserialization of Constraint!");
+	}
+	return result;
+}
+
+void CheckConstraint::FormatSerialize(FormatSerializer &serializer) const {
+	Constraint::FormatSerialize(serializer);
+	serializer.WriteProperty(200, "expression", expression);
+}
+
+unique_ptr<Constraint> CheckConstraint::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto expression = deserializer.ReadProperty<unique_ptr<ParsedExpression>>(200, "expression");
+	auto result = duckdb::unique_ptr<CheckConstraint>(new CheckConstraint(std::move(expression)));
+	return std::move(result);
+}
+
+void ForeignKeyConstraint::FormatSerialize(FormatSerializer &serializer) const {
+	Constraint::FormatSerialize(serializer);
+	serializer.WriteProperty(200, "pk_columns", pk_columns);
+	serializer.WriteProperty(201, "fk_columns", fk_columns);
+	serializer.WriteProperty(202, "fk_type", info.type);
+	serializer.WriteProperty(203, "schema", info.schema);
+	serializer.WriteProperty(204, "table", info.table);
+	serializer.WriteProperty(205, "pk_keys", info.pk_keys);
+	serializer.WriteProperty(206, "fk_keys", info.fk_keys);
+}
+
+unique_ptr<Constraint> ForeignKeyConstraint::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto result = duckdb::unique_ptr<ForeignKeyConstraint>(new ForeignKeyConstraint());
+	deserializer.ReadProperty(200, "pk_columns", result->pk_columns);
+	deserializer.ReadProperty(201, "fk_columns", result->fk_columns);
+	deserializer.ReadProperty(202, "fk_type", result->info.type);
+	deserializer.ReadProperty(203, "schema", result->info.schema);
+	deserializer.ReadProperty(204, "table", result->info.table);
+	deserializer.ReadProperty(205, "pk_keys", result->info.pk_keys);
+	deserializer.ReadProperty(206, "fk_keys", result->info.fk_keys);
+	return std::move(result);
+}
+
+void NotNullConstraint::FormatSerialize(FormatSerializer &serializer) const {
+	Constraint::FormatSerialize(serializer);
+	serializer.WriteProperty(200, "index", index);
+}
+
+unique_ptr<Constraint> NotNullConstraint::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto index = deserializer.ReadProperty<LogicalIndex>(200, "index");
+	auto result = duckdb::unique_ptr<NotNullConstraint>(new NotNullConstraint(index));
+	return std::move(result);
+}
+
+void UniqueConstraint::FormatSerialize(FormatSerializer &serializer) const {
+	Constraint::FormatSerialize(serializer);
+	serializer.WriteProperty(200, "is_primary_key", is_primary_key);
+	serializer.WriteProperty(201, "index", index);
+	serializer.WriteProperty(202, "columns", columns);
+}
+
+unique_ptr<Constraint> UniqueConstraint::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto result = duckdb::unique_ptr<UniqueConstraint>(new UniqueConstraint());
+	deserializer.ReadProperty(200, "is_primary_key", result->is_primary_key);
+	deserializer.ReadProperty(201, "index", result->index);
+	deserializer.ReadProperty(202, "columns", result->columns);
+	return std::move(result);
+}
+
+} // namespace duckdb