duckdb 0.8.2-dev4142.0 → 0.8.2-dev4314.0

package/src/duckdb/src/storage/partial_block_manager.cpp

@@ -2,6 +2,38 @@
 
 namespace duckdb {
 
+//===--------------------------------------------------------------------===//
+// PartialBlock
+//===--------------------------------------------------------------------===//
+
+PartialBlock::PartialBlock(PartialBlockState state, BlockManager &block_manager,
+                           const shared_ptr<BlockHandle> &block_handle)
+    : state(state), block_manager(block_manager), block_handle(block_handle) {
+}
+
+void PartialBlock::AddUninitializedRegion(idx_t start, idx_t end) {
+	uninitialized_regions.push_back({start, end});
+}
+
+void PartialBlock::FlushInternal(const idx_t free_space_left) {
+
+	// ensure that we do not leak any data
+	if (free_space_left > 0 || !uninitialized_regions.empty()) {
+		auto buffer_handle = block_manager.buffer_manager.Pin(block_handle);
+
+		// memset any uninitialized regions
+		for (auto &uninitialized : uninitialized_regions) {
+			memset(buffer_handle.Ptr() + uninitialized.start, 0, uninitialized.end - uninitialized.start);
+		}
+		// memset any free space at the end of the block to 0 prior to writing to disk
+		memset(buffer_handle.Ptr() + Storage::BLOCK_SIZE - free_space_left, 0, free_space_left);
+	}
+}
+
+//===--------------------------------------------------------------------===//
+// PartialBlockManager
+//===--------------------------------------------------------------------===//
+
 PartialBlockManager::PartialBlockManager(BlockManager &block_manager, CheckpointType checkpoint_type,
                                          uint32_t max_partial_block_size, uint32_t max_use_count)
     : block_manager(block_manager), checkpoint_type(checkpoint_type), max_partial_block_size(max_partial_block_size),
@@ -9,9 +41,7 @@ PartialBlockManager::PartialBlockManager(BlockManager &block_manager, Checkpoint
 }
 PartialBlockManager::~PartialBlockManager() {
 }
-//===--------------------------------------------------------------------===//
-// Partial Blocks
-//===--------------------------------------------------------------------===//
+
 PartialBlockAllocation PartialBlockManager::GetBlockAllocation(uint32_t segment_size) {
 	PartialBlockAllocation allocation;
 	allocation.block_manager = &block_manager;
@@ -47,7 +77,7 @@ void PartialBlockManager::AllocateBlock(PartialBlockState &state, uint32_t segme
 		state.block_id = INVALID_BLOCK;
 	}
 	state.block_size = Storage::BLOCK_SIZE;
-	state.offset_in_block = 0;
+	state.offset = 0;
 	state.block_use_count = 1;
 }
 
@@ -60,21 +90,22 @@ bool PartialBlockManager::GetPartialBlock(idx_t segment_size, unique_ptr<Partial
 	partial_block = std::move(entry->second);
 	partially_filled_blocks.erase(entry);
 
-	D_ASSERT(partial_block->state.offset_in_block > 0);
-	D_ASSERT(ValueIsAligned(partial_block->state.offset_in_block));
+	D_ASSERT(partial_block->state.offset > 0);
+	D_ASSERT(ValueIsAligned(partial_block->state.offset));
 	return true;
 }
 
 void PartialBlockManager::RegisterPartialBlock(PartialBlockAllocation &&allocation) {
 	auto &state = allocation.partial_block->state;
+	D_ASSERT(checkpoint_type != CheckpointType::FULL_CHECKPOINT || state.block_id >= 0);
 	if (state.block_use_count < max_use_count) {
-		auto unaligned_size = allocation.allocation_size + state.offset_in_block;
+		auto unaligned_size = allocation.allocation_size + state.offset;
 		auto new_size = AlignValue(unaligned_size);
 		if (new_size != unaligned_size) {
 			// register the uninitialized region so we can correctly initialize it before writing to disk
 			allocation.partial_block->AddUninitializedRegion(unaligned_size, new_size);
 		}
-		state.offset_in_block = new_size;
+		state.offset = new_size;
 		auto new_space_left = state.block_size - new_size;
 		// check if the block is STILL partially filled after adding the segment_size
 		if (new_space_left >= Storage::BLOCK_SIZE - max_partial_block_size) {
@@ -82,7 +113,7 @@ void PartialBlockManager::RegisterPartialBlock(PartialBlockAllocation &&allocati
 			partially_filled_blocks.insert(make_pair(new_space_left, std::move(allocation.partial_block)));
 		}
 	}
-	idx_t free_space = state.block_size - state.offset_in_block;
+	idx_t free_space = state.block_size - state.offset;
 	auto block_to_free = std::move(allocation.partial_block);
 	if (!block_to_free && partially_filled_blocks.size() > MAX_BLOCK_MAP_SIZE) {
 		// Free the page with the least space free.
@@ -98,10 +129,6 @@ void PartialBlockManager::RegisterPartialBlock(PartialBlockAllocation &&allocati
 	}
 }
 
-void PartialBlock::Merge(PartialBlock &other, idx_t offset, idx_t other_size) {
-	throw InternalException("PartialBlock::Merge not implemented for this block type");
-}
-
 void PartialBlockManager::Merge(PartialBlockManager &other) {
 	if (&other == this) {
 		throw InternalException("Cannot merge into itself");
@@ -117,10 +144,10 @@ void PartialBlockManager::Merge(PartialBlockManager &other) {
 			// we can merge this block into an existing block - merge them
 			// merge blocks
 			auto allocation = GetBlockAllocation(used_space);
-			allocation.partial_block->Merge(*e.second, allocation.state.offset_in_block, used_space);
+			allocation.partial_block->Merge(*e.second, allocation.state.offset, used_space);
 
 			// re-register the partial block
-			allocation.state.offset_in_block += used_space;
+			allocation.state.offset += used_space;
 			RegisterPartialBlock(std::move(allocation));
 		} else {
 			// we cannot merge this block - append it directly to the current block manager

package/src/duckdb/src/storage/serialization/serialize_constraint.cpp

@@ -3,31 +3,31 @@
 // Do not edit this file manually, your changes will be overwritten
 //===----------------------------------------------------------------------===//
 
-#include "duckdb/common/serializer/format_serializer.hpp"
-#include "duckdb/common/serializer/format_deserializer.hpp"
+#include "duckdb/common/serializer/serializer.hpp"
+#include "duckdb/common/serializer/deserializer.hpp"
 #include "duckdb/parser/constraints/list.hpp"
 
 namespace duckdb {
 
-void Constraint::FormatSerialize(FormatSerializer &serializer) const {
+void Constraint::Serialize(Serializer &serializer) const {
 	serializer.WriteProperty(100, "type", type);
 }
 
-unique_ptr<Constraint> Constraint::FormatDeserialize(FormatDeserializer &deserializer) {
+unique_ptr<Constraint> Constraint::Deserialize(Deserializer &deserializer) {
 	auto type = deserializer.ReadProperty<ConstraintType>(100, "type");
 	unique_ptr<Constraint> result;
 	switch (type) {
 	case ConstraintType::CHECK:
-		result = CheckConstraint::FormatDeserialize(deserializer);
+		result = CheckConstraint::Deserialize(deserializer);
 		break;
 	case ConstraintType::FOREIGN_KEY:
-		result = ForeignKeyConstraint::FormatDeserialize(deserializer);
+		result = ForeignKeyConstraint::Deserialize(deserializer);
 		break;
 	case ConstraintType::NOT_NULL:
-		result = NotNullConstraint::FormatDeserialize(deserializer);
+		result = NotNullConstraint::Deserialize(deserializer);
 		break;
 	case ConstraintType::UNIQUE:
-		result = UniqueConstraint::FormatDeserialize(deserializer);
+		result = UniqueConstraint::Deserialize(deserializer);
 		break;
 	default:
 		throw SerializationException("Unsupported type for deserialization of Constraint!");
@@ -35,19 +35,19 @@ unique_ptr<Constraint> Constraint::FormatDeserialize(FormatDeserializer &deseria
 	return result;
 }
 
-void CheckConstraint::FormatSerialize(FormatSerializer &serializer) const {
-	Constraint::FormatSerialize(serializer);
+void CheckConstraint::Serialize(Serializer &serializer) const {
+	Constraint::Serialize(serializer);
 	serializer.WriteProperty(200, "expression", expression);
 }
 
-unique_ptr<Constraint> CheckConstraint::FormatDeserialize(FormatDeserializer &deserializer) {
+unique_ptr<Constraint> CheckConstraint::Deserialize(Deserializer &deserializer) {
 	auto expression = deserializer.ReadProperty<unique_ptr<ParsedExpression>>(200, "expression");
 	auto result = duckdb::unique_ptr<CheckConstraint>(new CheckConstraint(std::move(expression)));
 	return std::move(result);
 }
 
-void ForeignKeyConstraint::FormatSerialize(FormatSerializer &serializer) const {
-	Constraint::FormatSerialize(serializer);
+void ForeignKeyConstraint::Serialize(Serializer &serializer) const {
+	Constraint::Serialize(serializer);
 	serializer.WriteProperty(200, "pk_columns", pk_columns);
 	serializer.WriteProperty(201, "fk_columns", fk_columns);
 	serializer.WriteProperty(202, "fk_type", info.type);
@@ -57,7 +57,7 @@ void ForeignKeyConstraint::FormatSerialize(FormatSerializer &serializer) const {
 	serializer.WriteProperty(206, "fk_keys", info.fk_keys);
 }
 
-unique_ptr<Constraint> ForeignKeyConstraint::FormatDeserialize(FormatDeserializer &deserializer) {
+unique_ptr<Constraint> ForeignKeyConstraint::Deserialize(Deserializer &deserializer) {
 	auto result = duckdb::unique_ptr<ForeignKeyConstraint>(new ForeignKeyConstraint());
 	deserializer.ReadProperty(200, "pk_columns", result->pk_columns);
 	deserializer.ReadProperty(201, "fk_columns", result->fk_columns);
@@ -69,25 +69,25 @@ unique_ptr<Constraint> ForeignKeyConstraint::FormatDeserialize(FormatDeserialize
 	return std::move(result);
 }
 
-void NotNullConstraint::FormatSerialize(FormatSerializer &serializer) const {
-	Constraint::FormatSerialize(serializer);
+void NotNullConstraint::Serialize(Serializer &serializer) const {
+	Constraint::Serialize(serializer);
 	serializer.WriteProperty(200, "index", index);
 }
 
-unique_ptr<Constraint> NotNullConstraint::FormatDeserialize(FormatDeserializer &deserializer) {
+unique_ptr<Constraint> NotNullConstraint::Deserialize(Deserializer &deserializer) {
 	auto index = deserializer.ReadProperty<LogicalIndex>(200, "index");
 	auto result = duckdb::unique_ptr<NotNullConstraint>(new NotNullConstraint(index));
 	return std::move(result);
 }
 
-void UniqueConstraint::FormatSerialize(FormatSerializer &serializer) const {
-	Constraint::FormatSerialize(serializer);
+void UniqueConstraint::Serialize(Serializer &serializer) const {
+	Constraint::Serialize(serializer);
 	serializer.WriteProperty(200, "is_primary_key", is_primary_key);
 	serializer.WriteProperty(201, "index", index);
 	serializer.WriteProperty(202, "columns", columns);
 }
 
-unique_ptr<Constraint> UniqueConstraint::FormatDeserialize(FormatDeserializer &deserializer) {
+unique_ptr<Constraint> UniqueConstraint::Deserialize(Deserializer &deserializer) {
 	auto result = duckdb::unique_ptr<UniqueConstraint>(new UniqueConstraint());
 	deserializer.ReadProperty(200, "is_primary_key", result->is_primary_key);
 	deserializer.ReadProperty(201, "index", result->index);

package/src/duckdb/src/storage/serialization/serialize_create_info.cpp

@@ -3,8 +3,8 @@
 // Do not edit this file manually, your changes will be overwritten
 //===----------------------------------------------------------------------===//
 
-#include "duckdb/common/serializer/format_serializer.hpp"
-#include "duckdb/common/serializer/format_deserializer.hpp"
+#include "duckdb/common/serializer/serializer.hpp"
+#include "duckdb/common/serializer/deserializer.hpp"
 #include "duckdb/parser/parsed_data/create_info.hpp"
 #include "duckdb/parser/parsed_data/create_index_info.hpp"
 #include "duckdb/parser/parsed_data/create_table_info.hpp"
@@ -16,7 +16,7 @@
 
 namespace duckdb {
 
-void CreateInfo::FormatSerialize(FormatSerializer &serializer) const {
+void CreateInfo::Serialize(Serializer &serializer) const {
 	serializer.WriteProperty(100, "type", type);
 	serializer.WriteProperty(101, "catalog", catalog);
 	serializer.WriteProperty(102, "schema", schema);
@@ -26,7 +26,7 @@ void CreateInfo::FormatSerialize(FormatSerializer &serializer) const {
 	serializer.WriteProperty(106, "sql", sql);
 }
 
-unique_ptr<CreateInfo> CreateInfo::FormatDeserialize(FormatDeserializer &deserializer) {
+unique_ptr<CreateInfo> CreateInfo::Deserialize(Deserializer &deserializer) {
 	auto type = deserializer.ReadProperty<CatalogType>(100, "type");
 	auto catalog = deserializer.ReadProperty<string>(101, "catalog");
 	auto schema = deserializer.ReadProperty<string>(102, "schema");
@@ -34,35 +34,37 @@ unique_ptr<CreateInfo> CreateInfo::FormatDeserialize(FormatDeserializer &deseria
 	auto internal = deserializer.ReadProperty<bool>(104, "internal");
 	auto on_conflict = deserializer.ReadProperty<OnCreateConflict>(105, "on_conflict");
 	auto sql = deserializer.ReadProperty<string>(106, "sql");
+	deserializer.Set<CatalogType>(type);
 	unique_ptr<CreateInfo> result;
 	switch (type) {
 	case CatalogType::INDEX_ENTRY:
-		result = CreateIndexInfo::FormatDeserialize(deserializer);
+		result = CreateIndexInfo::Deserialize(deserializer);
 		break;
 	case CatalogType::MACRO_ENTRY:
-		result = CreateMacroInfo::FormatDeserialize(deserializer);
+		result = CreateMacroInfo::Deserialize(deserializer);
 		break;
 	case CatalogType::SCHEMA_ENTRY:
-		result = CreateSchemaInfo::FormatDeserialize(deserializer);
+		result = CreateSchemaInfo::Deserialize(deserializer);
 		break;
 	case CatalogType::SEQUENCE_ENTRY:
-		result = CreateSequenceInfo::FormatDeserialize(deserializer);
+		result = CreateSequenceInfo::Deserialize(deserializer);
 		break;
 	case CatalogType::TABLE_ENTRY:
-		result = CreateTableInfo::FormatDeserialize(deserializer);
+		result = CreateTableInfo::Deserialize(deserializer);
 		break;
 	case CatalogType::TABLE_MACRO_ENTRY:
-		result = CreateMacroInfo::FormatDeserialize(deserializer);
+		result = CreateMacroInfo::Deserialize(deserializer);
 		break;
 	case CatalogType::TYPE_ENTRY:
-		result = CreateTypeInfo::FormatDeserialize(deserializer);
+		result = CreateTypeInfo::Deserialize(deserializer);
 		break;
 	case CatalogType::VIEW_ENTRY:
-		result = CreateViewInfo::FormatDeserialize(deserializer);
+		result = CreateViewInfo::Deserialize(deserializer);
 		break;
 	default:
 		throw SerializationException("Unsupported type for deserialization of CreateInfo!");
 	}
+	deserializer.Unset<CatalogType>();
 	result->catalog = std::move(catalog);
 	result->schema = std::move(schema);
 	result->temporary = temporary;
@@ -72,57 +74,59 @@ unique_ptr<CreateInfo> CreateInfo::FormatDeserialize(FormatDeserializer &deseria
 	return result;
 }
 
-void CreateIndexInfo::FormatSerialize(FormatSerializer &serializer) const {
-	CreateInfo::FormatSerialize(serializer);
+void CreateIndexInfo::Serialize(Serializer &serializer) const {
+	CreateInfo::Serialize(serializer);
 	serializer.WriteProperty(200, "name", index_name);
 	serializer.WriteProperty(201, "table", table);
 	serializer.WriteProperty(202, "index_type", index_type);
 	serializer.WriteProperty(203, "constraint_type", constraint_type);
 	serializer.WriteProperty(204, "parsed_expressions", parsed_expressions);
-	serializer.WriteProperty(205, "names", names);
-	serializer.WriteProperty(206, "column_ids", column_ids);
-	serializer.WriteProperty(207, "options", options);
-	serializer.WriteProperty(208, "index_type_name", index_type_name);
+	serializer.WriteProperty(205, "scan_types", scan_types);
+	serializer.WriteProperty(206, "names", names);
+	serializer.WriteProperty(207, "column_ids", column_ids);
+	serializer.WriteProperty(208, "options", options);
+	serializer.WriteProperty(209, "index_type_name", index_type_name);
 }
 
-unique_ptr<CreateInfo> CreateIndexInfo::FormatDeserialize(FormatDeserializer &deserializer) {
+unique_ptr<CreateInfo> CreateIndexInfo::Deserialize(Deserializer &deserializer) {
 	auto result = duckdb::unique_ptr<CreateIndexInfo>(new CreateIndexInfo());
 	deserializer.ReadProperty(200, "name", result->index_name);
 	deserializer.ReadProperty(201, "table", result->table);
 	deserializer.ReadProperty(202, "index_type", result->index_type);
 	deserializer.ReadProperty(203, "constraint_type", result->constraint_type);
 	deserializer.ReadProperty(204, "parsed_expressions", result->parsed_expressions);
-	deserializer.ReadProperty(205, "names", result->names);
-	deserializer.ReadProperty(206, "column_ids", result->column_ids);
-	deserializer.ReadProperty(207, "options", result->options);
-	deserializer.ReadProperty(208, "index_type_name", result->index_type_name);
+	deserializer.ReadProperty(205, "scan_types", result->scan_types);
+	deserializer.ReadProperty(206, "names", result->names);
+	deserializer.ReadProperty(207, "column_ids", result->column_ids);
+	deserializer.ReadProperty(208, "options", result->options);
+	deserializer.ReadProperty(209, "index_type_name", result->index_type_name);
 	return std::move(result);
 }
 
-void CreateMacroInfo::FormatSerialize(FormatSerializer &serializer) const {
-	CreateInfo::FormatSerialize(serializer);
+void CreateMacroInfo::Serialize(Serializer &serializer) const {
+	CreateInfo::Serialize(serializer);
 	serializer.WriteProperty(200, "name", name);
 	serializer.WriteProperty(201, "function", function);
 }
 
-unique_ptr<CreateInfo> CreateMacroInfo::FormatDeserialize(FormatDeserializer &deserializer) {
-	auto result = duckdb::unique_ptr<CreateMacroInfo>(new CreateMacroInfo());
+unique_ptr<CreateInfo> CreateMacroInfo::Deserialize(Deserializer &deserializer) {
+	auto result = duckdb::unique_ptr<CreateMacroInfo>(new CreateMacroInfo(deserializer.Get<CatalogType>()));
 	deserializer.ReadProperty(200, "name", result->name);
 	deserializer.ReadProperty(201, "function", result->function);
 	return std::move(result);
 }
 
-void CreateSchemaInfo::FormatSerialize(FormatSerializer &serializer) const {
-	CreateInfo::FormatSerialize(serializer);
+void CreateSchemaInfo::Serialize(Serializer &serializer) const {
+	CreateInfo::Serialize(serializer);
 }
 
-unique_ptr<CreateInfo> CreateSchemaInfo::FormatDeserialize(FormatDeserializer &deserializer) {
+unique_ptr<CreateInfo> CreateSchemaInfo::Deserialize(Deserializer &deserializer) {
 	auto result = duckdb::unique_ptr<CreateSchemaInfo>(new CreateSchemaInfo());
 	return std::move(result);
 }
 
-void CreateSequenceInfo::FormatSerialize(FormatSerializer &serializer) const {
-	CreateInfo::FormatSerialize(serializer);
+void CreateSequenceInfo::Serialize(Serializer &serializer) const {
+	CreateInfo::Serialize(serializer);
 	serializer.WriteProperty(200, "name", name);
 	serializer.WriteProperty(201, "usage_count", usage_count);
 	serializer.WriteProperty(202, "increment", increment);
@@ -132,7 +136,7 @@ void CreateSequenceInfo::FormatSerialize(FormatSerializer &serializer) const {
 	serializer.WriteProperty(206, "cycle", cycle);
 }
 
-unique_ptr<CreateInfo> CreateSequenceInfo::FormatDeserialize(FormatDeserializer &deserializer) {
+unique_ptr<CreateInfo> CreateSequenceInfo::Deserialize(Deserializer &deserializer) {
 	auto result = duckdb::unique_ptr<CreateSequenceInfo>(new CreateSequenceInfo());
 	deserializer.ReadProperty(200, "name", result->name);
 	deserializer.ReadProperty(201, "usage_count", result->usage_count);
@@ -144,15 +148,15 @@ unique_ptr<CreateInfo> CreateSequenceInfo::FormatDeserialize(FormatDeserializer
 	return std::move(result);
 }
 
-void CreateTableInfo::FormatSerialize(FormatSerializer &serializer) const {
-	CreateInfo::FormatSerialize(serializer);
+void CreateTableInfo::Serialize(Serializer &serializer) const {
+	CreateInfo::Serialize(serializer);
 	serializer.WriteProperty(200, "table", table);
 	serializer.WriteProperty(201, "columns", columns);
 	serializer.WriteProperty(202, "constraints", constraints);
 	serializer.WritePropertyWithDefault(203, "query", query, unique_ptr<SelectStatement>());
 }
 
-unique_ptr<CreateInfo> CreateTableInfo::FormatDeserialize(FormatDeserializer &deserializer) {
+unique_ptr<CreateInfo> CreateTableInfo::Deserialize(Deserializer &deserializer) {
 	auto result = duckdb::unique_ptr<CreateTableInfo>(new CreateTableInfo());
 	deserializer.ReadProperty(200, "table", result->table);
 	deserializer.ReadProperty(201, "columns", result->columns);
@@ -161,28 +165,28 @@ unique_ptr<CreateInfo> CreateTableInfo::FormatDeserialize(FormatDeserializer &de
 	return std::move(result);
 }
 
-void CreateTypeInfo::FormatSerialize(FormatSerializer &serializer) const {
-	CreateInfo::FormatSerialize(serializer);
+void CreateTypeInfo::Serialize(Serializer &serializer) const {
+	CreateInfo::Serialize(serializer);
 	serializer.WriteProperty(200, "name", name);
 	serializer.WriteProperty(201, "logical_type", type);
 }
 
-unique_ptr<CreateInfo> CreateTypeInfo::FormatDeserialize(FormatDeserializer &deserializer) {
+unique_ptr<CreateInfo> CreateTypeInfo::Deserialize(Deserializer &deserializer) {
 	auto result = duckdb::unique_ptr<CreateTypeInfo>(new CreateTypeInfo());
 	deserializer.ReadProperty(200, "name", result->name);
 	deserializer.ReadProperty(201, "logical_type", result->type);
 	return std::move(result);
 }
 
-void CreateViewInfo::FormatSerialize(FormatSerializer &serializer) const {
-	CreateInfo::FormatSerialize(serializer);
+void CreateViewInfo::Serialize(Serializer &serializer) const {
+	CreateInfo::Serialize(serializer);
 	serializer.WriteProperty(200, "view_name", view_name);
 	serializer.WriteProperty(201, "aliases", aliases);
 	serializer.WriteProperty(202, "types", types);
 	serializer.WritePropertyWithDefault(203, "query", query, unique_ptr<SelectStatement>());
 }
 
-unique_ptr<CreateInfo> CreateViewInfo::FormatDeserialize(FormatDeserializer &deserializer) {
+unique_ptr<CreateInfo> CreateViewInfo::Deserialize(Deserializer &deserializer) {
 	auto result = duckdb::unique_ptr<CreateViewInfo>(new CreateViewInfo());
 	deserializer.ReadProperty(200, "view_name", result->view_name);
 	deserializer.ReadProperty(201, "aliases", result->aliases);