duckdb 0.7.2-dev2867.0 → 0.7.2-dev3117.0

package/src/duckdb/src/storage/local_storage.cpp

@@ -9,111 +9,14 @@
 #include "duckdb/storage/partial_block_manager.hpp"
 
 #include "duckdb/storage/table/column_checkpoint_state.hpp"
-#include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/storage/table_io_manager.hpp"
 #include "duckdb/storage/table/scan_state.hpp"
 
 namespace duckdb {
 
-//===--------------------------------------------------------------------===//
-// OptimisticDataWriter
-//===--------------------------------------------------------------------===//
-OptimisticDataWriter::OptimisticDataWriter(DataTable &table) : table(table) {
-}
-
-OptimisticDataWriter::OptimisticDataWriter(DataTable &table, OptimisticDataWriter &parent)
-    : table(table), partial_manager(std::move(parent.partial_manager)),
-      written_blocks(std::move(parent.written_blocks)) {
-	if (partial_manager) {
-		partial_manager->FlushPartialBlocks();
-	}
-}
-
-OptimisticDataWriter::~OptimisticDataWriter() {
-}
-
-bool OptimisticDataWriter::PrepareWrite() {
-	// check if we should pre-emptively write the table to disk
-	if (table.info->IsTemporary() || StorageManager::Get(table.info->db).InMemory()) {
-		return false;
-	}
-	// we should! write the second-to-last row group to disk
-	// allocate the partial block-manager if none is allocated yet
-	if (!partial_manager) {
-		auto &block_manager = table.info->table_io_manager->GetBlockManagerForRowData();
-		partial_manager = make_uniq<PartialBlockManager>(block_manager);
-	}
-	return true;
-}
-
-void OptimisticDataWriter::CheckFlushToDisk(RowGroupCollection &row_groups) {
-	// we finished writing a complete row group
-	if (!PrepareWrite()) {
-		return;
-	}
-	// flush second-to-last row group
-	auto row_group = row_groups.GetRowGroup(-2);
-	FlushToDisk(row_group);
-}
-
-void OptimisticDataWriter::FlushToDisk(RowGroup *row_group) {
-	// flush the specified row group
-	D_ASSERT(row_group);
-	//! The set of column compression types (if any)
-	vector<CompressionType> compression_types;
-	D_ASSERT(compression_types.empty());
-	for (auto &column : table.column_definitions) {
-		compression_types.push_back(column.CompressionType());
-	}
-	auto row_group_pointer = row_group->WriteToDisk(*partial_manager, compression_types);
-
-	// update the set of written blocks
-	for (idx_t col_idx = 0; col_idx < row_group_pointer.statistics.size(); col_idx++) {
-		row_group_pointer.states[col_idx]->GetBlockIds(written_blocks);
-	}
-}
-
-void OptimisticDataWriter::FlushToDisk(RowGroupCollection &row_groups, bool force) {
-	if (!partial_manager) {
-		if (!force) {
-			// no partial manager - nothing to flush
-			return;
-		}
-		if (!PrepareWrite()) {
-			return;
-		}
-	}
-	// flush the last row group
-	FlushToDisk(row_groups.GetRowGroup(-1));
-}
-
-void OptimisticDataWriter::FinalFlush() {
-	if (!partial_manager) {
-		return;
-	}
-	// then flush the partial manager
-	partial_manager->FlushPartialBlocks();
-	partial_manager.reset();
-}
-
-void OptimisticDataWriter::Rollback() {
-	if (partial_manager) {
-		partial_manager->Clear();
-		partial_manager.reset();
-	}
-	if (!written_blocks.empty()) {
-		auto &block_manager = table.info->table_io_manager->GetBlockManagerForRowData();
-		for (auto block_id : written_blocks) {
-			block_manager.MarkBlockAsFree(block_id);
-		}
-	}
-}
-
-//===--------------------------------------------------------------------===//
-// Local Table Storage
-//===--------------------------------------------------------------------===//
 LocalTableStorage::LocalTableStorage(DataTable &table)
-    : table_ref(table), allocator(Allocator::Get(table.db)), deleted_rows(0), optimistic_writer(table) {
+    : table_ref(table), allocator(Allocator::Get(table.db)), deleted_rows(0), optimistic_writer(table),
+      merged_storage(false) {
 	auto types = table.GetTypes();
 	row_groups = make_shared<RowGroupCollection>(table.info, TableIOManager::Get(table).GetBlockManagerForRowData(),
 	                                             types, MAX_ROW_ID, 0);
@@ -140,7 +43,8 @@ LocalTableStorage::LocalTableStorage(ClientContext &context, DataTable &new_dt,
                                      idx_t changed_idx, const LogicalType &target_type,
                                      const vector<column_t> &bound_columns, Expression &cast_expr)
     : table_ref(new_dt), allocator(Allocator::Get(new_dt.db)), deleted_rows(parent.deleted_rows),
-      optimistic_writer(new_dt, parent.optimistic_writer), optimistic_writers(std::move(parent.optimistic_writers)) {
+      optimistic_writer(new_dt, parent.optimistic_writer), optimistic_writers(std::move(parent.optimistic_writers)),
+      merged_storage(parent.merged_storage) {
 	row_groups = parent.row_groups->AlterType(context, changed_idx, target_type, bound_columns, cast_expr);
 	parent.row_groups.reset();
 	indexes.Move(parent.indexes);
@@ -148,7 +52,8 @@ LocalTableStorage::LocalTableStorage(ClientContext &context, DataTable &new_dt,
 
 LocalTableStorage::LocalTableStorage(DataTable &new_dt, LocalTableStorage &parent, idx_t drop_idx)
     : table_ref(new_dt), allocator(Allocator::Get(new_dt.db)), deleted_rows(parent.deleted_rows),
-      optimistic_writer(new_dt, parent.optimistic_writer), optimistic_writers(std::move(parent.optimistic_writers)) {
+      optimistic_writer(new_dt, parent.optimistic_writer), optimistic_writers(std::move(parent.optimistic_writers)),
+      merged_storage(parent.merged_storage) {
 	row_groups = parent.row_groups->RemoveColumn(drop_idx);
 	parent.row_groups.reset();
 	indexes.Move(parent.indexes);
@@ -157,7 +62,8 @@ LocalTableStorage::LocalTableStorage(DataTable &new_dt, LocalTableStorage &paren
 LocalTableStorage::LocalTableStorage(ClientContext &context, DataTable &new_dt, LocalTableStorage &parent,
                                      ColumnDefinition &new_column, optional_ptr<Expression> default_value)
     : table_ref(new_dt), allocator(Allocator::Get(new_dt.db)), deleted_rows(parent.deleted_rows),
-      optimistic_writer(new_dt, parent.optimistic_writer), optimistic_writers(std::move(parent.optimistic_writers)) {
+      optimistic_writer(new_dt, parent.optimistic_writer), optimistic_writers(std::move(parent.optimistic_writers)),
+      merged_storage(parent.merged_storage) {
 	row_groups = parent.row_groups->AddColumn(context, new_column, default_value.get());
 	parent.row_groups.reset();
 	indexes.Move(parent.indexes);
@@ -187,16 +93,18 @@ idx_t LocalTableStorage::EstimatedSize() {
 	return appended_rows * row_size;
 }
 
-void LocalTableStorage::CheckFlushToDisk() {
+void LocalTableStorage::WriteNewRowGroup() {
 	if (deleted_rows != 0) {
 		// we have deletes - we cannot merge row groups
 		return;
 	}
-	optimistic_writer.CheckFlushToDisk(*row_groups);
+	optimistic_writer.WriteNewRowGroup(*row_groups);
 }
 
-void LocalTableStorage::FlushToDisk() {
-	optimistic_writer.FlushToDisk(*row_groups);
+void LocalTableStorage::FlushBlocks() {
+	if (!merged_storage && row_groups->GetTotalRows() > RowGroup::ROW_GROUP_SIZE) {
+		optimistic_writer.WriteLastRowGroup(*row_groups);
+	}
 	optimistic_writer.FinalFlush();
 }
 
@@ -286,12 +194,28 @@ OptimisticDataWriter &LocalTableStorage::CreateOptimisticWriter() {
 	return *optimistic_writers.back();
 }
 
+void LocalTableStorage::FinalizeOptimisticWriter(OptimisticDataWriter &writer) {
+	// remove the writer from the set of optimistic writers
+	unique_ptr<OptimisticDataWriter> owned_writer;
+	for (idx_t i = 0; i < optimistic_writers.size(); i++) {
+		if (optimistic_writers[i].get() == &writer) {
+			owned_writer = std::move(optimistic_writers[i]);
+			optimistic_writers.erase(optimistic_writers.begin() + i);
+			break;
+		}
+	}
+	if (!owned_writer) {
+		throw InternalException("Error in FinalizeOptimisticWriter - could not find writer");
+	}
+	optimistic_writer.Merge(*owned_writer);
+}
+
 void LocalTableStorage::Rollback() {
-	optimistic_writer.Rollback();
 	for (auto &writer : optimistic_writers) {
 		writer->Rollback();
 	}
 	optimistic_writers.clear();
+	optimistic_writer.Rollback();
 }
 
 //===--------------------------------------------------------------------===//
@@ -429,7 +353,7 @@ void LocalStorage::Append(LocalAppendState &state, DataChunk &chunk) {
 
 	//! Check if we should pre-emptively flush blocks to disk
 	if (new_row_group) {
-		storage->CheckFlushToDisk();
+		storage->WriteNewRowGroup();
 	}
 }
 
@@ -448,6 +372,7 @@ void LocalStorage::LocalMerge(DataTable &table, RowGroupCollection &collection)
 		}
 	}
 	storage.row_groups->MergeStorage(collection);
+	storage.merged_storage = true;
 }
 
 OptimisticDataWriter &LocalStorage::CreateOptimisticWriter(DataTable &table) {
@@ -455,6 +380,11 @@ OptimisticDataWriter &LocalStorage::CreateOptimisticWriter(DataTable &table) {
 	return storage.CreateOptimisticWriter();
 }
 
+void LocalStorage::FinalizeOptimisticWriter(DataTable &table, OptimisticDataWriter &writer) {
+	auto &storage = table_manager.GetOrCreateStorage(table);
+	storage.FinalizeOptimisticWriter(writer);
+}
+
 bool LocalStorage::ChangesMade() noexcept {
 	return !table_manager.IsEmpty();
 }
@@ -502,8 +432,8 @@ void LocalStorage::Flush(DataTable &table, LocalTableStorage &storage) {
 	if ((append_state.row_start == 0 || storage.row_groups->GetTotalRows() >= MERGE_THRESHOLD) &&
 	    storage.deleted_rows == 0) {
 		// table is currently empty OR we are bulk appending: move over the storage directly
-		// first flush any out-standing storage nodes
-		storage.FlushToDisk();
+		// first flush any outstanding blocks
+		storage.FlushBlocks();
 		// now append to the indexes (if there are any)
 		// FIXME: we should be able to merge the transaction-local index directly into the main table index
 		// as long we just rewrite some row-ids

package/src/duckdb/src/storage/optimistic_data_writer.cpp

@@ -0,0 +1,96 @@
+#include "duckdb/storage/optimistic_data_writer.hpp"
+#include "duckdb/storage/table/column_segment.hpp"
+#include "duckdb/storage/partial_block_manager.hpp"
+#include "duckdb/storage/table/column_checkpoint_state.hpp"
+
+namespace duckdb {
+
+OptimisticDataWriter::OptimisticDataWriter(DataTable &table) : table(table) {
+}
+
+OptimisticDataWriter::OptimisticDataWriter(DataTable &table, OptimisticDataWriter &parent) : table(table) {
+	if (parent.partial_manager) {
+		parent.partial_manager->ClearBlocks();
+	}
+}
+
+OptimisticDataWriter::~OptimisticDataWriter() {
+}
+
+bool OptimisticDataWriter::PrepareWrite() {
+	// check if we should pre-emptively write the table to disk
+	if (table.info->IsTemporary() || StorageManager::Get(table.info->db).InMemory()) {
+		return false;
+	}
+	// we should! write the second-to-last row group to disk
+	// allocate the partial block-manager if none is allocated yet
+	if (!partial_manager) {
+		auto &block_manager = table.info->table_io_manager->GetBlockManagerForRowData();
+		partial_manager = make_uniq<PartialBlockManager>(block_manager, CheckpointType::APPEND_TO_TABLE);
+	}
+	return true;
+}
+
+void OptimisticDataWriter::WriteNewRowGroup(RowGroupCollection &row_groups) {
+	// we finished writing a complete row group
+	if (!PrepareWrite()) {
+		return;
+	}
+	// flush second-to-last row group
+	auto row_group = row_groups.GetRowGroup(-2);
+	FlushToDisk(row_group);
+}
+
+void OptimisticDataWriter::WriteLastRowGroup(RowGroupCollection &row_groups) {
+	// we finished writing a complete row group
+	if (!PrepareWrite()) {
+		return;
+	}
+	// flush second-to-last row group
+	auto row_group = row_groups.GetRowGroup(-1);
+	if (!row_group) {
+		return;
+	}
+	FlushToDisk(row_group);
+}
+
+void OptimisticDataWriter::FlushToDisk(RowGroup *row_group) {
+	if (!row_group) {
+		throw InternalException("FlushToDisk called without a RowGroup");
+	}
+	//! The set of column compression types (if any)
+	vector<CompressionType> compression_types;
+	D_ASSERT(compression_types.empty());
+	for (auto &column : table.column_definitions) {
+		compression_types.push_back(column.CompressionType());
+	}
+	row_group->WriteToDisk(*partial_manager, compression_types);
+}
+
+void OptimisticDataWriter::Merge(OptimisticDataWriter &other) {
+	if (!other.partial_manager) {
+		return;
+	}
+	if (!partial_manager) {
+		partial_manager = std::move(other.partial_manager);
+		return;
+	}
+	partial_manager->Merge(*other.partial_manager);
+	other.partial_manager.reset();
+}
+
+void OptimisticDataWriter::FinalFlush() {
+	if (partial_manager) {
+		partial_manager->FlushPartialBlocks();
+		partial_manager.reset();
+	}
+}
+
+void OptimisticDataWriter::Rollback() {
+	if (partial_manager) {
+		partial_manager->Rollback();
+		partial_manager.reset();
+	}
+}
+
+} // namespace duckdb

package/src/duckdb/src/storage/partial_block_manager.cpp

@@ -2,9 +2,10 @@
 
 namespace duckdb {
 
-PartialBlockManager::PartialBlockManager(BlockManager &block_manager, uint32_t max_partial_block_size,
-                                         uint32_t max_use_count)
-    : block_manager(block_manager), max_partial_block_size(max_partial_block_size), max_use_count(max_use_count) {
+PartialBlockManager::PartialBlockManager(BlockManager &block_manager, CheckpointType checkpoint_type,
+                                         uint32_t max_partial_block_size, uint32_t max_use_count)
+    : block_manager(block_manager), checkpoint_type(checkpoint_type), max_partial_block_size(max_partial_block_size),
+      max_use_count(max_use_count) {
 }
 PartialBlockManager::~PartialBlockManager() {
 }
@@ -23,7 +24,9 @@ PartialBlockAllocation PartialBlockManager::GetBlockAllocation(uint32_t segment_
 		//! there is! increase the reference count of this block
 		allocation.partial_block->state.block_use_count += 1;
 		allocation.state = allocation.partial_block->state;
-		block_manager.IncreaseBlockReferenceCount(allocation.state.block_id);
+		if (checkpoint_type == CheckpointType::FULL_CHECKPOINT) {
+			block_manager.IncreaseBlockReferenceCount(allocation.state.block_id);
+		}
 	} else {
 		// full block: get a free block to write to
 		AllocateBlock(allocation.state, segment_size);
@@ -31,9 +34,18 @@ PartialBlockAllocation PartialBlockManager::GetBlockAllocation(uint32_t segment_
 	return allocation;
 }
 
+bool PartialBlockManager::HasBlockAllocation(uint32_t segment_size) {
+	return segment_size <= max_partial_block_size &&
+	       partially_filled_blocks.lower_bound(segment_size) != partially_filled_blocks.end();
+}
+
 void PartialBlockManager::AllocateBlock(PartialBlockState &state, uint32_t segment_size) {
 	D_ASSERT(segment_size <= Storage::BLOCK_SIZE);
-	state.block_id = block_manager.GetFreeBlockId();
+	if (checkpoint_type == CheckpointType::FULL_CHECKPOINT) {
+		state.block_id = block_manager.GetFreeBlockId();
+	} else {
+		state.block_id = INVALID_BLOCK;
+	}
 	state.block_size = Storage::BLOCK_SIZE;
 	state.offset_in_block = 0;
 	state.block_use_count = 1;
@@ -82,21 +94,73 @@ void PartialBlockManager::RegisterPartialBlock(PartialBlockAllocation &&allocati
 	// Flush any block that we're not going to reuse.
 	if (block_to_free) {
 		block_to_free->Flush(free_space);
+		AddWrittenBlock(block_to_free->state.block_id);
 	}
 }
 
-void PartialBlockManager::FlushPartialBlocks() {
+void PartialBlock::Merge(PartialBlock &other, idx_t offset, idx_t other_size) {
+	throw InternalException("PartialBlock::Merge not implemented for this block type");
+}
+
+void PartialBlockManager::Merge(PartialBlockManager &other) {
+	if (&other == this) {
+		throw InternalException("Cannot merge into itself");
+	}
+	// for each partially filled block in the other manager, check if we can merge it into an existing block in this
+	// manager
+	for (auto &e : other.partially_filled_blocks) {
+		if (!e.second) {
+			throw InternalException("Empty partially filled block found");
+		}
+		auto used_space = Storage::BLOCK_SIZE - e.first;
+		if (HasBlockAllocation(used_space)) {
+			// we can merge this block into an existing block - merge them
+			// merge blocks
+			auto allocation = GetBlockAllocation(used_space);
+			allocation.partial_block->Merge(*e.second, allocation.state.offset_in_block, used_space);
+
+			// re-register the partial block
+			allocation.state.offset_in_block += used_space;
+			RegisterPartialBlock(std::move(allocation));
+		} else {
+			// we cannot merge this block - append it directly to the current block manager
+			partially_filled_blocks.insert(make_pair(e.first, std::move(e.second)));
+		}
+	}
+	// copy over the written blocks
+	for (auto &block_id : other.written_blocks) {
+		AddWrittenBlock(block_id);
+	}
+	other.written_blocks.clear();
+	other.partially_filled_blocks.clear();
+}
+
+void PartialBlockManager::AddWrittenBlock(block_id_t block) {
+	auto entry = written_blocks.insert(block);
+	if (!entry.second) {
+		throw InternalException("Written block already exists");
+	}
+}
+
+void PartialBlockManager::ClearBlocks() {
 	for (auto &e : partially_filled_blocks) {
-		e.second->Flush(e.first);
+		e.second->Clear();
 	}
 	partially_filled_blocks.clear();
 }
 
-void PartialBlockManager::Clear() {
+void PartialBlockManager::FlushPartialBlocks() {
 	for (auto &e : partially_filled_blocks) {
-		e.second->Clear();
+		e.second->Flush(e.first);
 	}
 	partially_filled_blocks.clear();
 }
 
+void PartialBlockManager::Rollback() {
+	ClearBlocks();
+	for (auto &block_id : written_blocks) {
+		block_manager.MarkBlockAsFree(block_id);
+	}
+}
+
 } // namespace duckdb

package/src/duckdb/src/storage/single_file_block_manager.cpp

@@ -287,7 +287,9 @@ void SingleFileBlockManager::MarkBlockAsFree(block_id_t block_id) {
 	lock_guard<mutex> lock(block_lock);
 	D_ASSERT(block_id >= 0);
 	D_ASSERT(block_id < max_block);
-	D_ASSERT(free_list.find(block_id) == free_list.end());
+	if (free_list.find(block_id) != free_list.end()) {
+		throw InternalException("MarkBlockAsFree called but block %llu was already freed!", block_id);
+	}
 	multi_use_blocks.erase(block_id);
 	free_list.insert(block_id);
 }

package/src/duckdb/src/storage/standard_buffer_manager.cpp

@@ -90,15 +90,18 @@ TempBufferPoolReservation StandardBufferManager::EvictBlocksOrThrow(idx_t memory
                                                                     ARGS... args) {
 	auto r = buffer_pool.EvictBlocks(memory_delta, buffer_pool.maximum_memory, buffer);
 	if (!r.success) {
-		throw OutOfMemoryException(args..., InMemoryWarning());
+		string extra_text = StringUtil::Format(" (%s/%s used)", StringUtil::BytesToHumanReadableString(GetUsedMemory()),
+		                                       StringUtil::BytesToHumanReadableString(GetMaxMemory()));
+		extra_text += InMemoryWarning();
+		throw OutOfMemoryException(args..., extra_text);
 	}
 	return std::move(r.reservation);
 }
 
 shared_ptr<BlockHandle> StandardBufferManager::RegisterSmallMemory(idx_t block_size) {
 	D_ASSERT(block_size < Storage::BLOCK_SIZE);
-	auto res = EvictBlocksOrThrow(block_size, nullptr, "could not allocate block of %lld bytes (%lld/%lld used) %s",
-	                              block_size, GetUsedMemory(), GetMaxMemory());
+	auto res = EvictBlocksOrThrow(block_size, nullptr, "could not allocate block of size %s%s",
+	                              StringUtil::BytesToHumanReadableString(block_size));
 
 	auto buffer = ConstructManagedBuffer(block_size, nullptr, FileBufferType::TINY_BUFFER);
 
@@ -112,9 +115,8 @@ shared_ptr<BlockHandle> StandardBufferManager::RegisterMemory(idx_t block_size,
 	auto alloc_size = GetAllocSize(block_size);
 	// first evict blocks until we have enough memory to store this buffer
 	unique_ptr<FileBuffer> reusable_buffer;
-	auto res =
-	    EvictBlocksOrThrow(alloc_size, &reusable_buffer, "could not allocate block of %lld bytes (%lld/%lld used) %s",
-	                       alloc_size, GetUsedMemory(), GetMaxMemory());
+	auto res = EvictBlocksOrThrow(alloc_size, &reusable_buffer, "could not allocate block of size %s%s",
+	                              StringUtil::BytesToHumanReadableString(alloc_size));
 
 	auto buffer = ConstructManagedBuffer(block_size, std::move(reusable_buffer));
 
@@ -144,8 +146,9 @@ void StandardBufferManager::ReAllocate(shared_ptr<BlockHandle> &handle, idx_t bl
 		return;
 	} else if (memory_delta > 0) {
 		// evict blocks until we have space to resize this block
-		auto reservation = EvictBlocksOrThrow(memory_delta, nullptr, "failed to resize block from %lld to %lld%s",
-		                                      handle->memory_usage, req.alloc_size);
+		auto reservation = EvictBlocksOrThrow(memory_delta, nullptr, "failed to resize block from %s to %s%s",
+		                                      StringUtil::BytesToHumanReadableString(handle->memory_usage),
+		                                      StringUtil::BytesToHumanReadableString(req.alloc_size));
 		// EvictBlocks decrements 'current_memory' for us.
 		handle->memory_charge.Merge(std::move(reservation));
 	} else {
@@ -171,8 +174,8 @@ BufferHandle StandardBufferManager::Pin(shared_ptr<BlockHandle> &handle) {
 	}
 	// evict blocks until we have space for the current block
 	unique_ptr<FileBuffer> reusable_buffer;
-	auto reservation =
-	    EvictBlocksOrThrow(required_memory, &reusable_buffer, "failed to pin block of size %lld%s", required_memory);
+	auto reservation = EvictBlocksOrThrow(required_memory, &reusable_buffer, "failed to pin block of size %s%s",
+	                                      StringUtil::BytesToHumanReadableString(required_memory));
 	// lock the handle again and repeat the check (in case anybody loaded in the mean time)
 	lock_guard<mutex> lock(handle->lock);
 	// check if the block is already loaded
@@ -732,7 +735,8 @@ void StandardBufferManager::ReserveMemory(idx_t size) {
 	if (size == 0) {
 		return;
 	}
-	auto reservation = EvictBlocksOrThrow(size, nullptr, "failed to reserve memory data of size %lld%s", size);
+	auto reservation = EvictBlocksOrThrow(size, nullptr, "failed to reserve memory data of size %s%s",
+	                                      StringUtil::BytesToHumanReadableString(size));
 	reservation.size = 0;
 }
 
@@ -748,7 +752,8 @@ void StandardBufferManager::FreeReservedMemory(idx_t size) {
 //===--------------------------------------------------------------------===//
 data_ptr_t StandardBufferManager::BufferAllocatorAllocate(PrivateAllocatorData *private_data, idx_t size) {
 	auto &data = (BufferAllocatorData &)*private_data;
-	auto reservation = data.manager.EvictBlocksOrThrow(size, nullptr, "failed to allocate data of size %lld%s", size);
+	auto reservation = data.manager.EvictBlocksOrThrow(size, nullptr, "failed to allocate data of size %s%s",
+	                                                   StringUtil::BytesToHumanReadableString(size));
 	// We rely on manual tracking of this one. :(
 	reservation.size = 0;
 	return Allocator::Get(data.manager.db).AllocateData(size);

package/src/duckdb/src/storage/statistics/base_statistics.cpp

@@ -271,6 +271,7 @@ void BaseStatistics::Serialize(Serializer &serializer) const {
 	FieldWriter writer(serializer);
 	writer.WriteField<bool>(has_null);
 	writer.WriteField<bool>(has_no_null);
+	writer.WriteField<idx_t>(distinct_count);
 	Serialize(writer);
 	writer.Finalize();
 }
@@ -316,9 +317,11 @@ BaseStatistics BaseStatistics::Deserialize(Deserializer &source, LogicalType typ
 	FieldReader reader(source);
 	bool has_null = reader.ReadRequired<bool>();
 	bool has_no_null = reader.ReadRequired<bool>();
+	idx_t distinct_count = reader.ReadRequired<idx_t>();
 	auto result = DeserializeType(reader, std::move(type));
 	result.has_null = has_null;
 	result.has_no_null = has_no_null;
+	result.distinct_count = distinct_count;
 	reader.Finalize();
 	return result;
 }