duckdb 1.3.4 → 1.4.0

package/src/duckdb/src/storage/local_storage.cpp

@@ -1,21 +1,22 @@
 #include "duckdb/transaction/local_storage.hpp"
-#include "duckdb/execution/index/art/art.hpp"
-#include "duckdb/storage/table/append_state.hpp"
-#include "duckdb/storage/write_ahead_log.hpp"
+
 #include "duckdb/common/vector_operations/vector_operations.hpp"
-#include "duckdb/storage/table/row_group.hpp"
-#include "duckdb/transaction/duck_transaction.hpp"
+#include "duckdb/execution/index/art/art.hpp"
+#include "duckdb/execution/index/unbound_index.hpp"
 #include "duckdb/planner/table_filter.hpp"
 #include "duckdb/storage/partial_block_manager.hpp"
-
+#include "duckdb/storage/table/append_state.hpp"
 #include "duckdb/storage/table/column_checkpoint_state.hpp"
-#include "duckdb/storage/table_io_manager.hpp"
+#include "duckdb/storage/table/row_group.hpp"
 #include "duckdb/storage/table/scan_state.hpp"
+#include "duckdb/storage/table_io_manager.hpp"
+#include "duckdb/storage/write_ahead_log.hpp"
+#include "duckdb/transaction/duck_transaction.hpp"
 
 namespace duckdb {
 
 LocalTableStorage::LocalTableStorage(ClientContext &context, DataTable &table)
-    : table_ref(table), allocator(Allocator::Get(table.db)), deleted_rows(0), optimistic_writer(table),
+    : table_ref(table), allocator(Allocator::Get(table.db)), deleted_rows(0), optimistic_writer(context, table),
       merged_storage(false) {
 
 	auto types = table.GetTypes();
@@ -24,11 +25,18 @@ LocalTableStorage::LocalTableStorage(ClientContext &context, DataTable &table)
 	row_groups = make_shared_ptr<RowGroupCollection>(data_table_info, io_manager, types, MAX_ROW_ID, 0);
 	row_groups->InitializeEmpty();
 
-	data_table_info->GetIndexes().BindAndScan<ART>(context, *data_table_info, [&](ART &art) {
-		auto constraint_type = art.GetConstraintType();
-		if (constraint_type == IndexConstraintType::NONE) {
+	data_table_info->GetIndexes().Scan([&](Index &index) {
+		auto constraint = index.GetConstraintType();
+		if (constraint == IndexConstraintType::NONE) {
 			return false;
 		}
+		if (index.GetIndexType() != ART::TYPE_NAME) {
+			return false;
+		}
+		if (!index.IsBound()) {
+			return false;
+		}
+		auto &art = index.Cast<ART>();
 
 		// UNIQUE constraint.
 		vector<unique_ptr<Expression>> expressions;
@@ -39,13 +47,15 @@ LocalTableStorage::LocalTableStorage(ClientContext &context, DataTable &table)
 		}
 
 		// Create a delete index and a local index.
-		auto delete_index = make_uniq<ART>(art.GetIndexName(), constraint_type, art.GetColumnIds(),
-		                                   art.table_io_manager, std::move(delete_expressions), art.db);
+		auto &name = art.GetIndexName();
+		auto &io_manager = art.table_io_manager;
+		auto delete_index =
+		    make_uniq<ART>(name, constraint, art.GetColumnIds(), io_manager, std::move(delete_expressions), art.db);
 		delete_indexes.AddIndex(std::move(delete_index));
 
-		auto index = make_uniq<ART>(art.GetIndexName(), constraint_type, art.GetColumnIds(), art.table_io_manager,
-		                            std::move(expressions), art.db);
-		append_indexes.AddIndex(std::move(index));
+		auto append_index =
+		    make_uniq<ART>(name, constraint, art.GetColumnIds(), io_manager, std::move(expressions), art.db);
+		append_indexes.AddIndex(std::move(append_index));
 		return false;
 	});
 }
@@ -114,7 +124,9 @@ idx_t LocalTableStorage::EstimatedSize() {
 	// get the index size
 	idx_t index_sizes = 0;
 	append_indexes.Scan([&](Index &index) {
-		D_ASSERT(index.IsBound());
+		if (!index.IsBound()) {
+			return false;
+		}
 		index_sizes += index.Cast<BoundIndex>().GetInMemorySize();
 		return false;
 	});
@@ -142,30 +154,38 @@ void LocalTableStorage::FlushBlocks() {
 ErrorData LocalTableStorage::AppendToIndexes(DuckTransaction &transaction, RowGroupCollection &source,
                                              TableIndexList &index_list, const vector<LogicalType> &table_types,
                                              row_t &start_row) {
-	// only need to scan for index append
-	// figure out which columns we need to scan for the set of indexes
-	auto index_columns = index_list.GetRequiredColumns();
-	vector<StorageIndex> required_columns;
-	for (auto &col : index_columns) {
-		required_columns.emplace_back(col);
-	}
-	// create an empty mock chunk that contains all the correct types for the table
-	DataChunk mock_chunk;
-	mock_chunk.InitializeEmpty(table_types);
+	// In this function, we only care about scanning the indexed columns of a table.
+	auto indexed_columns = index_list.GetRequiredColumns();
+	vector<StorageIndex> mapped_column_ids;
+	for (auto &col : indexed_columns) {
+		mapped_column_ids.emplace_back(col);
+	}
+
+	// However, because the bound expressions of the indexes (and their bound
+	// column references) are in relation to ALL table columns, we create an
+	// empty table chunk based on the table types. It references the indexed columns,
+	// and contains nothing for all non-indexed columns.
+	DataChunk table_chunk;
+	table_chunk.InitializeEmpty(table_types);
+
 	ErrorData error;
-	source.Scan(transaction, required_columns, [&](DataChunk &chunk) -> bool {
-		// construct the mock chunk by referencing the required columns
-		for (idx_t i = 0; i < required_columns.size(); i++) {
-			auto col_id = required_columns[i].GetPrimaryIndex();
-			mock_chunk.data[col_id].Reference(chunk.data[i]);
+	source.Scan(transaction, mapped_column_ids, [&](DataChunk &index_chunk) -> bool {
+		D_ASSERT(index_chunk.ColumnCount() == mapped_column_ids.size());
+		for (idx_t i = 0; i < mapped_column_ids.size(); i++) {
+			auto col_id = mapped_column_ids[i].GetPrimaryIndex();
+			table_chunk.data[col_id].Reference(index_chunk.data[i]);
 		}
-		mock_chunk.SetCardinality(chunk);
-		// append this chunk to the indexes of the table
-		error = DataTable::AppendToIndexes(index_list, delete_indexes, mock_chunk, start_row, index_append_mode);
+		table_chunk.SetCardinality(index_chunk);
+
+		// Pass both the table and the index chunk.
+		// We need the table chunk for the bound indexes,
+		// and the index chunk for the unbound indexes (to buffer it).
+		error = DataTable::AppendToIndexes(index_list, delete_indexes, table_chunk, index_chunk, mapped_column_ids,
+		                                   start_row, index_append_mode);
 		if (error.HasError()) {
 			return false;
 		}
-		start_row += UnsafeNumericCast<row_t>(chunk.size());
+		start_row += UnsafeNumericCast<row_t>(index_chunk.size());
 		return true;
 	});
 	return error;
@@ -173,35 +193,60 @@ ErrorData LocalTableStorage::AppendToIndexes(DuckTransaction &transaction, RowGr
 
 void LocalTableStorage::AppendToIndexes(DuckTransaction &transaction, TableAppendState &append_state,
                                         bool append_to_table) {
+	// In this function, we might scan all table columns,
+	// as we might also append to the table itself (append_to_table).
+
 	auto &table = table_ref.get();
 	if (append_to_table) {
 		table.InitializeAppend(transaction, append_state);
 	}
+
+	auto data_table_info = table.GetDataTableInfo();
+	auto &index_list = data_table_info->GetIndexes();
 	ErrorData error;
+
 	if (append_to_table) {
-		// appending: need to scan entire
-		row_groups->Scan(transaction, [&](DataChunk &chunk) -> bool {
-			// append this chunk to the indexes of the table
-			error = table.AppendToIndexes(delete_indexes, chunk, append_state.current_row, index_append_mode);
-			if (error.HasError()) {
-				return false;
+		// Appending to the table: we need to scan the entire chunk.
+		DataChunk index_chunk;
+		vector<StorageIndex> mapped_column_ids;
+		if (table.HasIndexes() && index_list.HasUnbound()) {
+			TableIndexList::InitializeIndexChunk(index_chunk, row_groups->GetTypes(), mapped_column_ids,
+			                                     *data_table_info);
+		}
+
+		row_groups->Scan(transaction, [&](DataChunk &table_chunk) -> bool {
+			if (table.HasIndexes()) {
+				if (index_list.HasUnbound()) {
+					// The index chunk references all indexed columns.
+					TableIndexList::ReferenceIndexChunk(table_chunk, index_chunk, mapped_column_ids);
+				}
+
+				// Pass both the table and the index chunk.
+				// We need the table chunk for the bound indexes,
+				// and the index chunk for the unbound indexes (to buffer it).
+				error = table.AppendToIndexes(delete_indexes, table_chunk, index_chunk, mapped_column_ids,
+				                              append_state.current_row, index_append_mode);
+				if (error.HasError()) {
+					return false;
+				}
 			}
-			// append to base table
-			table.Append(chunk, append_state);
+
+			// Append to the base table.
+			table.Append(table_chunk, append_state);
 			return true;
 		});
+
 	} else {
-		auto data_table_info = table.GetDataTableInfo();
-		auto &index_list = data_table_info->GetIndexes();
+		// We only append to the indexes.
 		error = AppendToIndexes(transaction, *row_groups, index_list, table.GetTypes(), append_state.current_row);
 	}
 
 	if (error.HasError()) {
-		// need to revert all appended row ids
+		// Revert all appended row IDs.
 		row_t current_row = append_state.row_start;
-		// remove the data from the indexes, if there are any indexes
+		// Remove the data from the indexes, if any.
 		row_groups->Scan(transaction, [&](DataChunk &chunk) -> bool {
-			// Remove this chunk from the indexes.
+			// Remove the chunk.
 			try {
 				table.RemoveFromIndexes(append_state, chunk, current_row);
 			} catch (std::exception &ex) { // LCOV_EXCL_START
@@ -211,20 +256,23 @@ void LocalTableStorage::AppendToIndexes(DuckTransaction &transaction, TableAppen
 
 			current_row += UnsafeNumericCast<row_t>(chunk.size());
 			if (current_row >= append_state.current_row) {
-				// finished deleting all rows from the index: abort now
+				// Finished deleting all rows from the index.
 				return false;
 			}
 			return true;
 		});
+
 		if (append_to_table) {
 			table.RevertAppendInternal(NumericCast<idx_t>(append_state.row_start));
 		}
+
 #ifdef DEBUG
 		// Verify that our index memory is stable.
 		table.VerifyIndexBuffers();
 #endif
 		error.Throw();
 	}
+
 	if (append_to_table) {
 		table.FinalizeAppend(transaction, append_state);
 	}
@@ -381,13 +429,11 @@ bool LocalStorage::NextParallelScan(ClientContext &context, DataTable &table, Pa
 }
 
 void LocalStorage::InitializeAppend(LocalAppendState &state, DataTable &table) {
-	table.InitializeIndexes(context);
 	state.storage = &table_manager.GetOrCreateStorage(context, table);
 	state.storage->row_groups->InitializeAppend(TransactionData(transaction), state.append_state);
 }
 
 void LocalStorage::InitializeStorage(LocalAppendState &state, DataTable &table) {
-	table.InitializeIndexes(context);
 	state.storage = &table_manager.GetOrCreateStorage(context, table);
 }
 
@@ -396,12 +442,13 @@ void LocalTableStorage::AppendToDeleteIndexes(Vector &row_ids, DataChunk &delete
 		return;
 	}
 
-	delete_indexes.ScanBound<ART>([&](ART &art) {
-		if (!art.IsUnique()) {
+	delete_indexes.Scan([&](Index &index) {
+		D_ASSERT(index.IsBound());
+		if (!index.IsUnique()) {
 			return false;
 		}
 		IndexAppendInfo index_append_info(IndexAppendMode::IGNORE_DUPLICATES, nullptr);
-		auto result = art.Cast<BoundIndex>().Append(delete_chunk, row_ids, index_append_info);
+		auto result = index.Cast<BoundIndex>().Append(delete_chunk, row_ids, index_append_info);
 		if (result.HasError()) {
 			throw InternalException("unexpected constraint violation on delete ART: ", result.Message());
 		}
@@ -409,20 +456,33 @@ void LocalTableStorage::AppendToDeleteIndexes(Vector &row_ids, DataChunk &delete
 	});
 }
 
-void LocalStorage::Append(LocalAppendState &state, DataChunk &chunk) {
+void LocalStorage::Append(LocalAppendState &state, DataChunk &table_chunk, DataTableInfo &data_table_info) {
 	// Append to any unique indexes.
 	auto storage = state.storage;
 	auto offset = NumericCast<idx_t>(MAX_ROW_ID) + storage->row_groups->GetTotalRows();
 	idx_t base_id = offset + state.append_state.total_append_count;
 
-	auto error = DataTable::AppendToIndexes(storage->append_indexes, storage->delete_indexes, chunk,
-	                                        NumericCast<row_t>(base_id), storage->index_append_mode);
-	if (error.HasError()) {
-		error.Throw();
+	if (!storage->append_indexes.Empty()) {
+		DataChunk index_chunk;
+		vector<StorageIndex> mapped_column_ids;
+
+		// Only initialize the index_chunk, if there are unbound indexes.
+		if (storage->append_indexes.HasUnbound() || storage->delete_indexes.HasUnbound()) {
+			TableIndexList::InitializeIndexChunk(index_chunk, table_chunk.GetTypes(), mapped_column_ids,
+			                                     data_table_info);
+			TableIndexList::ReferenceIndexChunk(table_chunk, index_chunk, mapped_column_ids);
+		}
+
+		auto error =
+		    DataTable::AppendToIndexes(storage->append_indexes, storage->delete_indexes, table_chunk, index_chunk,
+		                               mapped_column_ids, NumericCast<row_t>(base_id), storage->index_append_mode);
+		if (error.HasError()) {
+			error.Throw();
+		}
 	}
 
 	// Append the chunk to the local storage.
-	auto new_row_group = storage->row_groups->Append(chunk, state.append_state);
+	auto new_row_group = storage->row_groups->Append(table_chunk, state.append_state);
 
 	// Check if we should pre-emptively flush blocks to disk.
 	if (new_row_group) {
@@ -516,10 +576,9 @@ void LocalStorage::Flush(DataTable &table, LocalTableStorage &storage, optional_
 		storage.Rollback();
 		return;
 	}
-	idx_t append_count = storage.row_groups->GetTotalRows() - storage.deleted_rows;
-	table.InitializeIndexes(context);
 
-	const idx_t row_group_size = storage.row_groups->GetRowGroupSize();
+	auto append_count = storage.row_groups->GetTotalRows() - storage.deleted_rows;
+	const auto row_group_size = storage.row_groups->GetRowGroupSize();
 
 	TableAppendState append_state;
 	table.AppendLock(append_state);
@@ -529,9 +588,7 @@ void LocalStorage::Flush(DataTable &table, LocalTableStorage &storage, optional_
 		// table is currently empty OR we are bulk appending: move over the storage directly
 		// first flush any outstanding blocks
 		storage.FlushBlocks();
-		// now append to the indexes (if there are any)
-		// FIXME: we should be able to merge the transaction-local index directly into the main table index
-		// as long we just rewrite some row-ids
+		// Append to the indexes.
 		if (table.HasIndexes()) {
 			storage.AppendToIndexes(transaction, append_state, false);
 		}
@@ -653,10 +710,17 @@ void LocalStorage::FetchChunk(DataTable &table, Vector &row_ids, idx_t count, co
 	if (!storage) {
 		throw InternalException("LocalStorage::FetchChunk - local storage not found");
 	}
-
 	storage->row_groups->Fetch(transaction, chunk, col_ids, row_ids, count, fetch_state);
 }
 
+bool LocalStorage::CanFetch(DataTable &table, const row_t row_id) {
+	auto storage = table_manager.GetStorage(table);
+	if (!storage) {
+		throw InternalException("LocalStorage::CanFetch - local storage not found");
+	}
+	return storage->row_groups->CanFetch(transaction, row_id);
+}
+
 TableIndexList &LocalStorage::GetIndexes(ClientContext &context, DataTable &table) {
 	auto &storage = table_manager.GetOrCreateStorage(context, table);
 	return storage.append_indexes;

package/src/duckdb/src/storage/magic_bytes.cpp

@@ -1,10 +1,11 @@
 #include "duckdb/storage/magic_bytes.hpp"
+#include "duckdb/main/client_context.hpp"
 #include "duckdb/common/local_file_system.hpp"
 #include "duckdb/storage/storage_info.hpp"
 
 namespace duckdb {
 
-DataFileType MagicBytes::CheckMagicBytes(FileSystem &fs, const string &path) {
+DataFileType MagicBytes::CheckMagicBytes(QueryContext context, FileSystem &fs, const string &path) {
 	if (path.empty() || path == IN_MEMORY_PATH) {
 		return DataFileType::DUCKDB_FILE;
 	}
@@ -16,7 +17,7 @@ DataFileType MagicBytes::CheckMagicBytes(FileSystem &fs, const string &path) {
 	constexpr const idx_t MAGIC_BYTES_READ_SIZE = 16;
 	char buffer[MAGIC_BYTES_READ_SIZE] = {};
 
-	handle->Read(buffer, MAGIC_BYTES_READ_SIZE);
+	handle->Read(context, buffer, MAGIC_BYTES_READ_SIZE);
 	if (memcmp(buffer, "SQLite format 3\0", 16) == 0) {
 		return DataFileType::SQLITE_FILE;
 	}

package/src/duckdb/src/storage/metadata/metadata_manager.cpp

@@ -1,12 +1,49 @@
 #include "duckdb/storage/metadata/metadata_manager.hpp"
-#include "duckdb/storage/buffer_manager.hpp"
-#include "duckdb/storage/buffer/block_handle.hpp"
-#include "duckdb/common/serializer/write_stream.hpp"
+
 #include "duckdb/common/serializer/read_stream.hpp"
+#include "duckdb/common/serializer/write_stream.hpp"
+#include "duckdb/main/client_context.hpp"
+#include "duckdb/storage/buffer/block_handle.hpp"
+#include "duckdb/storage/buffer_manager.hpp"
 #include "duckdb/storage/database_size.hpp"
 
 namespace duckdb {
 
+MetadataBlock::MetadataBlock() : block_id(INVALID_BLOCK), dirty(false) {
+}
+
+MetadataBlock::MetadataBlock(MetadataBlock &&other) noexcept : dirty(false) {
+	std::swap(block, other.block);
+	std::swap(block_id, other.block_id);
+	std::swap(free_blocks, other.free_blocks);
+	auto dirty_val = dirty.load();
+	dirty = other.dirty.load();
+	other.dirty = dirty_val;
+}
+
+MetadataBlock &MetadataBlock::operator=(MetadataBlock &&other) noexcept {
+	std::swap(block, other.block);
+	std::swap(block_id, other.block_id);
+	std::swap(free_blocks, other.free_blocks);
+	auto dirty_val = dirty.load();
+	dirty = other.dirty.load();
+	other.dirty = dirty_val;
+	return *this;
+}
+
+string MetadataBlock::ToString() const {
+	string result;
+	for (idx_t i = 0; i < MetadataManager::METADATA_BLOCK_COUNT; i++) {
+		if (std::find(free_blocks.begin(), free_blocks.end(), i) != free_blocks.end()) {
+			if (!result.empty()) {
+				result += ", ";
+			}
+			result += to_string(i);
+		}
+	}
+	return "block_id: " + to_string(block_id) + " [" + result + "]";
+}
+
 MetadataManager::MetadataManager(BlockManager &block_manager, BufferManager &buffer_manager)
     : block_manager(block_manager), buffer_manager(buffer_manager) {
 }
@@ -35,6 +72,8 @@ MetadataHandle MetadataManager::AllocateHandle() {
 	MetadataPointer pointer;
 	pointer.block_index = UnsafeNumericCast<idx_t>(free_block);
 	auto &block = blocks[free_block];
+	// the block is now dirty
+	block.dirty = true;
 	if (block.block->BlockId() < MAXIMUM_BLOCK) {
 		// this block is a disk-backed block, yet we are planning to write to it
 		// we need to convert it into a transient block before we can write to it
@@ -51,8 +90,19 @@ MetadataHandle MetadataManager::AllocateHandle() {
 }
 
 MetadataHandle MetadataManager::Pin(const MetadataPointer &pointer) {
+	return Pin(QueryContext(), pointer);
+}
+
+MetadataHandle MetadataManager::Pin(QueryContext context, const MetadataPointer &pointer) {
 	D_ASSERT(pointer.index < METADATA_BLOCK_COUNT);
 	auto &block = blocks[UnsafeNumericCast<int64_t>(pointer.block_index)];
+#ifdef DEBUG
+	for (auto &free_block : block.free_blocks) {
+		if (free_block == pointer.index) {
+			throw InternalException("Pinning block %d.%d but it is marked as a free block", block.block_id, free_block);
+		}
+	}
+#endif
 
 	MetadataHandle handle;
 	handle.pointer.block_index = pointer.block_index;
@@ -72,6 +122,7 @@ void MetadataManager::ConvertToTransient(MetadataBlock &metadata_block) {
 	// copy the data to the transient block
 	memcpy(new_buffer.Ptr(), old_buffer.Ptr(), block_manager.GetBlockSize());
 	metadata_block.block = std::move(new_block);
+	metadata_block.dirty = true;
 
 	// unregister the old block
 	block_manager.UnregisterBlock(metadata_block.block_id);
@@ -87,6 +138,7 @@ block_id_t MetadataManager::AllocateNewBlock() {
 	for (idx_t i = 0; i < METADATA_BLOCK_COUNT; i++) {
 		new_block.free_blocks.push_back(NumericCast<uint8_t>(METADATA_BLOCK_COUNT - i - 1));
 	}
+	new_block.dirty = true;
 	// zero-initialize the handle
 	memset(handle.Ptr(), 0, block_manager.GetBlockSize());
 	AddBlock(std::move(new_block));
@@ -107,6 +159,9 @@ void MetadataManager::AddAndRegisterBlock(MetadataBlock block) {
 	if (block.block) {
 		throw InternalException("Calling AddAndRegisterBlock on block that already exists");
 	}
+	if (block.block_id >= MAXIMUM_BLOCK) {
+		throw InternalException("AddAndRegisterBlock called with a transient block id");
+	}
 	block.block = block_manager.RegisterBlock(block.block_id);
 	AddBlock(std::move(block), true);
 }
@@ -143,7 +198,7 @@ MetadataPointer MetadataManager::RegisterDiskPointer(MetaBlockPointer pointer) {
 	auto block_id = pointer.GetBlockId();
 	MetadataBlock block;
 	block.block_id = block_id;
-	AddAndRegisterBlock(block);
+	AddAndRegisterBlock(std::move(block));
 	return FromDiskPointer(pointer);
 }
 
@@ -174,23 +229,32 @@ idx_t MetadataManager::BlockCount() {
 }
 
 void MetadataManager::Flush() {
+	// Write the blocks of the metadata manager to disk.
 	const idx_t total_metadata_size = GetMetadataBlockSize() * METADATA_BLOCK_COUNT;
 
-	// write the blocks of the metadata manager to disk
 	for (auto &kv : blocks) {
 		auto &block = kv.second;
+		if (!block.dirty) {
+			if (block.block->BlockId() >= MAXIMUM_BLOCK) {
+				throw InternalException("Transient blocks must always be marked as dirty");
+			}
+			continue;
+		}
 		auto handle = buffer_manager.Pin(block.block);
 		// zero-initialize the few leftover bytes
 		memset(handle.Ptr() + total_metadata_size, 0, block_manager.GetBlockSize() - total_metadata_size);
 		D_ASSERT(kv.first == block.block_id);
 		if (block.block->BlockId() >= MAXIMUM_BLOCK) {
-			// temporary block - convert to persistent
-			block.block = block_manager.ConvertToPersistent(kv.first, std::move(block.block), std::move(handle));
+			// Convert the temporary block to a persistent block.
+			block.block =
+			    block_manager.ConvertToPersistent(QueryContext(), kv.first, std::move(block.block), std::move(handle));
 		} else {
-			// already a persistent block - only need to write it
+			// Already a persistent block, so we only need to write it.
 			D_ASSERT(block.block->BlockId() == block.block_id);
-			block_manager.Write(handle.GetFileBuffer(), block.block_id);
+			block_manager.Write(QueryContext(), handle.GetFileBuffer(), block.block_id);
 		}
+		// the block is no longer dirty
+		block.dirty = false;
 	}
 }
 
@@ -243,18 +307,24 @@ idx_t MetadataBlock::FreeBlocksToInteger() {
 	return result;
 }
 
-void MetadataBlock::FreeBlocksFromInteger(idx_t free_list) {
-	free_blocks.clear();
-	if (free_list == 0) {
-		return;
-	}
+vector<uint8_t> MetadataBlock::BlocksFromInteger(idx_t free_list) {
+	vector<uint8_t> blocks;
 	for (idx_t i = 64; i > 0; i--) {
 		auto index = i - 1;
 		idx_t mask = idx_t(1) << index;
 		if (free_list & mask) {
-			free_blocks.push_back(UnsafeNumericCast<uint8_t>(index));
+			blocks.push_back(UnsafeNumericCast<uint8_t>(index));
 		}
 	}
+	return blocks;
+}
+
+void MetadataBlock::FreeBlocksFromInteger(idx_t free_list) {
+	free_blocks.clear();
+	if (free_list == 0) {
+		return;
+	}
+	free_blocks = BlocksFromInteger(free_list);
 }
 
 void MetadataManager::MarkBlocksAsModified() {
@@ -288,6 +358,9 @@ void MetadataManager::MarkBlocksAsModified() {
 }
 
 void MetadataManager::ClearModifiedBlocks(const vector<MetaBlockPointer> &pointers) {
+	if (pointers.empty()) {
+		return;
+	}
 	for (auto &pointer : pointers) {
 		auto block_id = pointer.GetBlockId();
 		auto block_index = pointer.GetBlockIndex();
@@ -296,8 +369,6 @@ void MetadataManager::ClearModifiedBlocks(const vector<MetaBlockPointer> &pointe
 			throw InternalException("ClearModifiedBlocks - Block id %llu not found in modified_blocks", block_id);
 		}
 		auto &modified_list = entry->second;
-		// verify the block has been modified
-		D_ASSERT(modified_list && (1ULL << block_index));
 		// unset the bit
 		modified_list &= ~(1ULL << block_index);
 	}

package/src/duckdb/src/storage/metadata/metadata_reader.cpp

@@ -7,7 +7,6 @@ MetadataReader::MetadataReader(MetadataManager &manager, MetaBlockPointer pointe
     : manager(manager), type(type), next_pointer(FromDiskPointer(pointer)), has_next_block(true),
       read_pointers(read_pointers_p), index(0), offset(0), next_offset(pointer.offset), capacity(0) {
 	if (read_pointers) {
-		D_ASSERT(read_pointers->empty());
 		read_pointers->push_back(pointer);
 	}
 }
@@ -28,6 +27,10 @@ MetadataReader::~MetadataReader() {
 }
 
 void MetadataReader::ReadData(data_ptr_t buffer, idx_t read_size) {
+	ReadData(QueryContext(), buffer, read_size);
+}
+
+void MetadataReader::ReadData(QueryContext context, data_ptr_t buffer, idx_t read_size) {
 	while (offset + read_size > capacity) {
 		// cannot read entire entry from block
 		// first read what we can from this block
@@ -39,7 +42,7 @@ void MetadataReader::ReadData(data_ptr_t buffer, idx_t read_size) {
 			offset += read_size;
 		}
 		// then move to the next block
-		ReadNextBlock();
+		ReadNextBlock(context);
 	}
 	// we have enough left in this block to read from the buffer
 	memcpy(buffer, Ptr(), read_size);
@@ -47,14 +50,34 @@ void MetadataReader::ReadData(data_ptr_t buffer, idx_t read_size) {
 }
 
 MetaBlockPointer MetadataReader::GetMetaBlockPointer() {
+	if (capacity == 0) {
+		throw InternalException("GetMetaBlockPointer called but there is no active pointer");
+	}
 	return manager.GetDiskPointer(block.pointer, UnsafeNumericCast<uint32_t>(offset));
 }
 
+vector<MetaBlockPointer> MetadataReader::GetRemainingBlocks(MetaBlockPointer last_block) {
+	vector<MetaBlockPointer> result;
+	while (has_next_block) {
+		auto next_block_pointer = manager.GetDiskPointer(next_pointer, UnsafeNumericCast<uint32_t>(next_offset));
+		if (last_block.IsValid() && next_block_pointer.block_pointer == last_block.block_pointer) {
+			break;
+		}
+		result.push_back(next_block_pointer);
+		ReadNextBlock();
+	}
+	return result;
+}
+
 void MetadataReader::ReadNextBlock() {
+	ReadNextBlock(QueryContext());
+}
+
+void MetadataReader::ReadNextBlock(QueryContext context) {
 	if (!has_next_block) {
 		throw IOException("No more data remaining in MetadataReader");
 	}
-	block = manager.Pin(next_pointer);
+	block = manager.Pin(context, next_pointer);
 	index = next_pointer.index;
 
 	idx_t next_block = Load<idx_t>(BasePtr());

package/src/duckdb/src/storage/metadata/metadata_writer.cpp

@@ -30,6 +30,13 @@ MetaBlockPointer MetadataWriter::GetMetaBlockPointer() {
 	return manager.GetDiskPointer(block.pointer, UnsafeNumericCast<uint32_t>(offset));
 }
 
+void MetadataWriter::SetWrittenPointers(optional_ptr<vector<MetaBlockPointer>> written_pointers_p) {
+	written_pointers = written_pointers_p;
+	if (written_pointers && capacity > 0) {
+		written_pointers->push_back(manager.GetDiskPointer(current_pointer));
+	}
+}
+
 MetadataHandle MetadataWriter::NextHandle() {
 	return manager.AllocateHandle();
 }

package/src/duckdb/src/storage/open_file_storage_extension.cpp

@@ -42,9 +42,9 @@ private:
 	string file;
 };
 
-unique_ptr<Catalog> OpenFileStorageAttach(StorageExtensionInfo *storage_info, ClientContext &context,
+unique_ptr<Catalog> OpenFileStorageAttach(optional_ptr<StorageExtensionInfo> storage_info, ClientContext &context,
                                           AttachedDatabase &db, const string &name, AttachInfo &info,
-                                          AccessMode access_mode) {
+                                          AttachOptions &attach_options) {
 	auto file = info.path;
 	// open an in-memory database
 	info.path = ":memory:";
@@ -66,7 +66,7 @@ unique_ptr<Catalog> OpenFileStorageAttach(StorageExtensionInfo *storage_info, Cl
 	return std::move(catalog);
 }
 
-unique_ptr<TransactionManager> OpenFileStorageTransactionManager(StorageExtensionInfo *storage_info,
+unique_ptr<TransactionManager> OpenFileStorageTransactionManager(optional_ptr<StorageExtensionInfo> storage_info,
                                                                  AttachedDatabase &db, Catalog &catalog) {
 	return make_uniq<DuckTransactionManager>(db);
 }