duckdb 0.10.2-dev0.0 → 0.10.2-dev5.0

package/src/duckdb/src/storage/statistics/struct_stats.cpp

@@ -107,7 +107,7 @@ void StructStats::Deserialize(Deserializer &deserializer, BaseStatistics &base)
 	auto &child_types = StructType::GetChildTypes(type);
 
 	deserializer.ReadList(200, "child_stats", [&](Deserializer::List &list, idx_t i) {
-		deserializer.Set<LogicalType &>(const_cast<LogicalType &>(child_types[i].second));
+		deserializer.Set<const LogicalType &>(child_types[i].second);
 		auto stat = list.ReadElement<BaseStatistics>();
 		base.child_stats[i].Copy(stat);
 		deserializer.Unset<LogicalType>();

package/src/duckdb/src/storage/storage_info.cpp

@@ -9,25 +9,25 @@ struct StorageVersionInfo {
 	idx_t storage_version;
 };
 
-static StorageVersionInfo storage_version_info[] = {{"v0.9.0, v0.9.1, v0.9.2 or v0.10.0", 64},
-                                                    {"v0.8.0 or v0.8.1", 51},
-                                                    {"v0.7.0 or v0.7.1", 43},
-                                                    {"v0.6.0 or v0.6.1", 39},
-                                                    {"v0.5.0 or v0.5.1", 38},
-                                                    {"v0.3.3, v0.3.4 or v0.4.0", 33},
-                                                    {"v0.3.2", 31},
-                                                    {"v0.3.1", 27},
-                                                    {"v0.3.0", 25},
-                                                    {"v0.2.9", 21},
-                                                    {"v0.2.8", 18},
-                                                    {"v0.2.7", 17},
-                                                    {"v0.2.6", 15},
-                                                    {"v0.2.5", 13},
-                                                    {"v0.2.4", 11},
-                                                    {"v0.2.3", 6},
-                                                    {"v0.2.2", 4},
-                                                    {"v0.2.1 and prior", 1},
-                                                    {nullptr, 0}};
+static const StorageVersionInfo storage_version_info[] = {{"v0.9.0, v0.9.1, v0.9.2 or v0.10.0", 64},
+                                                          {"v0.8.0 or v0.8.1", 51},
+                                                          {"v0.7.0 or v0.7.1", 43},
+                                                          {"v0.6.0 or v0.6.1", 39},
+                                                          {"v0.5.0 or v0.5.1", 38},
+                                                          {"v0.3.3, v0.3.4 or v0.4.0", 33},
+                                                          {"v0.3.2", 31},
+                                                          {"v0.3.1", 27},
+                                                          {"v0.3.0", 25},
+                                                          {"v0.2.9", 21},
+                                                          {"v0.2.8", 18},
+                                                          {"v0.2.7", 17},
+                                                          {"v0.2.6", 15},
+                                                          {"v0.2.5", 13},
+                                                          {"v0.2.4", 11},
+                                                          {"v0.2.3", 6},
+                                                          {"v0.2.2", 4},
+                                                          {"v0.2.1 and prior", 1},
+                                                          {nullptr, 0}};
 
 const char *GetDuckDBVersion(idx_t version_number) {
 	for (idx_t i = 0; storage_version_info[i].version_name; i++) {

package/src/duckdb/src/storage/storage_manager.cpp

@@ -44,6 +44,10 @@ BufferManager &BufferManager::GetBufferManager(ClientContext &context) {
 	return BufferManager::GetBufferManager(*context.db);
 }
 
+const BufferManager &BufferManager::GetBufferManager(const ClientContext &context) {
+	return BufferManager::GetBufferManager(*context.db);
+}
+
 ObjectCache &ObjectCache::GetObjectCache(ClientContext &context) {
 	return context.db->GetObjectCache();
 }
@@ -83,14 +87,14 @@ bool StorageManager::InMemory() {
 	return path == IN_MEMORY_PATH;
 }
 
-void StorageManager::Initialize() {
+void StorageManager::Initialize(optional_ptr<ClientContext> context) {
 	bool in_memory = InMemory();
 	if (in_memory && read_only) {
 		throw CatalogException("Cannot launch in-memory database in read-only mode!");
 	}
 
 	// create or load the database from disk, if not in-memory mode
-	LoadDatabase();
+	LoadDatabase(context);
 }
 
 ///////////////////////////////////////////////////////////////////////////
@@ -117,8 +121,7 @@ SingleFileStorageManager::SingleFileStorageManager(AttachedDatabase &db, string
     : StorageManager(db, std::move(path), read_only) {
 }
 
-void SingleFileStorageManager::LoadDatabase() {
-
+void SingleFileStorageManager::LoadDatabase(optional_ptr<ClientContext> context) {
 	if (InMemory()) {
 		block_manager = make_uniq<InMemoryBlockManager>(BufferManager::GetBufferManager(db));
 		table_io_manager = make_uniq<SingleFileTableIOManager>(*block_manager);
@@ -139,12 +142,11 @@ void SingleFileStorageManager::LoadDatabase() {
 	options.debug_initialize = config.options.debug_initialize;
 
 	// first check if the database exists
-	if (!fs.FileExists(path)) {
-		if (read_only) {
-			throw CatalogException("Cannot open database \"%s\" in read-only mode: database does not exist", path);
-		}
+	if (!read_only && !fs.FileExists(path)) {
+		// file does not exist and we are in read-write mode
+		// create a new file
 
-		// check if the WAL exists
+		// check if a WAL file already exists
 		auto wal_path = GetWALPath();
 		if (fs.FileExists(wal_path)) {
 			// WAL file exists but database file does not
@@ -157,8 +159,10 @@ void SingleFileStorageManager::LoadDatabase() {
 		sf_block_manager->CreateNewDatabase();
 		block_manager = std::move(sf_block_manager);
 		table_io_manager = make_uniq<SingleFileTableIOManager>(*block_manager);
-
 	} else {
+		// either the file exists, or we are in read-only mode
+		// try to read the existing file on disk
+
 		// initialize the block manager while loading the current db file
 		auto sf_block_manager = make_uniq<SingleFileBlockManager>(db, path, options);
 		sf_block_manager->LoadExistingDatabase();
@@ -167,13 +171,14 @@ void SingleFileStorageManager::LoadDatabase() {
 
 		// load the db from storage
 		auto checkpoint_reader = SingleFileCheckpointReader(*this);
-		checkpoint_reader.LoadFromStorage();
+		checkpoint_reader.LoadFromStorage(context);
 
 		// check if the WAL file exists
 		auto wal_path = GetWALPath();
-		if (fs.FileExists(wal_path)) {
+		auto handle = fs.OpenFile(wal_path, FileFlags::FILE_FLAGS_READ | FileFlags::FILE_FLAGS_NULL_IF_NOT_EXISTS);
+		if (handle) {
 			// replay the WAL
-			if (WriteAheadLog::Replay(db, wal_path)) {
+			if (WriteAheadLog::Replay(db, std::move(handle))) {
 				fs.RemoveFile(wal_path);
 			}
 		}

package/src/duckdb/src/storage/table/chunk_info.cpp

@@ -4,6 +4,7 @@
 #include "duckdb/common/serializer/serializer.hpp"
 #include "duckdb/common/serializer/deserializer.hpp"
 #include "duckdb/common/serializer/memory_stream.hpp"
+#include "duckdb/transaction/delete_info.hpp"
 
 namespace duckdb {
 
@@ -198,9 +199,16 @@ idx_t ChunkVectorInfo::Delete(transaction_t transaction_id, row_t rows[], idx_t
 	return deleted_tuples;
 }
 
-void ChunkVectorInfo::CommitDelete(transaction_t commit_id, row_t rows[], idx_t count) {
-	for (idx_t i = 0; i < count; i++) {
-		deleted[rows[i]] = commit_id;
+void ChunkVectorInfo::CommitDelete(transaction_t commit_id, const DeleteInfo &info) {
+	if (info.is_consecutive) {
+		for (idx_t i = 0; i < info.count; i++) {
+			deleted[i] = commit_id;
+		}
+	} else {
+		auto rows = info.GetRows();
+		for (idx_t i = 0; i < info.count; i++) {
+			deleted[rows[i]] = commit_id;
+		}
 	}
 }
 

package/src/duckdb/src/storage/table/column_data.cpp

@@ -57,6 +57,16 @@ const LogicalType &ColumnData::RootType() const {
 	return type;
 }
 
+bool ColumnData::HasUpdates() const {
+	lock_guard<mutex> update_guard(update_lock);
+	return updates.get();
+}
+
+void ColumnData::ClearUpdates() {
+	lock_guard<mutex> update_guard(update_lock);
+	updates.reset();
+}
+
 idx_t ColumnData::GetMaxEntry() {
 	return count;
 }
@@ -133,26 +143,52 @@ idx_t ColumnData::ScanVector(ColumnScanState &state, Vector &result, idx_t remai
 	return initial_remaining - remaining;
 }
 
-template <bool SCAN_COMMITTED, bool ALLOW_UPDATES>
-idx_t ColumnData::ScanVector(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result) {
-	bool has_updates;
-	{
-		lock_guard<mutex> update_guard(update_lock);
-		has_updates = updates ? true : false;
+unique_ptr<BaseStatistics> ColumnData::GetUpdateStatistics() {
+	lock_guard<mutex> update_guard(update_lock);
+	return updates ? updates->GetStatistics() : nullptr;
+}
+
+void ColumnData::FetchUpdates(TransactionData transaction, idx_t vector_index, Vector &result, idx_t scan_count,
+                              bool allow_updates, bool scan_committed) {
+	lock_guard<mutex> update_guard(update_lock);
+	if (!updates) {
+		return;
 	}
-	auto scan_count = ScanVector(state, result, STANDARD_VECTOR_SIZE, has_updates);
-	if (has_updates) {
-		lock_guard<mutex> update_guard(update_lock);
-		if (!ALLOW_UPDATES && updates->HasUncommittedUpdates(vector_index)) {
-			throw TransactionException("Cannot create index with outstanding updates");
-		}
-		result.Flatten(scan_count);
-		if (SCAN_COMMITTED) {
-			updates->FetchCommitted(vector_index, result);
-		} else {
-			updates->FetchUpdates(transaction, vector_index, result);
-		}
+	if (!allow_updates && updates->HasUncommittedUpdates(vector_index)) {
+		throw TransactionException("Cannot create index with outstanding updates");
+	}
+	result.Flatten(scan_count);
+	if (scan_committed) {
+		updates->FetchCommitted(vector_index, result);
+	} else {
+		updates->FetchUpdates(transaction, vector_index, result);
 	}
+}
+
+void ColumnData::FetchUpdateRow(TransactionData transaction, row_t row_id, Vector &result, idx_t result_idx) {
+	lock_guard<mutex> update_guard(update_lock);
+	if (!updates) {
+		return;
+	}
+	updates->FetchRow(transaction, row_id, result, result_idx);
+}
+
+void ColumnData::UpdateInternal(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids,
+                                idx_t update_count, Vector &base_vector) {
+	lock_guard<mutex> update_guard(update_lock);
+	if (!updates) {
+		updates = make_uniq<UpdateSegment>(*this);
+	}
+	updates->Update(transaction, column_index, update_vector, row_ids, update_count, base_vector);
+}
+
+template <bool SCAN_COMMITTED, bool ALLOW_UPDATES>
+idx_t ColumnData::ScanVector(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result) {
+	idx_t current_row = vector_index * STANDARD_VECTOR_SIZE;
+	auto vector_count = MinValue<idx_t>(STANDARD_VECTOR_SIZE, count - current_row);
+
+	auto scan_count = ScanVector(state, result, vector_count, HasUpdates());
+	FetchUpdates(transaction, vector_index, result, scan_count, ALLOW_UPDATES, SCAN_COMMITTED);
 	return scan_count;
 }
 
@@ -177,61 +213,62 @@ idx_t ColumnData::ScanCommitted(idx_t vector_index, ColumnScanState &state, Vect
 	}
 }
 
-void ColumnData::ScanCommittedRange(idx_t row_group_start, idx_t offset_in_row_group, idx_t count, Vector &result) {
+void ColumnData::ScanCommittedRange(idx_t row_group_start, idx_t offset_in_row_group, idx_t s_count, Vector &result) {
 	ColumnScanState child_state;
 	InitializeScanWithOffset(child_state, row_group_start + offset_in_row_group);
-	auto scan_count = ScanVector(child_state, result, count, updates ? true : false);
-	if (updates) {
+	bool has_updates = HasUpdates();
+	auto scan_count = ScanVector(child_state, result, s_count, has_updates);
+	if (has_updates) {
 		result.Flatten(scan_count);
-		updates->FetchCommittedRange(offset_in_row_group, count, result);
+		updates->FetchCommittedRange(offset_in_row_group, s_count, result);
 	}
 }
 
-idx_t ColumnData::ScanCount(ColumnScanState &state, Vector &result, idx_t count) {
-	if (count == 0) {
+idx_t ColumnData::ScanCount(ColumnScanState &state, Vector &result, idx_t scan_count) {
+	if (scan_count == 0) {
 		return 0;
 	}
 	// ScanCount can only be used if there are no updates
-	D_ASSERT(!updates);
-	return ScanVector(state, result, count, false);
+	D_ASSERT(!HasUpdates());
+	return ScanVector(state, result, scan_count, false);
 }
 
 void ColumnData::Select(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
-                        SelectionVector &sel, idx_t &count, const TableFilter &filter) {
+                        SelectionVector &sel, idx_t &s_count, const TableFilter &filter) {
 	idx_t scan_count = Scan(transaction, vector_index, state, result);
 
 	UnifiedVectorFormat vdata;
 	result.ToUnifiedFormat(scan_count, vdata);
-	ColumnSegment::FilterSelection(sel, result, vdata, filter, scan_count, count);
+	ColumnSegment::FilterSelection(sel, result, vdata, filter, scan_count, s_count);
 }
 
 void ColumnData::FilterScan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
-                            SelectionVector &sel, idx_t count) {
+                            SelectionVector &sel, idx_t s_count) {
 	Scan(transaction, vector_index, state, result);
-	result.Slice(sel, count);
+	result.Slice(sel, s_count);
 }
 
 void ColumnData::FilterScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, SelectionVector &sel,
-                                     idx_t count, bool allow_updates) {
+                                     idx_t s_count, bool allow_updates) {
 	ScanCommitted(vector_index, state, result, allow_updates);
-	result.Slice(sel, count);
+	result.Slice(sel, s_count);
 }
 
-void ColumnData::Skip(ColumnScanState &state, idx_t count) {
-	state.Next(count);
+void ColumnData::Skip(ColumnScanState &state, idx_t s_count) {
+	state.Next(s_count);
 }
 
-void ColumnData::Append(BaseStatistics &stats, ColumnAppendState &state, Vector &vector, idx_t count) {
+void ColumnData::Append(BaseStatistics &append_stats, ColumnAppendState &state, Vector &vector, idx_t append_count) {
 	UnifiedVectorFormat vdata;
 	vector.ToUnifiedFormat(count, vdata);
-	AppendData(stats, state, vdata, count);
+	AppendData(append_stats, state, vdata, append_count);
 }
 
-void ColumnData::Append(ColumnAppendState &state, Vector &vector, idx_t count) {
+void ColumnData::Append(ColumnAppendState &state, Vector &vector, idx_t append_count) {
 	if (parent || !stats) {
 		throw InternalException("ColumnData::Append called on a column with a parent or without stats");
 	}
-	Append(stats->statistics, state, vector, count);
+	Append(stats->statistics, state, vector, append_count);
 }
 
 bool ColumnData::CheckZonemap(TableFilter &filter) {
@@ -288,14 +325,15 @@ void ColumnData::InitializeAppend(ColumnAppendState &state) {
 	D_ASSERT(state.current->function.get().append);
 }
 
-void ColumnData::AppendData(BaseStatistics &stats, ColumnAppendState &state, UnifiedVectorFormat &vdata, idx_t count) {
+void ColumnData::AppendData(BaseStatistics &append_stats, ColumnAppendState &state, UnifiedVectorFormat &vdata,
+                            idx_t append_count) {
 	idx_t offset = 0;
-	this->count += count;
+	this->count += append_count;
 	while (true) {
 		// append the data from the vector
-		idx_t copied_elements = state.current->Append(state, vdata, offset, count);
-		stats.Merge(state.current->stats.statistics);
-		if (copied_elements == count) {
+		idx_t copied_elements = state.current->Append(state, vdata, offset, append_count);
+		append_stats.Merge(state.current->stats.statistics);
+		if (copied_elements == append_count) {
 			// finished copying everything
 			break;
 		}
@@ -308,7 +346,7 @@ void ColumnData::AppendData(BaseStatistics &stats, ColumnAppendState &state, Uni
 			state.current->InitializeAppend(state);
 		}
 		offset += copied_elements;
-		count -= copied_elements;
+		append_count -= copied_elements;
 	}
 }
 
@@ -352,24 +390,17 @@ void ColumnData::FetchRow(TransactionData transaction, ColumnFetchState &state,
 	// now perform the fetch within the segment
 	segment->FetchRow(state, row_id, result, result_idx);
 	// merge any updates made to this row
-	lock_guard<mutex> update_guard(update_lock);
-	if (updates) {
-		updates->FetchRow(transaction, row_id, result, result_idx);
-	}
+	FetchUpdateRow(transaction, row_id, result, result_idx);
 }
 
 void ColumnData::Update(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids,
                         idx_t update_count) {
-	lock_guard<mutex> update_guard(update_lock);
-	if (!updates) {
-		updates = make_uniq<UpdateSegment>(*this);
-	}
 	Vector base_vector(type);
 	ColumnScanState state;
 	auto fetch_count = Fetch(state, row_ids[0], base_vector);
 
 	base_vector.Flatten(fetch_count);
-	updates->Update(transaction, column_index, update_vector, row_ids, update_count, base_vector);
+	UpdateInternal(transaction, column_index, update_vector, row_ids, update_count, base_vector);
 }
 
 void ColumnData::UpdateColumn(TransactionData transaction, const vector<column_t> &column_path, Vector &update_vector,
@@ -379,11 +410,6 @@ void ColumnData::UpdateColumn(TransactionData transaction, const vector<column_t
 	ColumnData::Update(transaction, column_path[0], update_vector, row_ids, update_count);
 }
 
-unique_ptr<BaseStatistics> ColumnData::GetUpdateStatistics() {
-	lock_guard<mutex> update_guard(update_lock);
-	return updates ? updates->GetStatistics() : nullptr;
-}
-
 void ColumnData::AppendTransientSegment(SegmentLock &l, idx_t start_row) {
 
 	idx_t vector_segment_size = Storage::BLOCK_SIZE;
@@ -422,11 +448,11 @@ void ColumnData::CheckpointScan(ColumnSegment &segment, ColumnScanState &state,
 			segment.FetchRow(fetch_state, state.row_index + i, scan_vector, i);
 		}
 	} else {
-		segment.Scan(state, count, scan_vector, 0, true);
+		segment.Scan(state, count, scan_vector, 0, !HasUpdates());
 	}
 
 	if (updates) {
-		scan_vector.Flatten(count);
+		D_ASSERT(scan_vector.GetVectorType() == VectorType::FLAT_VECTOR);
 		updates->FetchCommittedRange(state.row_index - row_group_start, count, scan_vector);
 	}
 }
@@ -445,14 +471,13 @@ unique_ptr<ColumnCheckpointState> ColumnData::Checkpoint(RowGroup &row_group,
 		// empty table: flush the empty list
 		return checkpoint_state;
 	}
-	lock_guard<mutex> update_guard(update_lock);
 
 	ColumnDataCheckpointer checkpointer(*this, row_group, *checkpoint_state, checkpoint_info);
 	checkpointer.Checkpoint(std::move(nodes));
 
 	// replace the old tree with the new one
 	data.Replace(l, checkpoint_state->new_tree);
-	updates.reset();
+	ClearUpdates();
 
 	return checkpoint_state;
 }
@@ -527,10 +552,7 @@ void ColumnData::GetColumnSegmentInfo(idx_t row_group_index, vector<idx_t> col_p
 		column_info.segment_count = segment->count;
 		column_info.compression_type = CompressionTypeToString(segment->function.get().type);
 		column_info.segment_stats = segment->stats.statistics.ToString();
-		{
-			lock_guard<mutex> ulock(update_lock);
-			column_info.has_updates = updates ? true : false;
-		}
+		column_info.has_updates = ColumnData::HasUpdates();
 		// persistent
 		// block_id
 		// block_offset

package/src/duckdb/src/storage/table/row_group.cpp

@@ -25,15 +25,15 @@
 
 namespace duckdb {
 
-RowGroup::RowGroup(RowGroupCollection &collection, idx_t start, idx_t count)
-    : SegmentBase<RowGroup>(start, count), collection(collection), allocation_size(0) {
+RowGroup::RowGroup(RowGroupCollection &collection_p, idx_t start, idx_t count)
+    : SegmentBase<RowGroup>(start, count), collection(collection_p), allocation_size(0) {
 	Verify();
 }
 
-RowGroup::RowGroup(RowGroupCollection &collection, RowGroupPointer &&pointer)
-    : SegmentBase<RowGroup>(pointer.row_start, pointer.tuple_count), collection(collection), allocation_size(0) {
+RowGroup::RowGroup(RowGroupCollection &collection_p, RowGroupPointer pointer)
+    : SegmentBase<RowGroup>(pointer.row_start, pointer.tuple_count), collection(collection_p), allocation_size(0) {
 	// deserialize the columns
-	if (pointer.data_pointers.size() != collection.GetTypes().size()) {
+	if (pointer.data_pointers.size() != collection_p.GetTypes().size()) {
 		throw IOException("Row group column count is unaligned with table column count. Corrupt file?");
 	}
 	this->column_pointers = std::move(pointer.data_pointers);
@@ -48,8 +48,8 @@ RowGroup::RowGroup(RowGroupCollection &collection, RowGroupPointer &&pointer)
 	Verify();
 }
 
-void RowGroup::MoveToCollection(RowGroupCollection &collection, idx_t new_start) {
-	this->collection = collection;
+void RowGroup::MoveToCollection(RowGroupCollection &collection_p, idx_t new_start) {
+	this->collection = collection_p;
 	this->start = new_start;
 	for (auto &column : GetColumns()) {
 		column->SetStart(new_start);

package/src/duckdb/src/storage/table/row_version_manager.cpp

@@ -159,10 +159,10 @@ idx_t RowVersionManager::DeleteRows(idx_t vector_idx, transaction_t transaction_
 	return GetVectorInfo(vector_idx).Delete(transaction_id, rows, count);
 }
 
-void RowVersionManager::CommitDelete(idx_t vector_idx, transaction_t commit_id, row_t rows[], idx_t count) {
+void RowVersionManager::CommitDelete(idx_t vector_idx, transaction_t commit_id, const DeleteInfo &info) {
 	lock_guard<mutex> lock(version_lock);
 	has_changes = true;
-	GetVectorInfo(vector_idx).CommitDelete(commit_id, rows, count);
+	GetVectorInfo(vector_idx).CommitDelete(commit_id, info);
 }
 
 vector<MetaBlockPointer> RowVersionManager::Checkpoint(MetadataManager &manager) {

package/src/duckdb/src/storage/table/standard_column_data.cpp

@@ -21,6 +21,10 @@ void StandardColumnData::SetStart(idx_t new_start) {
 	validity.SetStart(new_start);
 }
 
+bool StandardColumnData::HasUpdates() const {
+	return ColumnData::HasUpdates() || validity.HasUpdates();
+}
+
 bool StandardColumnData::CheckZonemap(ColumnScanState &state, TableFilter &filter) {
 	if (!state.segment_checked) {
 		if (!state.current) {

package/src/duckdb/src/storage/table/update_segment.cpp

@@ -501,7 +501,9 @@ void UpdateSegment::RollbackUpdate(UpdateInfo &info) {
 	auto lock_handle = lock.GetExclusiveLock();
 
 	// move the data from the UpdateInfo back into the base info
-	D_ASSERT(root->info[info.vector_index]);
+	if (!root->info[info.vector_index]) {
+		return;
+	}
 	rollback_update_function(*root->info[info.vector_index]->info, info);
 
 	// clean up the update chain

package/src/duckdb/src/storage/table_index_list.cpp

@@ -55,7 +55,7 @@ bool TableIndexList::NameIsUnique(const string &name) {
 	return true;
 }
 
-void TableIndexList::InitializeIndexes(ClientContext &context, DataTableInfo &table_info) {
+void TableIndexList::InitializeIndexes(ClientContext &context, DataTableInfo &table_info, bool throw_on_failure) {
 	lock_guard<mutex> lock(indexes_lock);
 	for (auto &index : indexes) {
 		if (!index->IsUnknown()) {
@@ -68,6 +68,11 @@ void TableIndexList::InitializeIndexes(ClientContext &context, DataTableInfo &ta
 		// Do we know the type of this index now?
 		auto index_type = context.db->config.GetIndexTypes().FindByName(index_type_name);
 		if (!index_type) {
+			if (throw_on_failure) {
+				throw MissingExtensionException(
+				    "Cannot initialize index '%s', unknown index type '%s'. You probably need to load an extension.",
+				    unknown_index.name, index_type_name);
+			}
 			continue;
 		}
 

package/src/duckdb/src/storage/temporary_file_manager.cpp

@@ -135,7 +135,7 @@ void TemporaryFileHandle::CreateFileIfNotExists(TemporaryFileLock &) {
 		return;
 	}
 	auto &fs = FileSystem::GetFileSystem(db);
-	uint8_t open_flags = FileFlags::FILE_FLAGS_READ | FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE;
+	auto open_flags = FileFlags::FILE_FLAGS_READ | FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE;
 	handle = fs.OpenFile(path, open_flags);
 }
 

package/src/duckdb/src/storage/wal_replay.cpp

@@ -158,10 +158,10 @@ private:
 //===--------------------------------------------------------------------===//
 // Replay
 //===--------------------------------------------------------------------===//
-bool WriteAheadLog::Replay(AttachedDatabase &database, string &path) {
+bool WriteAheadLog::Replay(AttachedDatabase &database, unique_ptr<FileHandle> handle) {
 	Connection con(database.GetDatabase());
-	auto initial_source = make_uniq<BufferedFileReader>(FileSystem::Get(database), path.c_str());
-	if (initial_source->Finished()) {
+	BufferedFileReader reader(FileSystem::Get(database), std::move(handle));
+	if (reader.Finished()) {
 		// WAL is empty
 		return false;
 	}
@@ -174,10 +174,10 @@ bool WriteAheadLog::Replay(AttachedDatabase &database, string &path) {
 	try {
 		while (true) {
 			// read the current entry (deserialize only)
-			auto deserializer = WriteAheadLogDeserializer::Open(checkpoint_state, *initial_source, true);
+			auto deserializer = WriteAheadLogDeserializer::Open(checkpoint_state, reader, true);
 			if (deserializer.ReplayEntry()) {
 				// check if the file is exhausted
-				if (initial_source->Finished()) {
+				if (reader.Finished()) {
 					// we finished reading the file: break
 					break;
 				}
@@ -196,7 +196,6 @@ bool WriteAheadLog::Replay(AttachedDatabase &database, string &path) {
 		Printer::Print("Unknown Exception in WAL playback during initial read");
 		return false;
 	} // LCOV_EXCL_STOP
-	initial_source.reset();
 	if (checkpoint_state.checkpoint_id.IsValid()) {
 		// there is a checkpoint flag: check if we need to deserialize the WAL
 		auto &manager = database.GetStorageManager();
@@ -208,9 +207,11 @@ bool WriteAheadLog::Replay(AttachedDatabase &database, string &path) {
 	}
 
 	// we need to recover from the WAL: actually set up the replay state
-	BufferedFileReader reader(FileSystem::Get(database), path.c_str());
 	ReplayState state(database, *con.context);
 
+	// reset the reader - we are going to read the WAL from the beginning again
+	reader.Reset();
+
 	// replay the WAL
 	// note that everything is wrapped inside a try/catch block here
 	// there can be errors in WAL replay because of a corrupt WAL file

package/src/duckdb/src/storage/write_ahead_log.cpp

@@ -21,10 +21,9 @@ const uint64_t WAL_VERSION_NUMBER = 2;
 
 WriteAheadLog::WriteAheadLog(AttachedDatabase &database, const string &path) : skip_writing(false), database(database) {
 	wal_path = path;
-	writer = make_uniq<BufferedFileWriter>(FileSystem::Get(database), path.c_str(),
-	                                       NumericCast<uint8_t>(FileFlags::FILE_FLAGS_WRITE |
-	                                                            FileFlags::FILE_FLAGS_FILE_CREATE |
-	                                                            FileFlags::FILE_FLAGS_APPEND));
+	writer = make_uniq<BufferedFileWriter>(FileSystem::Get(database), path,
+	                                       FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE |
+	                                           FileFlags::FILE_FLAGS_APPEND);
 }
 
 WriteAheadLog::~WriteAheadLog() {

package/src/duckdb/src/transaction/cleanup_state.cpp

@@ -70,8 +70,15 @@ void CleanupState::CleanupDelete(DeleteInfo &info) {
 	indexed_tables[current_table->info->table] = current_table;
 
 	count = 0;
-	for (idx_t i = 0; i < info.count; i++) {
-		row_numbers[count++] = info.base_row + info.rows[i];
+	if (info.is_consecutive) {
+		for (idx_t i = 0; i < info.count; i++) {
+			row_numbers[count++] = info.base_row + i;
+		}
+	} else {
+		auto rows = info.GetRows();
+		for (idx_t i = 0; i < info.count; i++) {
+			row_numbers[count++] = info.base_row + rows[i];
+		}
 	}
 	Flush();
 }
@@ -87,7 +94,7 @@ void CleanupState::Flush() {
 	// delete the tuples from all the indexes
 	try {
 		current_table->RemoveFromIndexes(row_identifiers, count);
-	} catch (...) {
+	} catch (...) { // NOLINT: ignore errors here
 	}
 
 	count = 0;