duckdb 1.3.4 → 1.4.1-dev2.0

package/src/duckdb/src/storage/table/struct_column_data.cpp

@@ -19,6 +19,9 @@ StructColumnData::StructColumnData(BlockManager &block_manager, DataTableInfo &i
 	if (type.id() != LogicalTypeId::UNION && StructType::IsUnnamed(type)) {
 		throw InvalidInputException("A table cannot be created from an unnamed struct");
 	}
+	if (type.id() == LogicalTypeId::VARIANT) {
+		throw NotImplementedException("A table cannot be created from a VARIANT column yet");
+	}
 	// the sub column index, starting at 1 (0 is the validity mask)
 	idx_t sub_column_index = 1;
 	for (auto &child_type : child_types) {
@@ -204,17 +207,18 @@ idx_t StructColumnData::Fetch(ColumnScanState &state, row_t row_id, Vector &resu
 	return scan_count;
 }
 
-void StructColumnData::Update(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids,
-                              idx_t update_count) {
-	validity.Update(transaction, column_index, update_vector, row_ids, update_count);
+void StructColumnData::Update(TransactionData transaction, DataTable &data_table, idx_t column_index,
+                              Vector &update_vector, row_t *row_ids, idx_t update_count) {
+	validity.Update(transaction, data_table, column_index, update_vector, row_ids, update_count);
 	auto &child_entries = StructVector::GetEntries(update_vector);
 	for (idx_t i = 0; i < child_entries.size(); i++) {
-		sub_columns[i]->Update(transaction, column_index, *child_entries[i], row_ids, update_count);
+		sub_columns[i]->Update(transaction, data_table, column_index, *child_entries[i], row_ids, update_count);
 	}
 }
 
-void StructColumnData::UpdateColumn(TransactionData transaction, const vector<column_t> &column_path,
-                                    Vector &update_vector, row_t *row_ids, idx_t update_count, idx_t depth) {
+void StructColumnData::UpdateColumn(TransactionData transaction, DataTable &data_table,
+                                    const vector<column_t> &column_path, Vector &update_vector, row_t *row_ids,
+                                    idx_t update_count, idx_t depth) {
 	// we can never DIRECTLY update a struct column
 	if (depth >= column_path.size()) {
 		throw InternalException("Attempting to directly update a struct column - this should not be possible");
@@ -222,13 +226,13 @@ void StructColumnData::UpdateColumn(TransactionData transaction, const vector<co
 	auto update_column = column_path[depth];
 	if (update_column == 0) {
 		// update the validity column
-		validity.UpdateColumn(transaction, column_path, update_vector, row_ids, update_count, depth + 1);
+		validity.UpdateColumn(transaction, data_table, column_path, update_vector, row_ids, update_count, depth + 1);
 	} else {
 		if (update_column > sub_columns.size()) {
 			throw InternalException("Update column_path out of range");
 		}
-		sub_columns[update_column - 1]->UpdateColumn(transaction, column_path, update_vector, row_ids, update_count,
-		                                             depth + 1);
+		sub_columns[update_column - 1]->UpdateColumn(transaction, data_table, column_path, update_vector, row_ids,
+		                                             update_count, depth + 1);
 	}
 }
 
@@ -328,6 +332,18 @@ bool StructColumnData::IsPersistent() {
 	return true;
 }
 
+bool StructColumnData::HasAnyChanges() const {
+	if (validity.HasAnyChanges()) {
+		return true;
+	}
+	for (auto &child_col : sub_columns) {
+		if (child_col->HasAnyChanges()) {
+			return true;
+		}
+	}
+	return false;
+}
+
 PersistentColumnData StructColumnData::Serialize() {
 	PersistentColumnData persistent_data(PhysicalType::STRUCT);
 	persistent_data.child_columns.push_back(validity.Serialize());

package/src/duckdb/src/storage/table/update_segment.cpp

@@ -7,6 +7,7 @@
 #include "duckdb/transaction/duck_transaction.hpp"
 #include "duckdb/transaction/update_info.hpp"
 #include "duckdb/transaction/undo_buffer.hpp"
+#include "duckdb/storage/data_table.hpp"
 
 #include <algorithm>
 
@@ -21,6 +22,7 @@ static UpdateSegment::merge_update_function_t GetMergeUpdateFunction(PhysicalTyp
 static UpdateSegment::rollback_update_function_t GetRollbackUpdateFunction(PhysicalType type);
 static UpdateSegment::statistics_update_function_t GetStatisticsUpdateFunction(PhysicalType type);
 static UpdateSegment::fetch_row_function_t GetFetchRowFunction(PhysicalType type);
+static UpdateSegment::get_effective_updates_t GetEffectiveUpdatesFunction(PhysicalType type);
 
 UpdateSegment::UpdateSegment(ColumnData &column_data)
     : column_data(column_data), stats(column_data.type), heap(BufferAllocator::Get(column_data.GetDatabase())) {
@@ -36,6 +38,7 @@ UpdateSegment::UpdateSegment(ColumnData &column_data)
 	this->merge_update_function = GetMergeUpdateFunction(physical_type);
 	this->rollback_update_function = GetRollbackUpdateFunction(physical_type);
 	this->statistics_update_function = GetStatisticsUpdateFunction(physical_type);
+	this->get_effective_updates = GetEffectiveUpdatesFunction(physical_type);
 }
 
 UpdateSegment::~UpdateSegment() {
@@ -102,9 +105,10 @@ idx_t UpdateInfo::GetAllocSize(idx_t type_size) {
 	return AlignValue<idx_t>(sizeof(UpdateInfo) + (sizeof(sel_t) + type_size) * STANDARD_VECTOR_SIZE);
 }
 
-void UpdateInfo::Initialize(UpdateInfo &info, transaction_t transaction_id) {
+void UpdateInfo::Initialize(UpdateInfo &info, DataTable &data_table, transaction_t transaction_id) {
 	info.max = STANDARD_VECTOR_SIZE;
 	info.version_number = transaction_id;
+	info.table = &data_table;
 	info.segment = nullptr;
 	info.prev.entry = nullptr;
 	info.next.entry = nullptr;
@@ -573,10 +577,8 @@ void UpdateSegment::RollbackUpdate(UpdateInfo &info) {
 // Cleanup Update
 //===--------------------------------------------------------------------===//
 void UpdateSegment::CleanupUpdateInternal(const StorageLockKey &lock, UpdateInfo &info) {
-	D_ASSERT(info.HasPrev());
-	auto prev = info.prev;
-	{
-		auto pin = prev.Pin();
+	if (info.HasPrev()) {
+		auto pin = info.prev.Pin();
 		auto &prev_info = UpdateInfo::Get(pin);
 		prev_info.next = info.next;
 	}
@@ -584,7 +586,7 @@ void UpdateSegment::CleanupUpdateInternal(const StorageLockKey &lock, UpdateInfo
 		auto next = info.next;
 		auto next_pin = next.Pin();
 		auto &next_info = UpdateInfo::Get(next_pin);
-		next_info.prev = prev;
+		next_info.prev = info.prev;
 	}
 }
 
@@ -1053,6 +1055,9 @@ idx_t UpdateStringStatistics(UpdateSegment *segment, SegmentStatistics &stats, U
 				}
 			}
 		}
+		if (not_null_count == count) {
+			sel.Initialize(nullptr);
+		}
 		return not_null_count;
 	}
 }
@@ -1095,6 +1100,96 @@ UpdateSegment::statistics_update_function_t GetStatisticsUpdateFunction(Physical
 	}
 }
 
+//===--------------------------------------------------------------------===//
+// Get Effective Updates
+//===--------------------------------------------------------------------===//
+idx_t GetEffectiveUpdatesValidity(UnifiedVectorFormat &update, row_t *ids, idx_t count, SelectionVector &sel,
+                                  Vector &base_data, idx_t id_offset) {
+	auto &original_validity = FlatVector::Validity(base_data);
+
+	SelectionVector new_sel;
+	new_sel.Initialize(STANDARD_VECTOR_SIZE);
+
+	idx_t effective_count = 0;
+	for (idx_t i = 0; i < count; i++) {
+		auto sel_idx = sel.get_index(i);
+		auto update_idx = update.sel->get_index(sel_idx);
+		auto original_idx = UnsafeNumericCast<idx_t>(ids[sel_idx]) - id_offset;
+		if (original_validity.RowIsValid(original_idx) == update.validity.RowIsValid(update_idx)) {
+			continue;
+		}
+		new_sel.set_index(effective_count++, sel_idx);
+	}
+	sel.Initialize(new_sel);
+	return effective_count;
+}
+
+template <class T>
+idx_t TemplatedGetEffectiveUpdates(UnifiedVectorFormat &update, row_t *ids, idx_t count, SelectionVector &sel,
+                                   Vector &base_data, idx_t id_offset) {
+	auto data = UnifiedVectorFormat::GetData<T>(update);
+
+	auto original_data = FlatVector::GetData<T>(base_data);
+	auto &original_validity = FlatVector::Validity(base_data);
+
+	SelectionVector new_sel;
+	new_sel.Initialize(STANDARD_VECTOR_SIZE);
+
+	idx_t effective_count = 0;
+	for (idx_t i = 0; i < count; i++) {
+		auto sel_idx = sel.get_index(i);
+		auto update_idx = update.sel->get_index(sel_idx);
+		auto original_idx = UnsafeNumericCast<idx_t>(ids[sel_idx]) - id_offset;
+		// NULL values in the updates should have been filtered out before
+		D_ASSERT(update.validity.RowIsValid(update_idx));
+		if (original_validity.RowIsValid(original_idx) && data[update_idx] == original_data[original_idx]) {
+			// data is equivalent - skip
+			continue;
+		}
+		new_sel.set_index(effective_count++, sel_idx);
+	}
+	sel.Initialize(new_sel);
+	return effective_count;
+}
+
+UpdateSegment::get_effective_updates_t GetEffectiveUpdatesFunction(PhysicalType type) {
+	switch (type) {
+	case PhysicalType::BIT:
+		return GetEffectiveUpdatesValidity;
+	case PhysicalType::BOOL:
+	case PhysicalType::INT8:
+		return TemplatedGetEffectiveUpdates<int8_t>;
+	case PhysicalType::INT16:
+		return TemplatedGetEffectiveUpdates<int16_t>;
+	case PhysicalType::INT32:
+		return TemplatedGetEffectiveUpdates<int32_t>;
+	case PhysicalType::INT64:
+		return TemplatedGetEffectiveUpdates<int64_t>;
+	case PhysicalType::UINT8:
+		return TemplatedGetEffectiveUpdates<uint8_t>;
+	case PhysicalType::UINT16:
+		return TemplatedGetEffectiveUpdates<uint16_t>;
+	case PhysicalType::UINT32:
+		return TemplatedGetEffectiveUpdates<uint32_t>;
+	case PhysicalType::UINT64:
+		return TemplatedGetEffectiveUpdates<uint64_t>;
+	case PhysicalType::INT128:
+		return TemplatedGetEffectiveUpdates<hugeint_t>;
+	case PhysicalType::UINT128:
+		return TemplatedGetEffectiveUpdates<uhugeint_t>;
+	case PhysicalType::FLOAT:
+		return TemplatedGetEffectiveUpdates<float>;
+	case PhysicalType::DOUBLE:
+		return TemplatedGetEffectiveUpdates<double>;
+	case PhysicalType::INTERVAL:
+		return TemplatedGetEffectiveUpdates<interval_t>;
+	case PhysicalType::VARCHAR:
+		return TemplatedGetEffectiveUpdates<string_t>;
+	default:
+		throw NotImplementedException("Unimplemented type for uncompressed segment");
+	}
+}
+
 //===--------------------------------------------------------------------===//
 // Update
 //===--------------------------------------------------------------------===//
@@ -1143,11 +1238,11 @@ static idx_t SortSelectionVector(SelectionVector &sel, idx_t count, row_t *ids)
 	return pos;
 }
 
-UpdateInfo *CreateEmptyUpdateInfo(TransactionData transaction, idx_t type_size, idx_t count,
+UpdateInfo *CreateEmptyUpdateInfo(TransactionData transaction, DataTable &data_table, idx_t type_size, idx_t count,
                                   unsafe_unique_array<char> &data) {
 	data = make_unsafe_uniq_array_uninitialized<char>(UpdateInfo::GetAllocSize(type_size));
 	auto update_info = reinterpret_cast<UpdateInfo *>(data.get());
-	UpdateInfo::Initialize(*update_info, transaction.transaction_id);
+	UpdateInfo::Initialize(*update_info, data_table, transaction.transaction_id);
 	return update_info;
 }
 
@@ -1165,8 +1260,8 @@ void UpdateSegment::InitializeUpdateInfo(idx_t vector_idx) {
 	}
 }
 
-void UpdateSegment::Update(TransactionData transaction, idx_t column_index, Vector &update_p, row_t *ids, idx_t count,
-                           Vector &base_data) {
+void UpdateSegment::Update(TransactionData transaction, DataTable &data_table, idx_t column_index, Vector &update_p,
+                           row_t *ids, idx_t count, Vector &base_data) {
 	// obtain an exclusive lock
 	auto write_lock = lock.GetExclusiveLock();
 
@@ -1195,6 +1290,18 @@ void UpdateSegment::Update(TransactionData transaction, idx_t column_index, Vect
 	auto first_id = ids[sel.get_index(0)];
 	idx_t vector_index = (UnsafeNumericCast<idx_t>(first_id) - column_data.start) / STANDARD_VECTOR_SIZE;
 	idx_t vector_offset = column_data.start + vector_index * STANDARD_VECTOR_SIZE;
+
+	if (!root || vector_index >= root->info.size() || !root->info[vector_index].IsSet()) {
+		// get a list of effective updates - i.e. updates that actually change rows
+		// if updates have the same value as the base row we can skip them
+		// we only do that if we have no updates for this vector
+		// we could do it otherwise - but that would require merging updates in order to find the current values
+		count = get_effective_updates(update_format, ids, count, sel, base_data, vector_offset);
+		if (count == 0) {
+			return;
+		}
+	}
+
 	InitializeUpdateInfo(vector_index);
 
 	D_ASSERT(idx_t(first_id) >= column_data.start);
@@ -1217,10 +1324,10 @@ void UpdateSegment::Update(TransactionData transaction, idx_t column_index, Vect
 			// no updates made yet by this transaction: initially the update info to empty
 			if (transaction.transaction) {
 				auto &dtransaction = transaction.transaction->Cast<DuckTransaction>();
-				node_ref = dtransaction.CreateUpdateInfo(type_size, count);
+				node_ref = dtransaction.CreateUpdateInfo(type_size, data_table, count);
 				node = &UpdateInfo::Get(node_ref);
 			} else {
-				node = CreateEmptyUpdateInfo(transaction, type_size, count, update_info_data);
+				node = CreateEmptyUpdateInfo(transaction, data_table, type_size, count, update_info_data);
 			}
 			node->segment = this;
 			node->vector_index = vector_index;
@@ -1249,12 +1356,13 @@ void UpdateSegment::Update(TransactionData transaction, idx_t column_index, Vect
 		base_info.Verify();
 		node->Verify();
 	} else {
+
 		// there is no version info yet: create the top level update info and fill it with the updates
 		// allocate space for the UpdateInfo in the allocator
 		idx_t alloc_size = UpdateInfo::GetAllocSize(type_size);
 		auto handle = root->allocator.Allocate(alloc_size);
 		auto &update_info = UpdateInfo::Get(handle);
-		UpdateInfo::Initialize(update_info, TRANSACTION_ID_START - 1);
+		UpdateInfo::Initialize(update_info, data_table, TRANSACTION_ID_START - 1);
 		update_info.column_index = column_index;
 
 		InitializeUpdateInfo(update_info, ids, sel, count, vector_index, vector_offset);
@@ -1264,10 +1372,10 @@ void UpdateSegment::Update(TransactionData transaction, idx_t column_index, Vect
 		UndoBufferReference node_ref;
 		optional_ptr<UpdateInfo> transaction_node;
 		if (transaction.transaction) {
-			node_ref = transaction.transaction->CreateUpdateInfo(type_size, count);
+			node_ref = transaction.transaction->CreateUpdateInfo(type_size, data_table, count);
 			transaction_node = &UpdateInfo::Get(node_ref);
 		} else {
-			transaction_node = CreateEmptyUpdateInfo(transaction, type_size, count, update_info_data);
+			transaction_node = CreateEmptyUpdateInfo(transaction, data_table, type_size, count, update_info_data);
 		}
 
 		InitializeUpdateInfo(*transaction_node, ids, sel, count, vector_index, vector_offset);

package/src/duckdb/src/storage/table_index_list.cpp

@@ -12,41 +12,56 @@
 
 namespace duckdb {
 
+IndexEntry::IndexEntry(unique_ptr<Index> index_p) : index(std::move(index_p)) {
+	if (index->IsBound()) {
+		bind_state = IndexBindState::BOUND;
+	} else {
+		bind_state = IndexBindState::UNBOUND;
+	}
+}
+
 void TableIndexList::AddIndex(unique_ptr<Index> index) {
 	D_ASSERT(index);
-	lock_guard<mutex> lock(indexes_lock);
-	indexes.push_back(std::move(index));
+	lock_guard<mutex> lock(index_entries_lock);
+	auto index_entry = make_uniq<IndexEntry>(std::move(index));
+	index_entries.push_back(std::move(index_entry));
+	if (!index_entries.back()->index->IsBound()) {
+		unbound_count++;
+	}
 }
 
 void TableIndexList::RemoveIndex(const string &name) {
-	lock_guard<mutex> lock(indexes_lock);
-
-	for (idx_t i = 0; i < indexes.size(); i++) {
-		auto &index = indexes[i];
-		if (index->GetIndexName() == name) {
-			indexes.erase_at(i);
-			break;
+	lock_guard<mutex> lock(index_entries_lock);
+	for (idx_t i = 0; i < index_entries.size(); i++) {
+		auto &index = *index_entries[i]->index;
+		if (index.GetIndexName() == name) {
+			if (!index.IsBound()) {
+				unbound_count--;
+			}
+			index_entries.erase_at(i);
+			return;
 		}
 	}
 }
 
 void TableIndexList::CommitDrop(const string &name) {
-	lock_guard<mutex> lock(indexes_lock);
-
-	for (auto &index : indexes) {
-		if (index->GetIndexName() == name) {
-			index->CommitDrop();
+	lock_guard<mutex> lock(index_entries_lock);
+	for (auto &entry : index_entries) {
+		auto &index = *entry->index;
+		if (index.GetIndexName() == name) {
+			index.CommitDrop();
+			return;
 		}
 	}
 }
 
 bool TableIndexList::NameIsUnique(const string &name) {
-	lock_guard<mutex> lock(indexes_lock);
-
 	// Only covers PK, FK, and UNIQUE indexes.
-	for (const auto &index : indexes) {
-		if (index->IsPrimary() || index->IsForeign() || index->IsUnique()) {
-			if (index->GetIndexName() == name) {
+	lock_guard<mutex> lock(index_entries_lock);
+	for (auto &entry : index_entries) {
+		auto &index = *entry->index;
+		if (index.IsPrimary() || index.IsForeign() || index.IsUnique()) {
+			if (index.GetIndexName() == name) {
 				return false;
 			}
 		}
@@ -55,29 +70,26 @@ bool TableIndexList::NameIsUnique(const string &name) {
 }
 
 optional_ptr<BoundIndex> TableIndexList::Find(const string &name) {
-	for (auto &index : indexes) {
-		if (index->GetIndexName() == name) {
-			return index->Cast<BoundIndex>();
+	for (auto &entry : index_entries) {
+		auto &index = *entry->index;
+		if (index.GetIndexName() == name) {
+			if (!index.IsBound()) {
+				throw InternalException("cannot return an unbound index in TableIndexList::Find");
+			}
+			return index.Cast<BoundIndex>();
 		}
 	}
 	return nullptr;
 }
 
-void TableIndexList::InitializeIndexes(ClientContext &context, DataTableInfo &table_info, const char *index_type) {
-	// Fast path: do we have any unbound indexes?
-	bool needs_binding = false;
+void TableIndexList::Bind(ClientContext &context, DataTableInfo &table_info, const char *index_type) {
 	{
-		lock_guard<mutex> lock(indexes_lock);
-		for (auto &index : indexes) {
-			if (!index->IsBound() && (index_type == nullptr || index->GetIndexType() == index_type)) {
-				needs_binding = true;
-				break;
-			}
+		// Early-out, if we have no unbound indexes.
+		lock_guard<mutex> lock(index_entries_lock);
+		if (unbound_count == 0) {
+			return;
 		}
 	}
-	if (!needs_binding) {
-		return;
-	}
 
 	// Get the table from the catalog, so we can add it to the binder.
 	auto &catalog = table_info.GetDB().GetCatalog();
@@ -93,39 +105,62 @@ void TableIndexList::InitializeIndexes(ClientContext &context, DataTableInfo &ta
 		column_names.push_back(col.Name());
 	}
 
-	lock_guard<mutex> lock(indexes_lock);
-	for (auto &index : indexes) {
-		if (!index->IsBound() && (index_type == nullptr || index->GetIndexType() == index_type)) {
-			// Create a binder to bind this index.
-			auto binder = Binder::CreateBinder(context);
-
-			// Add the table to the binder.
-			vector<ColumnIndex> dummy_column_ids;
-			binder->bind_context.AddBaseTable(0, string(), column_names, column_types, dummy_column_ids, table);
+	unique_lock<mutex> lock(index_entries_lock);
+	// Busy-spin trying to bind all indexes.
+	while (true) {
+		optional_ptr<IndexEntry> index_entry;
+		for (auto &entry : index_entries) {
+			auto &index = *entry->index;
+			if (!index.IsBound() && (index_type == nullptr || index.GetIndexType() == index_type)) {
+				index_entry = entry.get();
+				break;
+			}
+		}
+		if (!index_entry) {
+			// We bound all indexes.
+			D_ASSERT(unbound_count == 0);
+			break;
+		}
+		if (index_entry->bind_state == IndexBindState::BINDING) {
+			// Another thread is binding the index.
+			// Lock and unlock the index entries so that the other thread can commit its changes.
+			lock.unlock();
+			lock.lock();
+			continue;
 
-			// Create an IndexBinder to bind the index
-			IndexBinder idx_binder(*binder, context);
+		} else if (index_entry->bind_state == IndexBindState::UNBOUND) {
+			// We are the thread that'll bind the index.
+			index_entry->bind_state = IndexBindState::BINDING;
+			lock.unlock();
 
-			// Replace the unbound index with a bound index.
-			auto bound_idx = idx_binder.BindIndex(index->Cast<UnboundIndex>());
-			index = std::move(bound_idx);
+		} else {
+			throw InternalException("index entry bind state cannot be BOUND here");
 		}
-	}
-}
 
-bool TableIndexList::Empty() {
-	lock_guard<mutex> lock(indexes_lock);
-	return indexes.empty();
-}
+		// Create a binder to bind this index.
+		auto binder = Binder::CreateBinder(context);
 
-idx_t TableIndexList::Count() {
-	lock_guard<mutex> lock(indexes_lock);
-	return indexes.size();
-}
+		// Add the table to the binder.
+		vector<ColumnIndex> dummy_column_ids;
+		binder->bind_context.AddBaseTable(0, string(), column_names, column_types, dummy_column_ids, table);
+
+		// Create an IndexBinder to bind the index
+		IndexBinder idx_binder(*binder, context);
+
+		// Apply any outstanding appends and replace the unbound index with a bound index.
+		auto &unbound_index = index_entry->index->Cast<UnboundIndex>();
+		auto bound_idx = idx_binder.BindIndex(unbound_index);
+		if (unbound_index.HasBufferedAppends()) {
+			bound_idx->ApplyBufferedAppends(column_types, unbound_index.GetBufferedAppends(),
+			                                unbound_index.GetMappedColumnIds());
+		}
 
-void TableIndexList::Move(TableIndexList &other) {
-	D_ASSERT(indexes.empty());
-	indexes = std::move(other.indexes);
+		// Commit the bound index to the index entry.
+		lock.lock();
+		index_entry->bind_state = IndexBindState::BOUND;
+		index_entry->index = std::move(bound_idx);
+		unbound_count--;
+	}
 }
 
 bool IsForeignKeyIndex(const vector<PhysicalIndex> &fk_keys, Index &index, ForeignKeyType fk_type) {
@@ -154,9 +189,10 @@ bool IsForeignKeyIndex(const vector<PhysicalIndex> &fk_keys, Index &index, Forei
 
 optional_ptr<Index> TableIndexList::FindForeignKeyIndex(const vector<PhysicalIndex> &fk_keys,
                                                         const ForeignKeyType fk_type) {
-	for (auto &index_elem : indexes) {
-		if (IsForeignKeyIndex(fk_keys, *index_elem, fk_type)) {
-			return index_elem;
+	for (auto &entry : index_entries) {
+		auto &index = *entry->index;
+		if (IsForeignKeyIndex(fk_keys, index, fk_type)) {
+			return index;
 		}
 	}
 	return nullptr;
@@ -164,7 +200,7 @@ optional_ptr<Index> TableIndexList::FindForeignKeyIndex(const vector<PhysicalInd
 
 void TableIndexList::VerifyForeignKey(optional_ptr<LocalTableStorage> storage, const vector<PhysicalIndex> &fk_keys,
                                       DataChunk &chunk, ConflictManager &conflict_manager) {
-	auto fk_type = conflict_manager.LookupType() == VerifyExistenceType::APPEND_FK
+	auto fk_type = conflict_manager.GetVerifyExistenceType() == VerifyExistenceType::APPEND_FK
 	                   ? ForeignKeyType::FK_TYPE_PRIMARY_KEY_TABLE
 	                   : ForeignKeyType::FK_TYPE_FOREIGN_KEY_TABLE;
 
@@ -182,31 +218,59 @@ void TableIndexList::VerifyForeignKey(optional_ptr<LocalTableStorage> storage, c
 }
 
 unordered_set<column_t> TableIndexList::GetRequiredColumns() {
-	lock_guard<mutex> lock(indexes_lock);
+	lock_guard<mutex> lock(index_entries_lock);
 	unordered_set<column_t> column_ids;
-	for (auto &index : indexes) {
-		for (auto col_id : index->GetColumnIds()) {
+	for (auto &entry : index_entries) {
+		auto &index = *entry->index;
+		for (auto col_id : index.GetColumnIds()) {
 			column_ids.insert(col_id);
 		}
 	}
 	return column_ids;
 }
 
-vector<IndexStorageInfo> TableIndexList::GetStorageInfos(const case_insensitive_map_t<Value> &options) {
+vector<IndexStorageInfo> TableIndexList::SerializeToDisk(QueryContext context,
+                                                         const case_insensitive_map_t<Value> &options) {
 	vector<IndexStorageInfo> infos;
-	for (auto &index : indexes) {
-		if (index->IsBound()) {
-			auto info = index->Cast<BoundIndex>().GetStorageInfo(options, false);
+	for (auto &entry : index_entries) {
+		auto &index = *entry->index;
+		if (index.IsBound()) {
+			auto info = index.Cast<BoundIndex>().SerializeToDisk(context, options);
 			D_ASSERT(info.IsValid() && !info.name.empty());
 			infos.push_back(info);
 			continue;
 		}
 
-		auto info = index->Cast<UnboundIndex>().GetStorageInfo();
+		auto info = index.Cast<UnboundIndex>().GetStorageInfo();
 		D_ASSERT(!info.name.empty());
 		infos.push_back(info);
 	}
 	return infos;
 }
 
+void TableIndexList::InitializeIndexChunk(DataChunk &index_chunk, const vector<LogicalType> &table_types,
+                                          vector<StorageIndex> &mapped_column_ids, DataTableInfo &data_table_info) {
+	// table_chunk contains all table columns.
+	// We only reference the index columns in the index chunk.
+	auto &index_list = data_table_info.GetIndexes();
+	auto indexed_columns = index_list.GetRequiredColumns();
+
+	vector<LogicalType> index_types;
+	for (auto &col : indexed_columns) {
+		index_types.push_back(table_types[col]);
+		mapped_column_ids.emplace_back(col);
+	}
+
+	index_chunk.InitializeEmpty(index_types);
+}
+
+void TableIndexList::ReferenceIndexChunk(DataChunk &table_chunk, DataChunk &index_chunk,
+                                         vector<StorageIndex> &mapped_column_ids) {
+	for (idx_t i = 0; i < mapped_column_ids.size(); i++) {
+		auto col_id = mapped_column_ids[i].GetPrimaryIndex();
+		index_chunk.data[i].Reference(table_chunk.data[col_id]);
+	}
+	index_chunk.SetCardinality(table_chunk);
+}
+
 } // namespace duckdb