duckdb 1.3.2 → 1.3.3-dev4.0

package/src/duckdb/src/include/duckdb/storage/table/column_segment.hpp

@@ -20,19 +20,21 @@
 #include "duckdb/storage/table/segment_base.hpp"
 
 namespace duckdb {
-class ColumnSegment;
+
+class BaseStatistics;
 class BlockManager;
 class ColumnData;
+class ColumnSegment;
 class DatabaseInstance;
+class TableFilter;
 class Transaction;
-class BaseStatistics;
 class UpdateSegment;
-class TableFilter;
-struct TableFilterState;
+
+struct ColumnAppendState;
 struct ColumnFetchState;
 struct ColumnScanState;
-struct ColumnAppendState;
 struct PrefetchState;
+struct TableFilterState;
 
 enum class ColumnSegmentType : uint8_t { TRANSIENT, PERSISTENT };
 //! TableFilter represents a filter pushed down into the table scan.
@@ -97,12 +99,13 @@ public:
 	//! Revert an append made to this segment
 	void RevertAppend(idx_t start_row);
 
-	//! Convert a transient in-memory segment into a persistent segment blocked by an on-disk block.
+	//! Convert a transient in-memory segment to a persistent segment backed by an on-disk block.
 	//! Only used during checkpointing.
-	void ConvertToPersistent(optional_ptr<BlockManager> block_manager, block_id_t block_id);
+	void ConvertToPersistent(QueryContext context, optional_ptr<BlockManager> block_manager, const block_id_t block_id);
 	//! Updates pointers to refer to the given block and offset. This is only used
 	//! when sharing a block among segments. This is invoked only AFTER the block is written.
 	void MarkAsPersistent(shared_ptr<BlockHandle> block, uint32_t offset_in_block);
+	void SetBlock(shared_ptr<BlockHandle> block, uint32_t offset);
 	//! Gets a data pointer from a persistent column segment
 	DataPointer GetDataPointer();
 

package/src/duckdb/src/include/duckdb/storage/table/data_table_info.hpp

@@ -23,9 +23,9 @@ struct DataTableInfo {
 public:
 	DataTableInfo(AttachedDatabase &db, shared_ptr<TableIOManager> table_io_manager_p, string schema, string table);
 
-	//! Initialize any unknown indexes whose types might now be present after an extension load, optionally throwing an
-	//! exception if an index can't be initialized
-	void InitializeIndexes(ClientContext &context, const char *index_type = nullptr);
+	//! Bind unknown indexes throwing an exception if binding fails.
+	//! Only binds the specified index type, or all, if nullptr.
+	void BindIndexes(ClientContext &context, const char *index_type = nullptr);
 
 	//! Whether or not the table is temporary
 	bool IsTemporary() const;

package/src/duckdb/src/include/duckdb/storage/table/in_memory_checkpoint.hpp

@@ -0,0 +1,93 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/table/in_memory_checkpoint.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/storage/checkpoint/row_group_writer.hpp"
+#include "duckdb/storage/checkpoint/table_data_writer.hpp"
+#include "duckdb/storage/checkpoint_manager.hpp"
+
+namespace duckdb {
+
+class InMemoryCheckpointer final : public CheckpointWriter {
+public:
+	InMemoryCheckpointer(QueryContext context, AttachedDatabase &db, BlockManager &block_manager,
+	                     StorageManager &storage_manager, CheckpointType checkpoint_type);
+
+	void CreateCheckpoint() override;
+
+	MetadataWriter &GetMetadataWriter() override;
+	MetadataManager &GetMetadataManager() override;
+	unique_ptr<TableDataWriter> GetTableDataWriter(TableCatalogEntry &table) override;
+	optional_ptr<ClientContext> GetClientContext() const {
+		return context;
+	}
+	CheckpointType GetCheckpointType() const {
+		return checkpoint_type;
+	}
+	PartialBlockManager &GetPartialBlockManager() {
+		return partial_block_manager;
+	}
+
+public:
+	void WriteTable(TableCatalogEntry &table, Serializer &serializer) override;
+
+private:
+	optional_ptr<ClientContext> context;
+	PartialBlockManager partial_block_manager;
+	StorageManager &storage_manager;
+	CheckpointType checkpoint_type;
+};
+
+class InMemoryTableDataWriter : public TableDataWriter {
+public:
+	InMemoryTableDataWriter(InMemoryCheckpointer &checkpoint_manager, TableCatalogEntry &table);
+
+public:
+	void WriteUnchangedTable(MetaBlockPointer pointer, idx_t total_rows) override;
+	void FinalizeTable(const TableStatistics &global_stats, DataTableInfo &info, RowGroupCollection &collection,
+	                   Serializer &serializer) override;
+	unique_ptr<RowGroupWriter> GetRowGroupWriter(RowGroup &row_group) override;
+	CheckpointType GetCheckpointType() const override;
+	MetadataManager &GetMetadataManager() override;
+
+private:
+	InMemoryCheckpointer &checkpoint_manager;
+};
+
+class InMemoryRowGroupWriter : public RowGroupWriter {
+public:
+	InMemoryRowGroupWriter(TableCatalogEntry &table, PartialBlockManager &partial_block_manager,
+	                       InMemoryCheckpointer &checkpoint_manager);
+
+public:
+	CheckpointType GetCheckpointType() const override;
+	WriteStream &GetPayloadWriter() override;
+	MetaBlockPointer GetMetaBlockPointer() override;
+	optional_ptr<MetadataManager> GetMetadataManager() override;
+
+private:
+	//! Underlying writer object
+	InMemoryCheckpointer &checkpoint_manager;
+	// Nop metadata writer
+	MemoryStream metadata_writer;
+};
+
+struct InMemoryPartialBlock : public PartialBlock {
+public:
+	InMemoryPartialBlock(ColumnData &data, ColumnSegment &segment, PartialBlockState state,
+	                     BlockManager &block_manager);
+	~InMemoryPartialBlock() override;
+
+public:
+	void Flush(QueryContext context, const idx_t free_space_left) override;
+	void Merge(PartialBlock &other, idx_t offset, idx_t other_size) override;
+	void AddSegmentToTail(ColumnData &data, ColumnSegment &segment, uint32_t offset_in_block) override;
+	void Clear() override;
+};
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/list_column_data.hpp

@@ -59,6 +59,7 @@ public:
 	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, ColumnCheckpointInfo &info) override;
 
 	bool IsPersistent() override;
+	bool HasAnyChanges() const override;
 	PersistentColumnData Serialize() override;
 	void InitializeColumn(PersistentColumnData &column_data, BaseStatistics &target_stats) override;
 

package/src/duckdb/src/include/duckdb/storage/table/persistent_table_data.hpp

@@ -22,6 +22,7 @@ public:
 	explicit PersistentTableData(idx_t column_count);
 	~PersistentTableData();
 
+	MetaBlockPointer base_table_pointer;
 	TableStatistics table_stats;
 	idx_t total_rows;
 	idx_t row_group_count;

package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp

@@ -65,6 +65,7 @@ struct RowGroupWriteInfo {
 struct RowGroupWriteData {
 	vector<unique_ptr<ColumnCheckpointState>> states;
 	vector<BaseStatistics> statistics;
+	vector<MetaBlockPointer> existing_pointers;
 };
 
 class RowGroup : public SegmentBase<RowGroup> {
@@ -92,6 +93,12 @@ public:
 	RowGroupCollection &GetCollection() {
 		return collection.get();
 	}
+	//! Returns the list of meta block pointers used by the columns
+	vector<MetaBlockPointer> GetColumnPointers();
+	//! Returns the list of meta block pointers used by the deletes
+	const vector<MetaBlockPointer> &GetDeletesPointers() const {
+		return deletes_pointers;
+	}
 	BlockManager &GetBlockManager();
 	DataTableInfo &GetTableInfo();
 
@@ -106,6 +113,7 @@ public:
 	void CommitDropColumn(const idx_t column_index);
 
 	void InitializeEmpty(const vector<LogicalType> &types);
+	bool HasChanges() const;
 
 	//! Initialize a scan over this row_group
 	bool InitializeScan(CollectionScanState &state);
@@ -156,7 +164,7 @@ public:
 	            const vector<PhysicalIndex> &column_ids);
 	//! Update a single column; corresponds to DataTable::UpdateColumn
 	//! This method should only be called from the WAL
-	void UpdateColumn(TransactionData transaction, DataChunk &updates, Vector &row_ids,
+	void UpdateColumn(TransactionData transaction, DataChunk &updates, Vector &row_ids, idx_t offset, idx_t count,
 	                  const vector<column_t> &column_path);
 
 	void MergeStatistics(idx_t column_idx, const BaseStatistics &other);
@@ -196,6 +204,8 @@ private:
 	ColumnData &GetColumn(const StorageIndex &c);
 	idx_t GetColumnCount() const;
 	vector<shared_ptr<ColumnData>> &GetColumns();
+	ColumnData &GetRowIdColumnData();
+	void SetCount(idx_t count);
 
 	template <TableScanType TYPE>
 	void TemplatedScan(TransactionData transaction, CollectionScanState &state, DataChunk &result);
@@ -209,8 +219,13 @@ private:
 	vector<MetaBlockPointer> column_pointers;
 	unique_ptr<atomic<bool>[]> is_loaded;
 	vector<MetaBlockPointer> deletes_pointers;
+	bool has_metadata_blocks = false;
+	vector<idx_t> extra_metadata_blocks;
 	atomic<bool> deletes_is_loaded;
-	idx_t allocation_size;
+	atomic<idx_t> allocation_size;
+	unique_ptr<ColumnData> row_id_column_data;
+	atomic<bool> row_id_is_loaded;
+	atomic<bool> has_changes;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp

@@ -51,6 +51,7 @@ public:
 	void Initialize(PersistentCollectionData &data);
 	void Initialize(PersistentTableData &data);
 	void InitializeEmpty();
+	void FinalizeCheckpoint(MetaBlockPointer pointer);
 
 	bool IsEmpty() const;
 
@@ -75,6 +76,8 @@ public:
 
 	void Fetch(TransactionData transaction, DataChunk &result, const vector<StorageIndex> &column_ids,
 	           const Vector &row_identifiers, idx_t fetch_count, ColumnFetchState &state);
+	//! Returns true, if the row group can fetch the row id for the transaction.
+	bool CanFetch(TransactionData, const row_t row_id);
 
 	//! Initialize an append of a variable number of rows. FinalizeAppend must be called after appending is done.
 	void InitializeAppend(TableAppendState &state);
@@ -143,10 +146,13 @@ public:
 	idx_t GetRowGroupSize() const {
 		return row_group_size;
 	}
+	void SetAppendRequiresNewRowGroup();
 
 private:
 	bool IsEmpty(SegmentLock &) const;
 
+	optional_ptr<RowGroup> NextUpdateRowGroup(row_t *ids, idx_t &pos, idx_t count) const;
+
 private:
 	//! BlockManager
 	BlockManager &block_manager;
@@ -164,7 +170,11 @@ private:
 	//! Table statistics
 	TableStatistics stats;
 	//! Allocation size, only tracked for appends
-	idx_t allocation_size;
+	atomic<idx_t> allocation_size;
+	//! Root metadata pointer, if the collection is loaded from disk
+	MetaBlockPointer metadata_pointer;
+	//! Whether or not we need to append a new row group prior to appending
+	bool requires_new_row_group;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/row_group_segment_tree.hpp

@@ -23,6 +23,10 @@ public:
 
 	void Initialize(PersistentTableData &data);
 
+	MetaBlockPointer GetRootPointer() const {
+		return root_pointer;
+	}
+
 protected:
 	unique_ptr<RowGroup> LoadSegment() override;
 
@@ -30,6 +34,7 @@ protected:
 	idx_t current_row_group;
 	idx_t max_row_group;
 	unique_ptr<MetadataReader> reader;
+	MetaBlockPointer root_pointer;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/row_id_column_data.hpp

@@ -0,0 +1,68 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/table/row_id_column_data.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/storage/table/column_data.hpp"
+
+namespace duckdb {
+
+class RowIdColumnData : public ColumnData {
+public:
+	RowIdColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t start_row);
+
+public:
+	void InitializePrefetch(PrefetchState &prefetch_state, ColumnScanState &scan_state, idx_t rows) override;
+	void InitializeScan(ColumnScanState &state) override;
+	void InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx) override;
+
+	idx_t Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
+	           idx_t scan_count) override;
+	idx_t ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates,
+	                    idx_t scan_count) override;
+	void ScanCommittedRange(idx_t row_group_start, idx_t offset_in_row_group, idx_t count, Vector &result) override;
+	idx_t ScanCount(ColumnScanState &state, Vector &result, idx_t count, idx_t result_offset = 0) override;
+
+	void Filter(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
+	            SelectionVector &sel, idx_t &count, const TableFilter &filter, TableFilterState &filter_state) override;
+	void Select(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
+	            SelectionVector &sel, idx_t count) override;
+	void SelectCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, SelectionVector &sel, idx_t count,
+	                     bool allow_updates) override;
+
+	idx_t Fetch(ColumnScanState &state, row_t row_id, Vector &result) override;
+	void FetchRow(TransactionData transaction, ColumnFetchState &state, row_t row_id, Vector &result,
+	              idx_t result_idx) override;
+
+	void Skip(ColumnScanState &state, idx_t count = STANDARD_VECTOR_SIZE) override;
+
+	FilterPropagateResult CheckZonemap(ColumnScanState &state, TableFilter &filter) override;
+
+	void InitializeAppend(ColumnAppendState &state) override;
+	void Append(BaseStatistics &stats, ColumnAppendState &state, Vector &vector, idx_t count) override;
+	void AppendData(BaseStatistics &stats, ColumnAppendState &state, UnifiedVectorFormat &vdata, idx_t count) override;
+	void RevertAppend(row_t start_row) override;
+
+	void Update(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids,
+	            idx_t update_count) override;
+	void UpdateColumn(TransactionData transaction, const vector<column_t> &column_path, Vector &update_vector,
+	                  row_t *row_ids, idx_t update_count, idx_t depth) override;
+
+	void CommitDropColumn() override;
+
+	unique_ptr<ColumnCheckpointState> CreateCheckpointState(RowGroup &row_group,
+	                                                        PartialBlockManager &partial_block_manager) override;
+	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, ColumnCheckpointInfo &info) override;
+
+	void CheckpointScan(ColumnSegment &segment, ColumnScanState &state, idx_t row_group_start, idx_t count,
+	                    Vector &scan_vector) override;
+
+	bool IsPersistent() override;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/segment_lock.hpp

@@ -31,6 +31,10 @@ public:
 		return *this;
 	}
 
+	void Release() {
+		lock.unlock();
+	}
+
 private:
 	unique_lock<mutex> lock;
 };

package/src/duckdb/src/include/duckdb/storage/table/segment_tree.hpp

@@ -38,7 +38,7 @@ public:
 
 	//! Locks the segment tree. All methods to the segment tree either lock the segment tree, or take an already
 	//! obtained lock.
-	SegmentLock Lock() {
+	SegmentLock Lock() const {
 		return SegmentLock(node_lock);
 	}
 
@@ -76,12 +76,15 @@ public:
 		auto l = Lock();
 		return ReferenceSegments(l);
 	}
+	const vector<SegmentNode<T>> &ReferenceLoadedSegments(SegmentLock &l) const {
+		return nodes;
+	}
 
 	idx_t GetSegmentCount() {
 		auto l = Lock();
 		return GetSegmentCount(l);
 	}
-	idx_t GetSegmentCount(SegmentLock &l) {
+	idx_t GetSegmentCount(SegmentLock &l) const {
 		return nodes.size();
 	}
 	//! Gets a pointer to the nth segment. Negative numbers start from the back.
@@ -180,10 +183,10 @@ public:
 	//! Erase all segments after a specific segment
 	void EraseSegments(SegmentLock &l, idx_t segment_start) {
 		LoadAllSegments(l);
-		if (segment_start >= nodes.size() - 1) {
+		if (segment_start >= nodes.size()) {
 			return;
 		}
-		nodes.erase(nodes.begin() + UnsafeNumericCast<int64_t>(segment_start) + 1, nodes.end());
+		nodes.erase(nodes.begin() + UnsafeNumericCast<int64_t>(segment_start), nodes.end());
 	}
 
 	//! Get the segment index of the column segment for the given row
@@ -216,7 +219,14 @@ public:
 		// binary search to find the node
 		while (lower <= upper) {
 			idx_t index = (lower + upper) / 2;
-			D_ASSERT(index < nodes.size());
+			if (index >= nodes.size()) {
+				string segments;
+				for (auto &entry : nodes) {
+					segments += StringUtil::Format("Start %d Count %d", entry.row_start, entry.node->count.load());
+				}
+				throw InternalException("Segment tree index not found for row number %d\nSegments:%s", row_number,
+				                        segments);
+			}
 			auto &entry = nodes[index];
 			D_ASSERT(entry.row_start == entry.node->start);
 			if (row_number < entry.row_start) {
@@ -252,6 +262,10 @@ public:
 		return SegmentIterationHelper(*this);
 	}
 
+	SegmentIterationHelper Segments(SegmentLock &l) {
+		return SegmentIterationHelper(*this, l);
+	}
+
 	void Reinitialize() {
 		if (nodes.empty()) {
 			return;
@@ -274,37 +288,42 @@ protected:
 		return nullptr;
 	}
 
+	T *GetRootSegmentInternal() const {
+		return nodes.empty() ? nullptr : nodes[0].node.get();
+	}
+
 private:
 	//! The nodes in the tree, can be binary searched
 	vector<SegmentNode<T>> nodes;
 	//! Lock to access or modify the nodes
-	mutex node_lock;
+	mutable mutex node_lock;
 
 private:
-	T *GetRootSegmentInternal() {
-		return nodes.empty() ? nullptr : nodes[0].node.get();
-	}
-
 	class SegmentIterationHelper {
 	public:
 		explicit SegmentIterationHelper(SegmentTree &tree) : tree(tree) {
 		}
+		SegmentIterationHelper(SegmentTree &tree, SegmentLock &l) : tree(tree), lock(l) {
+		}
 
 	private:
 		SegmentTree &tree;
+		optional_ptr<SegmentLock> lock;
 
 	private:
 		class SegmentIterator {
 		public:
-			SegmentIterator(SegmentTree &tree_p, T *current_p) : tree(tree_p), current(current_p) {
+			SegmentIterator(SegmentTree &tree_p, T *current_p, optional_ptr<SegmentLock> lock)
+			    : tree(tree_p), current(current_p), lock(lock) {
 			}
 
 			SegmentTree &tree;
 			T *current;
+			optional_ptr<SegmentLock> lock;
 
 		public:
 			void Next() {
-				current = tree.GetNextSegment(current);
+				current = lock ? tree.GetNextSegment(*lock, current) : tree.GetNextSegment(current);
 			}
 
 			SegmentIterator &operator++() {
@@ -322,10 +341,11 @@ private:
 
 	public:
 		SegmentIterator begin() { // NOLINT: match stl API
-			return SegmentIterator(tree, tree.GetRootSegment());
+			auto root = lock ? tree.GetRootSegment(*lock) : tree.GetRootSegment();
+			return SegmentIterator(tree, root, lock);
 		}
 		SegmentIterator end() { // NOLINT: match stl API
-			return SegmentIterator(tree, nullptr);
+			return SegmentIterator(tree, nullptr, lock);
 		}
 	};
 

package/src/duckdb/src/include/duckdb/storage/table/standard_column_data.hpp

@@ -65,6 +65,7 @@ public:
 	                          vector<duckdb::ColumnSegmentInfo> &result) override;
 
 	bool IsPersistent() override;
+	bool HasAnyChanges() const override;
 	PersistentColumnData Serialize() override;
 	void InitializeColumn(PersistentColumnData &column_data, BaseStatistics &target_stats) override;
 

package/src/duckdb/src/include/duckdb/storage/table/struct_column_data.hpp

@@ -59,6 +59,7 @@ public:
 	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, ColumnCheckpointInfo &info) override;
 
 	bool IsPersistent() override;
+	bool HasAnyChanges() const override;
 	PersistentColumnData Serialize() override;
 	void InitializeColumn(PersistentColumnData &column_data, BaseStatistics &target_stats) override;
 

package/src/duckdb/src/include/duckdb/storage/table/table_index_list.hpp

@@ -20,47 +20,32 @@ class LocalTableStorage;
 struct IndexStorageInfo;
 struct DataTableInfo;
 
+//! IndexBindState to transition index binding phases preventing lock order inversion.
+enum class IndexBindState : uint8_t { UNBOUND, BINDING, BOUND };
+
+//! IndexEntry contains an atomic in addition to the index to ensure correct binding.
+struct IndexEntry {
+	explicit IndexEntry(unique_ptr<Index> index);
+	atomic<IndexBindState> bind_state;
+	unique_ptr<Index> index;
+};
+
 class TableIndexList {
 public:
-	//! Scan the indexes, invoking the callback method for every entry.
+	//! Scan the index entries, invoking the callback method for every entry.
 	template <class T>
 	void Scan(T &&callback) {
-		lock_guard<mutex> lock(indexes_lock);
-		for (auto &index : indexes) {
-			if (callback(*index)) {
+		lock_guard<mutex> lock(index_entries_lock);
+		for (auto &entry : index_entries) {
+			if (callback(*entry->index)) {
 				break;
 			}
 		}
 	}
 
-	//! Scan the indexes, invoking the callback method for every bound entry of type T.
-	template <class T, class FUNC>
-	void ScanBound(FUNC &&callback) {
-		lock_guard<mutex> lock(indexes_lock);
-		for (auto &index : indexes) {
-			if (index->IsBound() && T::TYPE_NAME == index->GetIndexType()) {
-				if (callback(index->Cast<T>())) {
-					break;
-				}
-			}
-		}
-	}
-
-	// Bind any unbound indexes of type T and invoke the callback method.
-	template <class T, class FUNC>
-	void BindAndScan(ClientContext &context, DataTableInfo &table_info, FUNC &&callback) {
-		// FIXME: optimize this by only looping through the indexes once without re-acquiring the lock.
-		InitializeIndexes(context, table_info, T::TYPE_NAME);
-		ScanBound<T>(callback);
-	}
-
-	//! Returns a reference to the indexes.
-	const vector<unique_ptr<Index>> &Indexes() const {
-		return indexes;
-	}
-	//! Adds an index to the list of indexes.
+	//! Adds an index entry to the list of index entries.
 	void AddIndex(unique_ptr<Index> index);
-	//! Removes an index from the list of indexes.
+	//! Removes an index entry from the list of index entries.
 	void RemoveIndex(const string &name);
 	//! Removes all remaining memory of an index after dropping the catalog entry.
 	void CommitDrop(const string &name);
@@ -68,30 +53,54 @@ public:
 	bool NameIsUnique(const string &name);
 	//! Returns an optional pointer to the index matching the name.
 	optional_ptr<BoundIndex> Find(const string &name);
-	//! Initializes unknown indexes that are possibly present after an extension load, optionally throwing an exception
-	//! on failure.
-	void InitializeIndexes(ClientContext &context, DataTableInfo &table_info, const char *index_type = nullptr);
-	//! Returns true, if there are no indexes in this list.
-	bool Empty();
-	//! Returns the number of indexes in this list.
-	idx_t Count();
+	//! Binds unbound indexes possibly present after loading an extension.
+	void Bind(ClientContext &context, DataTableInfo &table_info, const char *index_type = nullptr);
+	//! Returns true, if there are no index entries.
+	bool Empty() {
+		lock_guard<mutex> lock(index_entries_lock);
+		return index_entries.empty();
+	}
+	//! Returns the number of index entries.
+	idx_t Count() {
+		lock_guard<mutex> lock(index_entries_lock);
+		return index_entries.size();
+	}
+	//! Returns true, if there are unbound indexes.
+	bool HasUnbound() {
+		lock_guard<mutex> lock(index_entries_lock);
+		return unbound_count != 0;
+	}
 	//! Overwrite this list with the other list.
-	void Move(TableIndexList &other);
+	void Move(TableIndexList &other) {
+		D_ASSERT(index_entries.empty());
+		index_entries = std::move(other.index_entries);
+	}
+	//! Returns true, if all indexes
 	//! Find the foreign key matching the keys.
 	optional_ptr<Index> FindForeignKeyIndex(const vector<PhysicalIndex> &fk_keys, const ForeignKeyType fk_type);
 	//! Verify a foreign key constraint.
 	void VerifyForeignKey(optional_ptr<LocalTableStorage> storage, const vector<PhysicalIndex> &fk_keys,
 	                      DataChunk &chunk, ConflictManager &conflict_manager);
-	//! Get the combined column ids of the indexes in this list.
+	//! Get the combined column ids of the indexes.
 	unordered_set<column_t> GetRequiredColumns();
-	//! Serialize all indexes of this table.
-	vector<IndexStorageInfo> GetStorageInfos(const case_insensitive_map_t<Value> &options);
+	//! Serialize all indexes of the table.
+	vector<IndexStorageInfo> SerializeToDisk(QueryContext context, const case_insensitive_map_t<Value> &options);
+
+public:
+	//! Initialize an index_chunk from a table.
+	static void InitializeIndexChunk(DataChunk &index_chunk, const vector<LogicalType> &table_types,
+	                                 vector<StorageIndex> &mapped_column_ids, DataTableInfo &data_table_info);
+	//! Reference the indexed columns of a table chunk.
+	static void ReferenceIndexChunk(DataChunk &table_chunk, DataChunk &index_chunk,
+	                                vector<StorageIndex> &mapped_column_ids);
 
 private:
-	//! A lock to prevent any concurrent changes to the indexes.
-	mutex indexes_lock;
-	//! Indexes associated with the table.
-	vector<unique_ptr<Index>> indexes;
+	//! A lock to prevent any concurrent changes to the index entries.
+	mutex index_entries_lock;
+	//! The index entries of the table.
+	vector<unique_ptr<IndexEntry>> index_entries;
+	//! Contains the number of unbound indexes.
+	idx_t unbound_count = 0;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/update_segment.hpp

@@ -81,6 +81,8 @@ public:
 	typedef void (*rollback_update_function_t)(UpdateInfo &base_info, UpdateInfo &rollback_info);
 	typedef idx_t (*statistics_update_function_t)(UpdateSegment *segment, SegmentStatistics &stats,
 	                                              UnifiedVectorFormat &update, idx_t count, SelectionVector &sel);
+	typedef idx_t (*get_effective_updates_t)(UnifiedVectorFormat &update_format, row_t *ids, idx_t count,
+	                                         SelectionVector &sel, Vector &base_data, idx_t id_offset);
 
 private:
 	initialize_update_function_t initialize_update_function;
@@ -91,6 +93,7 @@ private:
 	fetch_row_function_t fetch_row_function;
 	rollback_update_function_t rollback_update_function;
 	statistics_update_function_t statistics_update_function;
+	get_effective_updates_t get_effective_updates;
 
 private:
 	UndoBufferPointer GetUpdateNode(StorageLockKey &lock, idx_t vector_idx) const;