duckdb 0.8.2-dev4025.0 → 0.8.2-dev4126.0

package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp

@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 #pragma once
 
+#include "duckdb/execution/index/fixed_size_allocator.hpp"
 #include "duckdb/execution/index/art/art.hpp"
-#include "duckdb/execution/index/art/fixed_size_allocator.hpp"
 #include "duckdb/execution/index/art/node.hpp"
 
 namespace duckdb {
@@ -17,30 +17,28 @@ namespace duckdb {
 class ARTKey;
 
 //! The Prefix is a special node type that contains up to PREFIX_SIZE bytes, and one byte for the count,
-//! and a Node pointer. This pointer either points to another prefix
-//! node or the 'actual' ART node.
+//! and a Node pointer. This pointer either points to a prefix node or another Node.
 class Prefix {
 public:
+	//! Delete copy constructors, as any Prefix can never own its memory
+	Prefix(const Prefix &) = delete;
+	Prefix &operator=(const Prefix &) = delete;
+
 	//! Up to PREFIX_SIZE bytes of prefix data and the count
 	uint8_t data[Node::PREFIX_SIZE + 1];
-	//! A pointer to the next ART node
+	//! A pointer to the next Node
 	Node ptr;
 
 public:
 	//! Get a new empty prefix node, might cause a new buffer allocation
 	static Prefix &New(ART &art, Node &node);
 	//! Create a new prefix node containing a single byte and a pointer to a next node
-	static Prefix &New(ART &art, Node &node, uint8_t byte, Node next);
+	static Prefix &New(ART &art, Node &node, uint8_t byte, const Node &next = Node());
 	//! Get a new chain of prefix nodes, might cause new buffer allocations,
 	//! with the node parameter holding the tail of the chain
 	static void New(ART &art, reference<Node> &node, const ARTKey &key, const uint32_t depth, uint32_t count);
 	//! Free the node (and its subtree)
 	static void Free(ART &art, Node &node);
-	//! Get a reference to the prefix
-	static inline Prefix &Get(const ART &art, const Node ptr) {
-		D_ASSERT(!ptr.IsSerialized());
-		return *Node::GetAllocator(art, NType::PREFIX).Get<Prefix>(ptr);
-	}
 
 	//! Initializes a merge by incrementing the buffer ID of the prefix and its child node(s)
 	static void InitializeMerge(ART &art, Node &node, const ARTFlags &flags);
@@ -51,13 +49,17 @@ public:
 	static void Concatenate(ART &art, Node &prefix_node, const uint8_t byte, Node &child_prefix_node);
 	//! Traverse a prefix and a key until (1) encountering a non-prefix node, or (2) encountering
 	//! a mismatching byte, in which case depth indexes the mismatching byte in the key
-	static idx_t Traverse(ART &art, reference<Node> &prefix_node, const ARTKey &key, idx_t &depth);
+	static idx_t Traverse(ART &art, reference<const Node> &prefix_node, const ARTKey &key, idx_t &depth);
+	//! Traverse a prefix and a key until (1) encountering a non-prefix node, or (2) encountering
+	//! a mismatching byte, in which case depth indexes the mismatching byte in the key
+	static idx_t TraverseMutable(ART &art, reference<Node> &prefix_node, const ARTKey &key, idx_t &depth);
 	//! Traverse two prefixes to find (1) that they match (so far), or (2) that they have a mismatching position,
-	//! or (3) that one prefix contains the other prefix
+	//! or (3) that one prefix contains the other prefix. This function aids in merging Nodes, and, therefore,
+	//! the nodes are not const
 	static bool Traverse(ART &art, reference<Node> &l_node, reference<Node> &r_node, idx_t &mismatch_position);
 	//! Returns the byte at position
 	static inline uint8_t GetByte(const ART &art, const Node &prefix_node, const idx_t position) {
-		auto prefix = Prefix::Get(art, prefix_node);
+		auto &prefix = Node::Ref<const Prefix>(art, prefix_node, NType::PREFIX);
 		D_ASSERT(position < Node::PREFIX_SIZE);
 		D_ASSERT(position < prefix.data[Node::PREFIX_SIZE]);
 		return prefix.data[position];
@@ -71,12 +73,7 @@ public:
 	static void Split(ART &art, reference<Node> &prefix_node, Node &child_node, idx_t position);
 
 	//! Returns the string representation of the node, or only traverses and verifies the node and its subtree
-	static string VerifyAndToString(ART &art, Node &node, const bool only_verify);
-
-	//! Serialize this node and all subsequent nodes
-	static BlockPointer Serialize(ART &art, Node &node, MetadataWriter &writer);
-	//! Deserialize this node and all subsequent prefix nodes
-	static void Deserialize(ART &art, Node &node, MetadataReader &reader);
+	static string VerifyAndToString(ART &art, const Node &node, const bool only_verify);
 
 	//! Vacuum the child of the node
 	static void Vacuum(ART &art, Node &node, const ARTFlags &flags);
@@ -88,8 +85,5 @@ private:
 	//! Appends the other_prefix and all its subsequent prefix nodes to this prefix node.
 	//! Also frees all copied/appended nodes
 	void Append(ART &art, Node other_prefix);
-	//! Get the total count of bytes in the chain of prefixes, with the node reference pointing to first non-prefix node
-	static idx_t TotalCount(ART &art, reference<Node> &node);
 };
-
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/index/fixed_size_allocator.hpp

@@ -0,0 +1,126 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/index/fixed_size_allocator.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/types/validity_mask.hpp"
+#include "duckdb/common/unordered_set.hpp"
+#include "duckdb/storage/buffer_manager.hpp"
+#include "duckdb/storage/metadata/metadata_manager.hpp"
+#include "duckdb/storage/metadata/metadata_writer.hpp"
+#include "duckdb/execution/index/fixed_size_buffer.hpp"
+#include "duckdb/execution/index/index_pointer.hpp"
+#include "duckdb/common/unordered_map.hpp"
+#include "duckdb/common/constants.hpp"
+#include "duckdb/common/map.hpp"
+
+namespace duckdb {
+
+//! The FixedSizeAllocator provides pointers to fixed-size memory segments of pre-allocated memory buffers.
+//! The pointers are IndexPointers, and the leftmost byte (metadata) must always be zero.
+//! It is also possible to directly request a C++ pointer to the underlying segment of an index pointer.
+class FixedSizeAllocator {
+public:
+	//! We can vacuum 10% or more of the total in-memory footprint
+	static constexpr uint8_t VACUUM_THRESHOLD = 10;
+
+	//! Constants for fast offset calculations in the bitmask
+	static constexpr idx_t BASE[] = {0x00000000FFFFFFFF, 0x0000FFFF, 0x00FF, 0x0F, 0x3, 0x1};
+	static constexpr uint8_t SHIFT[] = {32, 16, 8, 4, 2, 1};
+
+public:
+	FixedSizeAllocator(const idx_t segment_size, BlockManager &block_manager);
+
+	//! Block manager of the database instance
+	BlockManager &block_manager;
+	//! Buffer manager of the database instance
+	BufferManager &buffer_manager;
+	//! Metadata manager for (de)serialization
+	MetadataManager &metadata_manager;
+
+public:
+	//! Get a new IndexPointer to a segment, might cause a new buffer allocation
+	IndexPointer New();
+	//! Free the segment of the IndexPointer
+	void Free(const IndexPointer ptr);
+	//! Returns a pointer of type T to a segment. If dirty is false, then T should be a const class
+	template <class T>
+	inline T *Get(const IndexPointer ptr, const bool dirty = true) {
+		return (T *)Get(ptr, dirty);
+	}
+
+	//! Resets the allocator, e.g., during 'DELETE FROM table'
+	void Reset();
+
+	//! Returns the in-memory usage in bytes
+	inline idx_t GetMemoryUsage() const;
+
+	//! Returns the upper bound of the available buffer IDs, i.e., upper_bound > max_buffer_id
+	idx_t GetUpperBoundBufferId() const;
+	//! Merge another FixedSizeAllocator into this allocator. Both must have the same segment size
+	void Merge(FixedSizeAllocator &other);
+
+	//! Initialize a vacuum operation, and return true, if the allocator needs a vacuum
+	bool InitializeVacuum();
+	//! Finalize a vacuum operation by freeing all vacuumed buffers
+	void FinalizeVacuum();
+	//! Returns true, if an IndexPointer qualifies for a vacuum operation, and false otherwise
+	inline bool NeedsVacuum(const IndexPointer ptr) const {
+		if (vacuum_buffers.find(ptr.GetBufferId()) != vacuum_buffers.end()) {
+			return true;
+		}
+		return false;
+	}
+	//! Vacuums an IndexPointer
+	IndexPointer VacuumPointer(const IndexPointer ptr);
+
+	//! Serializes all in-memory buffers and the metadata
+	BlockPointer Serialize(MetadataWriter &writer);
+	//! Deserializes all metadata
+	void Deserialize(const BlockPointer &block_pointer);
+
+private:
+	//! Allocation size of one segment in a buffer
+	//! We only need this value to calculate bitmask_count, bitmask_offset, and
+	//! available_segments_per_buffer
+	idx_t segment_size;
+
+	//! Number of validity_t values in the bitmask
+	idx_t bitmask_count;
+	//! First starting byte of the payload (segments)
+	idx_t bitmask_offset;
+	//! Number of possible segment allocations per buffer
+	idx_t available_segments_per_buffer;
+
+	//! Total number of allocated segments in all buffers
+	//! We can recalculate this by iterating over all buffers
+	idx_t total_segment_count;
+
+	//! Buffers containing the segments
+	unordered_map<idx_t, FixedSizeBuffer> buffers;
+	//! Buffers with free space
+	unordered_set<idx_t> buffers_with_free_space;
+	//! Buffers qualifying for a vacuum (helper field to allow for fast NeedsVacuum checks)
+	unordered_set<idx_t> vacuum_buffers;
+
+private:
+	//! Returns the data_ptr_t to a segment, and sets the dirty flag of the buffer containing that segment
+	inline data_ptr_t Get(const IndexPointer ptr, const bool dirty = true) {
+		D_ASSERT(ptr.GetOffset() < available_segments_per_buffer);
+		D_ASSERT(buffers.find(ptr.GetBufferId()) != buffers.end());
+		auto &buffer = buffers.find(ptr.GetBufferId())->second;
+		auto buffer_ptr = buffer.Get(dirty);
+		return buffer_ptr + ptr.GetOffset() * segment_size + bitmask_offset;
+	}
+	//! Returns the first free offset in a bitmask
+	uint32_t GetOffset(ValidityMask &mask, const idx_t segment_count);
+	//! Returns an available buffer id
+	idx_t GetAvailableBufferId() const;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/index/fixed_size_buffer.hpp

@@ -0,0 +1,79 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/index/fixed_size_buffer.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/typedefs.hpp"
+#include "duckdb/storage/buffer/block_handle.hpp"
+#include "duckdb/storage/buffer/buffer_handle.hpp"
+
+namespace duckdb {
+
+class FixedSizeAllocator;
+class MetadataWriter;
+
+//! A fixed-size buffer holds fixed-size segments of data. It lazily deserializes a buffer, if on-disk and not
+//! yet in memory, and it only serializes dirty and non-written buffers to disk during
+//! serialization.
+class FixedSizeBuffer {
+public:
+	//! Constructor for a new in-memory buffer
+	explicit FixedSizeBuffer(BlockManager &block_manager);
+	//! Constructor for deserializing buffer metadata from disk
+	FixedSizeBuffer(BlockManager &block_manager, const idx_t segment_count, const block_id_t &block_id);
+
+	//! Block manager of the database instance
+	BlockManager &block_manager;
+
+	//! The number of allocated segments
+	idx_t segment_count;
+
+	//! True: the in-memory buffer is no longer consistent with a (possibly existing) copy on disk
+	bool dirty;
+	//! True: can be vacuumed after the vacuum operation
+	bool vacuum;
+
+public:
+	//! Returns true, if the buffer is in-memory
+	inline bool InMemory() const {
+		return buffer_handle.IsValid();
+	}
+	//! Returns true, if the block is on-disk
+	inline bool OnDisk() const {
+		return (block_handle != nullptr) && (block_handle->BlockId() < MAXIMUM_BLOCK);
+	}
+	//! Returns the block ID
+	inline block_id_t BlockId() const {
+		D_ASSERT(OnDisk());
+		return block_handle->BlockId();
+	}
+	//! Returns a pointer to the buffer in memory, and calls Deserialize, if the buffer is not in memory
+	inline data_ptr_t Get(const bool dirty_p = true) {
+		if (!InMemory()) {
+			Pin();
+		}
+		if (dirty_p) {
+			dirty = dirty_p;
+		}
+		return buffer_handle.Ptr();
+	}
+	//! Destroys the in-memory buffer and the on-disk block
+	void Destroy();
+	//! Serializes a buffer (if dirty or not on disk)
+	void Serialize();
+	//! Pin a buffer (if not in-memory)
+	void Pin();
+
+private:
+	//! The buffer handle of the in-memory buffer
+	BufferHandle buffer_handle;
+	//! The block handle of the on-disk buffer
+	shared_ptr<BlockHandle> block_handle;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/index/index_pointer.hpp

@@ -0,0 +1,96 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/index/index_pointer.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/typedefs.hpp"
+
+namespace duckdb {
+
+class IndexPointer {
+public:
+	//! Bit-shifting
+	static constexpr idx_t SHIFT_OFFSET = 32;
+	static constexpr idx_t SHIFT_METADATA = 56;
+	//! AND operations
+	static constexpr idx_t AND_OFFSET = 0x0000000000FFFFFF;
+	static constexpr idx_t AND_BUFFER_ID = 0x00000000FFFFFFFF;
+	static constexpr idx_t AND_METADATA = 0xFF00000000000000;
+
+public:
+	//! Constructs an empty IndexPointer
+	IndexPointer() : data(0) {};
+	//! Constructs an in-memory IndexPointer with a buffer ID and an offset
+	IndexPointer(const uint32_t buffer_id, const uint32_t offset) : data(0) {
+		auto shifted_offset = ((idx_t)offset) << SHIFT_OFFSET;
+		data += shifted_offset;
+		data += buffer_id;
+	};
+
+public:
+	//! Get data (all 64 bits)
+	inline idx_t Get() const {
+		return data;
+	}
+	//! Set data (all 64 bits)
+	inline void Set(const idx_t data_p) {
+		data = data_p;
+	}
+
+	//! Returns false, if the metadata is empty
+	inline bool HasMetadata() const {
+		return data & AND_METADATA;
+	}
+	//! Get metadata (zero to 7th bit)
+	inline uint8_t GetMetadata() const {
+		return data >> SHIFT_METADATA;
+	}
+	//! Set metadata (zero to 7th bit)
+	inline void SetMetadata(const uint8_t metadata) {
+		data += (idx_t)metadata << SHIFT_METADATA;
+	}
+
+	//! Get the offset (8th to 23rd bit)
+	inline idx_t GetOffset() const {
+		auto offset = data >> SHIFT_OFFSET;
+		return offset & AND_OFFSET;
+	}
+	//! Get the buffer ID (24th to 63rd bit)
+	inline idx_t GetBufferId() const {
+		return data & AND_BUFFER_ID;
+	}
+
+	//! Resets the IndexPointer
+	inline void Clear() {
+		data = 0;
+	}
+
+	//! Adds an idx_t to a buffer ID, the rightmost 32 bits of data contain the buffer ID
+	inline void IncreaseBufferId(const idx_t summand) {
+		data += summand;
+	}
+
+	//! Comparison operator
+	inline bool operator==(const IndexPointer &ptr) const {
+		return data == ptr.data;
+	}
+
+private:
+	//! Data holds all the information contained in an IndexPointer
+	//! [0 - 7: metadata,
+	//! 8 - 23: offset, 24 - 63: buffer ID]
+	//! NOTE: we do not use bit fields because when using bit fields Windows compiles
+	//! the IndexPointer class into 16 bytes instead of the intended 8 bytes, doubling the
+	//! space requirements
+	//! https://learn.microsoft.com/en-us/cpp/cpp/cpp-bit-fields?view=msvc-170
+	idx_t data;
+};
+
+static_assert(sizeof(IndexPointer) == sizeof(idx_t), "Invalid size for IndexPointer.");
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp

@@ -68,7 +68,7 @@ public:
 	void Signal(idx_t n);
 
 	//! Yield to other threads
-	void YieldThread();
+	static void YieldThread();
 
 	//! Set the allocator flush threshold
 	void SetAllocatorFlushTreshold(idx_t threshold);

package/src/duckdb/src/include/duckdb/storage/block.hpp

@@ -35,7 +35,7 @@ struct BlockPointer {
 	block_id_t block_id;
 	uint32_t offset;
 
-	bool IsValid() {
+	bool IsValid() const {
 		return block_id != INVALID_BLOCK;
 	}
 };

package/src/duckdb/src/include/duckdb/storage/index.hpp

@@ -54,8 +54,6 @@ public:
 
 	//! Attached database instance
 	AttachedDatabase &db;
-	//! Buffer manager of the database instance
-	BufferManager &buffer_manager;
 
 public:
 	//! Initialize a single predicate scan on the index with the given expression and column IDs
@@ -85,6 +83,10 @@ public:
 	//! Performs constraint checking for a chunk of input data
 	virtual void CheckConstraintsForChunk(DataChunk &input, ConflictManager &conflict_manager) = 0;
 
+	//! Deletes all data from the index. The lock obtained from InitializeLock must be held
+	virtual void CommitDrop(IndexLock &index_lock) = 0;
+	//! Deletes all data from the index
+	void CommitDrop();
 	//! Delete a chunk of entries from the index. The lock obtained from InitializeLock must be held
 	virtual void Delete(IndexLock &state, DataChunk &entries, Vector &row_identifiers) = 0;
 	//! Obtains a lock and calls Delete while holding that lock
@@ -125,11 +127,11 @@ public:
 		return (constraint_type == IndexConstraintType::FOREIGN);
 	}
 
-	//! Serializes the index and returns the pair of block_id offset positions
+	//! Serializes the index to disk
 	virtual BlockPointer Serialize(MetadataWriter &writer);
-	//! Returns the serialized data pointer to the block and offset of the serialized index
-	BlockPointer GetSerializedDataPointer() const {
-		return serialized_data_pointer;
+	//! Returns the serialized root block pointer
+	BlockPointer GetRootBlockPointer() const {
+		return root_block_pointer;
 	}
 
 	//! Execute the index expressions on an input chunk
@@ -139,8 +141,8 @@ public:
 protected:
 	//! Lock used for any changes to the index
 	mutex lock;
-	//! Pointer to serialized index data
-	BlockPointer serialized_data_pointer;
+	//! Pointer to the index on disk
+	BlockPointer root_block_pointer;
 
 private:
 	//! Bound expressions used during expression execution

package/src/duckdb/src/include/duckdb/storage/metadata/metadata_writer.hpp

@@ -14,6 +14,9 @@ namespace duckdb {
 
 class MetadataWriter : public Serializer {
 public:
+	MetadataWriter(const MetadataWriter &) = delete;
+	MetadataWriter &operator=(const MetadataWriter &) = delete;
+
 	explicit MetadataWriter(MetadataManager &manager);
 	~MetadataWriter() override;
 

package/src/duckdb/src/planner/binder/expression/bind_function_expression.cpp

@@ -69,21 +69,30 @@ BindResult ExpressionBinder::BindExpression(FunctionExpression &function, idx_t
 	}
 
 	switch (func->type) {
-	case CatalogType::SCALAR_FUNCTION_ENTRY:
+	case CatalogType::SCALAR_FUNCTION_ENTRY: {
 		// scalar function
 
 		// check for lambda parameters, ignore ->> operator (JSON extension)
+		bool try_bind_lambda = false;
 		if (function.function_name != "->>") {
 			for (auto &child : function.children) {
 				if (child->expression_class == ExpressionClass::LAMBDA) {
-					return BindLambdaFunction(function, func->Cast<ScalarFunctionCatalogEntry>(), depth);
+					try_bind_lambda = true;
 				}
 			}
 		}
 
+		if (try_bind_lambda) {
+			auto result = BindLambdaFunction(function, func->Cast<ScalarFunctionCatalogEntry>(), depth);
+			if (!result.HasError()) {
+				// Lambda bind successful
+				return result;
+			}
+		}
+
 		// other scalar function
 		return BindFunction(function, func->Cast<ScalarFunctionCatalogEntry>(), depth);
-
+	}
 	case CatalogType::MACRO_ENTRY:
 		// macro function
 		return BindMacro(function, func->Cast<ScalarMacroCatalogEntry>(), depth, expr_ptr);
@@ -134,7 +143,7 @@ BindResult ExpressionBinder::BindLambdaFunction(FunctionExpression &function, Sc
 	string error;
 
 	if (function.children.size() != 2) {
-		throw BinderException("Invalid function arguments!");
+		return BindResult("Invalid function arguments!");
 	}
 	D_ASSERT(function.children[1]->GetExpressionClass() == ExpressionClass::LAMBDA);
 
@@ -148,7 +157,7 @@ BindResult ExpressionBinder::BindLambdaFunction(FunctionExpression &function, Sc
 	auto &list_child = BoundExpression::GetExpression(*function.children[0]);
 	if (list_child->return_type.id() != LogicalTypeId::LIST && list_child->return_type.id() != LogicalTypeId::SQLNULL &&
 	    list_child->return_type.id() != LogicalTypeId::UNKNOWN) {
-		throw BinderException(" Invalid LIST argument to " + function.function_name + "!");
+		return BindResult(" Invalid LIST argument to " + function.function_name + "!");
 	}
 
 	LogicalType list_child_type = list_child->return_type.id();

package/src/duckdb/src/storage/checkpoint/table_data_writer.cpp

@@ -69,9 +69,8 @@ void SingleFileTableDataWriter::FinalizeTable(TableStatistics &&global_stats, Da
 
 	// Write-off to metadata block ids and offsets of indexes
 	meta_data_writer.Write<idx_t>(index_pointers.size());
-	for (auto &block_info : index_pointers) {
-		meta_data_writer.Write<block_id_t>(block_info.block_id);
-		meta_data_writer.Write<uint32_t>(block_info.offset);
+	for (const auto &index_pointer : index_pointers) {
+		meta_data_writer.Write<BlockPointer>(index_pointer);
 	}
 }
 

package/src/duckdb/src/storage/checkpoint_manager.cpp

@@ -262,7 +262,8 @@ void CheckpointWriter::WriteSchema(SchemaCatalogEntry &schema) {
 void CheckpointReader::ReadSchema(ClientContext &context, MetadataReader &reader) {
 	// read the schema and create it in the catalog
 	auto info = CatalogEntry::Deserialize(reader);
-	// we set create conflict to ignore to ignore the failure of recreating the main schema
+
+	// we set create conflict to IGNORE_ON_CONFLICT, so that we can ignore a failure when recreating the main schema
 	info->on_conflict = OnCreateConflict::IGNORE_ON_CONFLICT;
 	catalog.CreateSchema(context, info->Cast<CreateSchemaInfo>());
 
@@ -336,14 +337,12 @@ void CheckpointReader::ReadSequence(ClientContext &context, MetadataReader &read
 // Indexes
 //===--------------------------------------------------------------------===//
 void CheckpointWriter::WriteIndex(IndexCatalogEntry &index_catalog) {
-	// The index data should already have been written as part of WriteTableData.
-	// Here, we need only serialize the pointer to that data.
-	auto root_offset = index_catalog.index->GetSerializedDataPointer();
+	// we write the index data in WriteTableData
+	// here, we only write the root pointer
+	const auto root_block_pointer = index_catalog.index->GetRootBlockPointer();
 	auto &metadata_writer = GetMetadataWriter();
 	index_catalog.Serialize(metadata_writer);
-	// Serialize the Block id and offset of root node
-	metadata_writer.Write(root_offset.block_id);
-	metadata_writer.Write(root_offset.offset);
+	metadata_writer.Write(root_block_pointer);
 }
 
 void CheckpointReader::ReadIndex(ClientContext &context, MetadataReader &reader) {
@@ -358,10 +357,8 @@ void CheckpointReader::ReadIndex(ClientContext &context, MetadataReader &reader)
 	auto &index_catalog = schema_catalog.CreateIndex(context, index_info, table_catalog)->Cast<DuckIndexEntry>();
 	index_catalog.info = table_catalog.GetStorage().info;
 
-	// we deserialize the index lazily, i.e., we do not need to load any node information
-	// except the root block id and offset
-	auto root_block_id = reader.Read<block_id_t>();
-	auto root_offset = reader.Read<uint32_t>();
+	// we deserialize the index lazily, i.e., we only load the root block pointer
+	const auto index_block_pointer = reader.Read<BlockPointer>();
 
 	// obtain the expressions of the ART from the index metadata
 	vector<unique_ptr<Expression>> unbound_expressions;
@@ -401,9 +398,8 @@ void CheckpointReader::ReadIndex(ClientContext &context, MetadataReader &reader)
 	switch (index_info.index_type) {
 	case IndexType::ART: {
 		auto &storage = table_catalog.GetStorage();
-		auto art =
-		    make_uniq<ART>(index_info.column_ids, TableIOManager::Get(storage), std::move(unbound_expressions),
-		                   index_info.constraint_type, storage.db, nullptr, BlockPointer(root_block_id, root_offset));
+		auto art = make_uniq<ART>(index_info.column_ids, TableIOManager::Get(storage), std::move(unbound_expressions),
+		                          index_info.constraint_type, storage.db, nullptr, index_block_pointer);
 		index_catalog.index = art.get();
 		storage.info->indexes.AddIndex(std::move(art));
 		break;
@@ -450,7 +446,7 @@ void CheckpointReader::ReadTableMacro(ClientContext &context, MetadataReader &re
 // Table Metadata
 //===--------------------------------------------------------------------===//
 void CheckpointWriter::WriteTable(TableCatalogEntry &table) {
-	// write the table meta data
+	// write the table metadata
 	table.Serialize(GetMetadataWriter());
 	// now we need to write the table data.
 	if (auto writer = GetTableDataWriter(table)) {
@@ -483,12 +479,11 @@ void CheckpointReader::ReadTableData(ClientContext &context, MetadataReader &rea
 	data_reader.ReadTableData();
 	bound_info.data->total_rows = reader.Read<idx_t>();
 
-	// Get any indexes block info
-	idx_t num_indexes = reader.Read<idx_t>();
-	for (idx_t i = 0; i < num_indexes; i++) {
-		auto idx_block_id = reader.Read<block_id_t>();
-		auto idx_offset = reader.Read<uint32_t>();
-		bound_info.indexes.emplace_back(idx_block_id, idx_offset);
+	// get the root block pointers of each index
+	idx_t index_count = reader.Read<idx_t>();
+	for (idx_t i = 0; i < index_count; i++) {
+		const auto index_pointer = reader.Read<BlockPointer>();
+		bound_info.indexes.emplace_back(index_pointer);
 	}
 }
 

package/src/duckdb/src/storage/data_table.cpp

@@ -429,13 +429,13 @@ void DataTable::VerifyForeignKeyConstraint(const BoundForeignKeyConstraint &bfk,
 	data_table.info->indexes.VerifyForeignKey(*dst_keys_ptr, dst_chunk, regular_conflicts);
 	regular_conflicts.Finalize();
 	auto &regular_matches = regular_conflicts.Conflicts();
-	// check whether or not the chunk can be inserted or deleted into the referenced table' transaction local storage
-	auto &local_storage = LocalStorage::Get(context, db);
 
+	// check if we can insert the chunk into the reference table's local storage
+	auto &local_storage = LocalStorage::Get(context, db);
 	bool error = IsForeignKeyConstraintError(is_append, count, regular_matches);
 	bool transaction_error = false;
-
 	bool transaction_check = local_storage.Find(data_table);
+
 	if (transaction_check) {
 		auto &transact_index = local_storage.GetIndexes(data_table);
 		transact_index.VerifyForeignKey(*dst_keys_ptr, dst_chunk, transaction_conflicts);

package/src/duckdb/src/storage/index.cpp

@@ -12,7 +12,7 @@ Index::Index(AttachedDatabase &db, IndexType type, TableIOManager &table_io_mana
              IndexConstraintType constraint_type_p)
 
     : type(type), table_io_manager(table_io_manager), column_ids(column_ids_p), constraint_type(constraint_type_p),
-      db(db), buffer_manager(BufferManager::GetBufferManager(db)) {
+      db(db) {
 
 	for (auto &expr : unbound_expressions) {
 		types.push_back(expr->return_type.InternalType());
@@ -39,6 +39,12 @@ PreservedError Index::Append(DataChunk &entries, Vector &row_identifiers) {
 	return Append(state, entries, row_identifiers);
 }
 
+void Index::CommitDrop() {
+	IndexLock index_lock;
+	InitializeLock(index_lock);
+	CommitDrop(index_lock);
+}
+
 void Index::Delete(DataChunk &entries, Vector &row_identifiers) {
 	IndexLock state;
 	InitializeLock(state);