duckdb 0.9.3-dev0.0 → 0.9.3-dev3.0

package/src/duckdb/src/include/duckdb/storage/compression/patas/patas_analyze.hpp

@@ -16,124 +16,24 @@ namespace duckdb {
 struct EmptyPatasWriter;
 
 template <class T>
-struct PatasAnalyzeState : public AnalyzeState {
-public:
-	using EXACT_TYPE = typename FloatingToExact<T>::type;
-
-	PatasAnalyzeState() : state((void *)this) {
-		state.AssignDataBuffer(nullptr);
-	}
-	PatasState<T, true> state;
-	idx_t group_idx = 0;
-	idx_t data_byte_size = 0;
-	idx_t metadata_byte_size = 0;
-	//! To optimally store NULL, we keep track of the directly previous value
-	EXACT_TYPE previous_value;
-
-public:
-	void WriteValue(EXACT_TYPE value, bool is_valid) {
-		if (!is_valid) {
-			value = previous_value;
-		}
-		//! Keep track of when a segment would end, to accurately simulate Reset()s in compress step
-		if (!HasEnoughSpace()) {
-			StartNewSegment();
-		}
-		patas::PatasCompression<EXACT_TYPE, true>::Store(value, state.patas_state);
-		previous_value = value;
-		group_idx++;
-		if (group_idx == PatasPrimitives::PATAS_GROUP_SIZE) {
-			StartNewGroup();
-		}
-	}
-
-	idx_t CurrentGroupMetadataSize() const {
-		idx_t metadata_size = 0;
-
-		// Offset to the data of the group
-		metadata_size += sizeof(uint32_t);
-		// Packed Trailing zeros + significant bytes + index_offsets for group
-		metadata_size += 2 * group_idx;
-		return metadata_size;
-	}
-
-	void StartNewSegment() {
-		StartNewGroup();
-		data_byte_size += UsedSpace();
-		metadata_byte_size += PatasPrimitives::HEADER_SIZE;
-		state.patas_state.byte_writer.SetStream(nullptr);
-	}
-
-	idx_t RequiredSpace() const {
-		idx_t required_space = 0;
-		required_space += sizeof(EXACT_TYPE);
-		required_space += sizeof(uint16_t);
-		return required_space;
-	}
-
-	void StartNewGroup() {
-		previous_value = 0;
-		metadata_byte_size += CurrentGroupMetadataSize();
-		group_idx = 0;
-		state.patas_state.Reset();
-	}
-
-	idx_t UsedSpace() const {
-		return state.patas_state.byte_writer.BytesWritten();
-	}
-
-	bool HasEnoughSpace() {
-		idx_t total_bytes_used = 0;
-		total_bytes_used += AlignValue(PatasPrimitives::HEADER_SIZE + UsedSpace() + RequiredSpace());
-		total_bytes_used += CurrentGroupMetadataSize();
-		total_bytes_used += metadata_byte_size;
-		return total_bytes_used <= Storage::BLOCK_SIZE;
-	}
-
-	idx_t TotalUsedBytes() const {
-		return metadata_byte_size + AlignValue(data_byte_size + UsedSpace());
-	}
-};
-
-struct EmptyPatasWriter {
-
-	template <class VALUE_TYPE>
-	static void Operation(VALUE_TYPE uncompressed_value, bool is_valid, void *state_p) {
-		using EXACT_TYPE = typename FloatingToExact<VALUE_TYPE>::type;
-
-		auto state_wrapper = (PatasAnalyzeState<VALUE_TYPE> *)state_p;
-		state_wrapper->WriteValue(Load<EXACT_TYPE>(const_data_ptr_cast(&uncompressed_value)), is_valid);
-	}
-};
+struct PatasAnalyzeState : public AnalyzeState {};
 
 template <class T>
 unique_ptr<AnalyzeState> PatasInitAnalyze(ColumnData &col_data, PhysicalType type) {
-	return make_uniq<PatasAnalyzeState<T>>();
+	// This compression type is deprecated
+	return nullptr;
 }
 
 template <class T>
 bool PatasAnalyze(AnalyzeState &state, Vector &input, idx_t count) {
-	auto &analyze_state = (PatasAnalyzeState<T> &)state;
-	UnifiedVectorFormat vdata;
-	input.ToUnifiedFormat(count, vdata);
-
-	auto data = UnifiedVectorFormat::GetData<T>(vdata);
-	for (idx_t i = 0; i < count; i++) {
-		auto idx = vdata.sel->get_index(i);
-		analyze_state.state.template Update<EmptyPatasWriter>(data[idx], vdata.validity.RowIsValid(idx));
-	}
-	return true;
+	throw InternalException("Patas has been deprecated, can no longer be used to compress data");
+	return false;
 }
 
 template <class T>
 idx_t PatasFinalAnalyze(AnalyzeState &state) {
-	auto &patas_state = (PatasAnalyzeState<T> &)state;
-	// Finish the last "segment"
-	patas_state.StartNewSegment();
-	const auto final_analyze_size = patas_state.TotalUsedBytes();
-	// Multiply the final size to factor in the extra cost of decompression time
-	const auto multiplier = 1.2;
-	return final_analyze_size * multiplier;
+	throw InternalException("Patas has been deprecated, can no longer be used to compress data");
+	return DConstants::INVALID_INDEX;
 }
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/compression/patas/patas_compress.hpp

@@ -30,204 +30,25 @@ namespace duckdb {
 // State
 
 template <class T>
-struct PatasCompressionState : public CompressionState {
-public:
-	using EXACT_TYPE = typename FloatingToExact<T>::type;
-
-	struct PatasWriter {
-
-		template <class VALUE_TYPE>
-		static void Operation(VALUE_TYPE value, bool is_valid, void *state_p) {
-			//! Need access to the CompressionState to be able to flush the segment
-			auto state_wrapper = (PatasCompressionState<VALUE_TYPE> *)state_p;
-
-			if (!state_wrapper->HasEnoughSpace()) {
-				// Segment is full
-				auto row_start = state_wrapper->current_segment->start + state_wrapper->current_segment->count;
-				state_wrapper->FlushSegment();
-				state_wrapper->CreateEmptySegment(row_start);
-			}
-
-			if (is_valid) {
-				NumericStats::Update<VALUE_TYPE>(state_wrapper->current_segment->stats.statistics, value);
-			}
-
-			state_wrapper->WriteValue(Load<EXACT_TYPE>(const_data_ptr_cast(&value)));
-		}
-	};
-
-	explicit PatasCompressionState(ColumnDataCheckpointer &checkpointer, PatasAnalyzeState<T> *analyze_state)
-	    : checkpointer(checkpointer),
-	      function(checkpointer.GetCompressionFunction(CompressionType::COMPRESSION_PATAS)) {
-		CreateEmptySegment(checkpointer.GetRowGroup().start);
-
-		state.data_ptr = (void *)this;
-		state.patas_state.packed_data_buffer.SetBuffer(packed_data);
-		state.patas_state.Reset();
-	}
-
-	ColumnDataCheckpointer &checkpointer;
-	CompressionFunction &function;
-	unique_ptr<ColumnSegment> current_segment;
-	BufferHandle handle;
-	idx_t group_idx = 0;
-	uint16_t packed_data[PatasPrimitives::PATAS_GROUP_SIZE];
-
-	// Ptr to next free spot in segment;
-	data_ptr_t segment_data;
-	data_ptr_t metadata_ptr;
-	uint32_t next_group_byte_index_start = PatasPrimitives::HEADER_SIZE;
-	// The total size of metadata in the current segment
-	idx_t metadata_byte_size = 0;
-
-	PatasState<T, false> state;
-
-public:
-	idx_t RequiredSpace() const {
-		idx_t required_space = sizeof(EXACT_TYPE);
-		// byte offset of data
-		required_space += sizeof(byte_index_t);
-		// byte size of the packed_data_block
-		required_space += sizeof(uint16_t);
-		return required_space;
-	}
-
-	// How many bytes the data occupies for the current segment
-	idx_t UsedSpace() const {
-		return state.patas_state.byte_writer.BytesWritten();
-	}
-
-	idx_t RemainingSpace() const {
-		return metadata_ptr - (handle.Ptr() + UsedSpace());
-	}
-
-	idx_t CurrentGroupMetadataSize() const {
-		idx_t metadata_size = 0;
-
-		metadata_size += sizeof(byte_index_t);
-		metadata_size += sizeof(uint16_t) * group_idx;
-		return metadata_size;
-	}
-
-	// The current segment has enough space to fit this new value
-	bool HasEnoughSpace() {
-		if (handle.Ptr() + AlignValue(PatasPrimitives::HEADER_SIZE + UsedSpace() + RequiredSpace()) >=
-		    (metadata_ptr - CurrentGroupMetadataSize())) {
-			return false;
-		}
-		return true;
-	}
-
-	void CreateEmptySegment(idx_t row_start) {
-		next_group_byte_index_start = PatasPrimitives::HEADER_SIZE;
-		group_idx = 0;
-		metadata_byte_size = 0;
-		auto &db = checkpointer.GetDatabase();
-		auto &type = checkpointer.GetType();
-		auto compressed_segment = ColumnSegment::CreateTransientSegment(db, type, row_start);
-		compressed_segment->function = function;
-		current_segment = std::move(compressed_segment);
-
-		auto &buffer_manager = BufferManager::GetBufferManager(db);
-		handle = buffer_manager.Pin(current_segment->block);
-
-		segment_data = handle.Ptr() + PatasPrimitives::HEADER_SIZE;
-		metadata_ptr = handle.Ptr() + Storage::BLOCK_SIZE;
-		state.AssignDataBuffer(segment_data);
-		state.patas_state.Reset();
-	}
-
-	void Append(UnifiedVectorFormat &vdata, idx_t count) {
-		auto data = UnifiedVectorFormat::GetData<T>(vdata);
-
-		for (idx_t i = 0; i < count; i++) {
-			auto idx = vdata.sel->get_index(i);
-			state.template Update<PatasWriter>(data[idx], vdata.validity.RowIsValid(idx));
-		}
-	}
-
-	void WriteValue(EXACT_TYPE value) {
-		current_segment->count++;
-		patas::PatasCompression<EXACT_TYPE, false>::Store(value, state.patas_state);
-		group_idx++;
-		if (group_idx == PatasPrimitives::PATAS_GROUP_SIZE) {
-			FlushGroup();
-		}
-	}
-
-	void FlushGroup() {
-		metadata_ptr -= sizeof(byte_index_t);
-		metadata_byte_size += sizeof(byte_index_t);
-		// Store where this groups data starts, relative to the start of the segment
-		Store<byte_index_t>(next_group_byte_index_start, metadata_ptr);
-		next_group_byte_index_start = PatasPrimitives::HEADER_SIZE + UsedSpace();
-
-		// Store the packed data blocks (7 + 6 + 3 bits)
-		metadata_ptr -= group_idx * sizeof(uint16_t);
-		metadata_byte_size += group_idx * sizeof(uint16_t);
-		memcpy(metadata_ptr, packed_data, sizeof(uint16_t) * group_idx);
-
-		state.patas_state.Reset();
-		group_idx = 0;
-	}
-
-	//! FIXME: only compact if the unused space meets a certain threshold (20%)
-	void FlushSegment() {
-		if (group_idx != 0) {
-			FlushGroup();
-		}
-		auto &checkpoint_state = checkpointer.GetCheckpointState();
-		auto dataptr = handle.Ptr();
-
-		// Compact the segment by moving the metadata next to the data.
-		idx_t bytes_used_by_data = PatasPrimitives::HEADER_SIZE + UsedSpace();
-		idx_t metadata_offset = AlignValue(bytes_used_by_data);
-		// Verify that the metadata_ptr does not cross this threshold
-		D_ASSERT(dataptr + metadata_offset <= metadata_ptr);
-		idx_t metadata_size = dataptr + Storage::BLOCK_SIZE - metadata_ptr;
-		idx_t total_segment_size = metadata_offset + metadata_size;
-#ifdef DEBUG
-		//! Copy the first 4 bytes of the metadata
-		uint32_t verify_bytes;
-		std::memcpy((void *)&verify_bytes, metadata_ptr, 4);
-#endif
-		memmove(dataptr + metadata_offset, metadata_ptr, metadata_size);
-#ifdef DEBUG
-		//! Now assert that the memmove was correct
-		D_ASSERT(verify_bytes == *(uint32_t *)(dataptr + metadata_offset));
-#endif
-		// Store the offset to the metadata
-		Store<uint32_t>(metadata_offset + metadata_size, dataptr);
-		handle.Destroy();
-		checkpoint_state.FlushSegment(std::move(current_segment), total_segment_size);
-	}
-
-	void Finalize() {
-		FlushSegment();
-		current_segment.reset();
-	}
-};
+struct PatasCompressionState : public CompressionState {};
 
 // Compression Functions
 
 template <class T>
 unique_ptr<CompressionState> PatasInitCompression(ColumnDataCheckpointer &checkpointer,
                                                   unique_ptr<AnalyzeState> state) {
-	return make_uniq<PatasCompressionState<T>>(checkpointer, (PatasAnalyzeState<T> *)state.get());
+	throw InternalException("Patas has been deprecated, can no longer be used to compress data");
+	return nullptr;
 }
 
 template <class T>
 void PatasCompress(CompressionState &state_p, Vector &scan_vector, idx_t count) {
-	auto &state = (PatasCompressionState<T> &)state_p;
-	UnifiedVectorFormat vdata;
-	scan_vector.ToUnifiedFormat(count, vdata);
-	state.Append(vdata, count);
+	throw InternalException("Patas has been deprecated, can no longer be used to compress data");
 }
 
 template <class T>
 void PatasFinalizeCompress(CompressionState &state_p) {
-	auto &state = (PatasCompressionState<T> &)state_p;
-	state.Finalize();
+	throw InternalException("Patas has been deprecated, can no longer be used to compress data");
 }
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/data_table.hpp

@@ -8,21 +8,21 @@
 
 #pragma once
 
-#include "duckdb/common/enums/index_type.hpp"
+#include "duckdb/common/enums/index_constraint_type.hpp"
 #include "duckdb/common/enums/scan_options.hpp"
 #include "duckdb/common/mutex.hpp"
 #include "duckdb/common/types/data_chunk.hpp"
-#include "duckdb/storage/index.hpp"
-#include "duckdb/storage/table/table_statistics.hpp"
+#include "duckdb/common/unique_ptr.hpp"
 #include "duckdb/storage/block.hpp"
+#include "duckdb/storage/index.hpp"
 #include "duckdb/storage/statistics/column_statistics.hpp"
 #include "duckdb/storage/table/column_segment.hpp"
+#include "duckdb/storage/table/data_table_info.hpp"
 #include "duckdb/storage/table/persistent_table_data.hpp"
-#include "duckdb/storage/table/row_group_collection.hpp"
 #include "duckdb/storage/table/row_group.hpp"
+#include "duckdb/storage/table/row_group_collection.hpp"
+#include "duckdb/storage/table/table_statistics.hpp"
 #include "duckdb/transaction/local_storage.hpp"
-#include "duckdb/storage/table/data_table_info.hpp"
-#include "duckdb/common/unique_ptr.hpp"
 
 namespace duckdb {
 class BoundForeignKeyConstraint;
@@ -125,12 +125,6 @@ public:
 	void UpdateColumn(TableCatalogEntry &table, ClientContext &context, Vector &row_ids,
 	                  const vector<column_t> &column_path, DataChunk &updates);
 
-	//! Add an index to the DataTable. NOTE: for CREATE (UNIQUE) INDEX statements, we use the PhysicalCreateARTIndex
-	//! operator. This function is only used during the WAL replay, and is a much less performant index creation
-	//! approach.
-	void WALAddIndex(ClientContext &context, unique_ptr<Index> index,
-	                 const vector<unique_ptr<Expression>> &expressions);
-
 	//! Fetches an append lock
 	void AppendLock(TableAppendState &state);
 	//! Begin appending structs to this table, obtaining necessary locks, etc
@@ -153,8 +147,8 @@ public:
 
 	//! Append a chunk with the row ids [row_start, ..., row_start + chunk.size()] to all indexes of the table, returns
 	//! whether or not the append succeeded
-	PreservedError AppendToIndexes(DataChunk &chunk, row_t row_start);
-	static PreservedError AppendToIndexes(TableIndexList &indexes, DataChunk &chunk, row_t row_start);
+	ErrorData AppendToIndexes(DataChunk &chunk, row_t row_start);
+	static ErrorData AppendToIndexes(TableIndexList &indexes, DataChunk &chunk, row_t row_start);
 	//! Remove a chunk with the row ids [row_start, ..., row_start + chunk.size()] from all indexes of the table
 	void RemoveFromIndexes(TableAppendState &state, DataChunk &chunk, row_t row_start);
 	//! Remove the chunk with the specified set of row identifiers from all indexes of the table
@@ -175,7 +169,7 @@ public:
 	void SetDistinct(column_t column_id, unique_ptr<DistinctStatistics> distinct_stats);
 
 	//! Checkpoint the table to the specified table data writer
-	void Checkpoint(TableDataWriter &writer, Serializer &metadata_serializer);
+	void Checkpoint(TableDataWriter &writer, Serializer &serializer);
 	void CommitDropTable();
 	void CommitDropColumn(idx_t index);
 
@@ -184,10 +178,11 @@ public:
 	vector<ColumnSegmentInfo> GetColumnSegmentInfo();
 	static bool IsForeignKeyIndex(const vector<PhysicalIndex> &fk_keys, Index &index, ForeignKeyType fk_type);
 
-	//! Initializes a special scan that is used to create an index on the table, it keeps locks on the table
-	void InitializeWALCreateIndexScan(CreateIndexScanState &state, const vector<column_t> &column_ids);
 	//! Scans the next chunk for the CREATE INDEX operator
 	bool CreateIndexScan(TableScanState &state, DataChunk &result, TableScanType type);
+	//! Returns true, if the index name is unique (i.e., no PK, UNIQUE, FK constraint has the same name)
+	//! FIXME: This is only necessary until we treat all indexes as catalog entries, allowing to alter constraints
+	bool IndexNameIsUnique(const string &name);
 
 	//! Verify constraints with a chunk from the Append containing all columns of the table
 	void VerifyAppendConstraints(TableCatalogEntry &table, ClientContext &context, DataChunk &chunk,

package/src/duckdb/src/include/duckdb/storage/database_size.hpp

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "duckdb/common/common.hpp"
+#include "duckdb/storage/storage_info.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/include/duckdb/storage/index.hpp

@@ -8,15 +8,14 @@
 
 #pragma once
 
-#include "duckdb/common/unordered_set.hpp"
-#include "duckdb/common/enums/index_type.hpp"
+#include "duckdb/common/enums/index_constraint_type.hpp"
+#include "duckdb/common/types/constraint_conflict_info.hpp"
 #include "duckdb/common/types/data_chunk.hpp"
-#include "duckdb/common/sort/sort.hpp"
+#include "duckdb/common/unordered_set.hpp"
+#include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/parser/parsed_expression.hpp"
 #include "duckdb/planner/expression.hpp"
-#include "duckdb/storage/metadata/metadata_writer.hpp"
-#include "duckdb/execution/expression_executor.hpp"
-#include "duckdb/common/types/constraint_conflict_info.hpp"
+#include "duckdb/storage/table_storage_info.hpp"
 
 namespace duckdb {
 
@@ -31,16 +30,22 @@ struct IndexScanState;
 //! The index is an abstract base class that serves as the basis for indexes
 class Index {
 public:
-	Index(AttachedDatabase &db, IndexType type, TableIOManager &table_io_manager, const vector<column_t> &column_ids,
-	      const vector<unique_ptr<Expression>> &unbound_expressions, IndexConstraintType constraint_type);
+	Index(const string &name, const string &index_type, IndexConstraintType index_constraint_type,
+	      const vector<column_t> &column_ids, TableIOManager &table_io_manager,
+	      const vector<unique_ptr<Expression>> &unbound_expressions, AttachedDatabase &db);
 	virtual ~Index() = default;
 
-	//! The type of the index
-	IndexType type;
+	//! The name of the index
+	string name;
+	//! The index type (ART, B+-tree, Skip-List, ...)
+	string index_type;
+	//! The index constraint type
+	IndexConstraintType index_constraint_type;
+	//! The logical column ids of the indexed table
+	vector<column_t> column_ids;
+
 	//! Associated table io manager
 	TableIOManager &table_io_manager;
-	//! Column identifiers to extract key columns from the base table
-	vector<column_t> column_ids;
 	//! Unordered set of column_ids used by the index
 	unordered_set<column_t> column_id_set;
 	//! Unbound expressions used by the index during optimizations
@@ -49,33 +54,22 @@ public:
 	vector<PhysicalType> types;
 	//! The logical types of the expressions
 	vector<LogicalType> logical_types;
-	//! Index constraint type (primary key, foreign key, ...)
-	IndexConstraintType constraint_type;
 
 	//! Attached database instance
 	AttachedDatabase &db;
 
 public:
-	//! Initialize a single predicate scan on the index with the given expression and column IDs
-	virtual unique_ptr<IndexScanState> InitializeScanSinglePredicate(const Transaction &transaction, const Value &value,
-	                                                                 const ExpressionType expression_type) = 0;
-	//! Initialize a two predicate scan on the index with the given expression and column IDs
-	virtual unique_ptr<IndexScanState> InitializeScanTwoPredicates(const Transaction &transaction,
-	                                                               const Value &low_value,
-	                                                               const ExpressionType low_expression_type,
-	                                                               const Value &high_value,
-	                                                               const ExpressionType high_expression_type) = 0;
-	//! Performs a lookup on the index, fetching up to max_count result IDs. Returns true if all row IDs were fetched,
-	//! and false otherwise
-	virtual bool Scan(const Transaction &transaction, const DataTable &table, IndexScanState &state,
-	                  const idx_t max_count, vector<row_t> &result_ids) = 0;
+	//! Returns true if the index is a unknown index, and false otherwise
+	virtual bool IsUnknown() {
+		return false;
+	}
 
 	//! Obtain a lock on the index
-	virtual void InitializeLock(IndexLock &state);
+	void InitializeLock(IndexLock &state);
 	//! Called when data is appended to the index. The lock obtained from InitializeLock must be held
-	virtual PreservedError Append(IndexLock &state, DataChunk &entries, Vector &row_identifiers) = 0;
+	virtual ErrorData Append(IndexLock &state, DataChunk &entries, Vector &row_identifiers) = 0;
 	//! Obtains a lock and calls Append while holding that lock
-	PreservedError Append(DataChunk &entries, Vector &row_identifiers);
+	ErrorData Append(DataChunk &entries, Vector &row_identifiers);
 	//! Verify that data can be appended to the index without a constraint violation
 	virtual void VerifyAppend(DataChunk &chunk) = 0;
 	//! Verify that data can be appended to the index without a constraint violation using the conflict manager
@@ -93,7 +87,7 @@ public:
 	void Delete(DataChunk &entries, Vector &row_identifiers);
 
 	//! Insert a chunk of entries into the index
-	virtual PreservedError Insert(IndexLock &lock, DataChunk &input, Vector &row_identifiers) = 0;
+	virtual ErrorData Insert(IndexLock &lock, DataChunk &input, Vector &row_identifiers) = 0;
 
 	//! Merge another index into this index. The lock obtained from InitializeLock must be held, and the other
 	//! index must also be locked during the merge
@@ -106,6 +100,11 @@ public:
 	//! Obtains a lock and calls Vacuum while holding that lock
 	void Vacuum();
 
+	//! Returns the in-memory usage of the index. The lock obtained from InitializeLock must be held
+	virtual idx_t GetInMemorySize(IndexLock &state) = 0;
+	//! Returns the in-memory usage of the index
+	idx_t GetInMemorySize();
+
 	//! Returns the string representation of an index, or only traverses and verifies the index
 	virtual string VerifyAndToString(IndexLock &state, const bool only_verify) = 0;
 	//! Obtains a lock and calls VerifyAndToString while holding that lock
@@ -116,33 +115,32 @@ public:
 
 	//! Returns unique flag
 	bool IsUnique() {
-		return (constraint_type == IndexConstraintType::UNIQUE || constraint_type == IndexConstraintType::PRIMARY);
+		return (index_constraint_type == IndexConstraintType::UNIQUE ||
+		        index_constraint_type == IndexConstraintType::PRIMARY);
 	}
 	//! Returns primary key flag
 	bool IsPrimary() {
-		return (constraint_type == IndexConstraintType::PRIMARY);
+		return (index_constraint_type == IndexConstraintType::PRIMARY);
 	}
 	//! Returns foreign key flag
 	bool IsForeign() {
-		return (constraint_type == IndexConstraintType::FOREIGN);
+		return (index_constraint_type == IndexConstraintType::FOREIGN);
 	}
 
-	//! Serializes the index to disk
-	virtual BlockPointer Serialize(MetadataWriter &writer);
-	//! Returns the serialized root block pointer
-	BlockPointer GetRootBlockPointer() const {
-		return root_block_pointer;
-	}
+	//! Returns all index storage information for serialization
+	virtual IndexStorageInfo GetStorageInfo(const bool get_buffers);
 
 	//! Execute the index expressions on an input chunk
 	void ExecuteExpressions(DataChunk &input, DataChunk &result);
 	static string AppendRowError(DataChunk &input, idx_t index);
 
+	//! Throw a constraint violation exception
+	virtual string GetConstraintViolationMessage(VerifyExistenceType verify_type, idx_t failed_index,
+	                                             DataChunk &input) = 0;
+
 protected:
 	//! Lock used for any changes to the index
 	mutex lock;
-	//! Pointer to the index on disk
-	BlockPointer root_block_pointer;
 
 private:
 	//! Bound expressions used during expression execution

package/src/duckdb/src/include/duckdb/storage/index_storage_info.hpp

@@ -0,0 +1,77 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/index_storage_info.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/types/value.hpp"
+#include "duckdb/storage/block.hpp"
+#include "duckdb/storage/storage_info.hpp"
+#include "duckdb/common/unordered_set.hpp"
+
+namespace duckdb {
+
+//! Information to serialize a FixedSizeAllocator, which holds the index data
+struct FixedSizeAllocatorInfo {
+	idx_t segment_size;
+	vector<idx_t> buffer_ids;
+	vector<BlockPointer> block_pointers;
+	vector<idx_t> segment_counts;
+	vector<idx_t> allocation_sizes;
+	vector<idx_t> buffers_with_free_space;
+
+	void Serialize(Serializer &serializer) const;
+	static FixedSizeAllocatorInfo Deserialize(Deserializer &deserializer);
+};
+
+//! Information to serialize an index buffer to the WAL
+struct IndexBufferInfo {
+	IndexBufferInfo(data_ptr_t buffer_ptr, const idx_t allocation_size)
+	    : buffer_ptr(buffer_ptr), allocation_size(allocation_size) {
+	}
+
+	data_ptr_t buffer_ptr;
+	idx_t allocation_size;
+};
+
+//! Information to serialize an index
+struct IndexStorageInfo {
+	IndexStorageInfo() {};
+	explicit IndexStorageInfo(string name) : name(std::move(name)) {};
+
+	//! The name of the index
+	string name;
+	//! The root of the index
+	idx_t root;
+	//! Information to serialize the index memory held by the fixed-size allocators
+	vector<FixedSizeAllocatorInfo> allocator_infos;
+
+	//! Contains all buffer pointers and their allocation size for serializing to the WAL
+	//! First dimension: all fixed-size allocators, second dimension: the buffers of each allocator
+	vector<vector<IndexBufferInfo>> buffers;
+
+	//! The root block pointer of the index, which is necessary to support older storage files
+	BlockPointer root_block_ptr;
+
+	//! Returns true, if the struct contains index information
+	bool IsValid() const {
+		return root_block_ptr.IsValid() || !allocator_infos.empty();
+	}
+
+	void Serialize(Serializer &serializer) const;
+	static IndexStorageInfo Deserialize(Deserializer &deserializer);
+};
+
+//! Additional index information for tables
+struct IndexInfo {
+	bool is_unique;
+	bool is_primary;
+	bool is_foreign;
+	unordered_set<column_t> column_set;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/metadata/metadata_manager.hpp

@@ -42,10 +42,10 @@ struct MetadataHandle {
 
 class MetadataManager {
 public:
-	//! The size of metadata blocks
-	static constexpr const idx_t METADATA_BLOCK_SIZE = 4088;
 	//! The amount of metadata blocks per storage block
 	static constexpr const idx_t METADATA_BLOCK_COUNT = 64;
+	//! The size of metadata blocks
+	static constexpr const idx_t METADATA_BLOCK_SIZE = AlignValueFloor(Storage::BLOCK_SIZE / METADATA_BLOCK_COUNT);
 
 public:
 	MetadataManager(BlockManager &block_manager, BufferManager &buffer_manager);
@@ -88,4 +88,8 @@ protected:
 	void ConvertToTransient(MetadataBlock &block);
 };
 
+//! Detect mismatching constant values
+static_assert(MetadataManager::METADATA_BLOCK_SIZE * MetadataManager::METADATA_BLOCK_COUNT <= Storage::BLOCK_SIZE,
+              "metadata block count exceeds total block alloc size");
+
 } // namespace duckdb