duckdb 0.9.3-dev2.0 → 0.10.0

package/src/duckdb/src/storage/wal_replay.cpp

@@ -1,10 +1,14 @@
+#include "duckdb/catalog/catalog_entry/duck_index_entry.hpp"
+#include "duckdb/catalog/catalog_entry/duck_table_entry.hpp"
 #include "duckdb/catalog/catalog_entry/scalar_macro_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_entry/type_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_entry/view_catalog_entry.hpp"
 #include "duckdb/common/printer.hpp"
+#include "duckdb/common/serializer/binary_deserializer.hpp"
 #include "duckdb/common/serializer/buffered_file_reader.hpp"
 #include "duckdb/common/string_util.hpp"
+#include "duckdb/main/attached_database.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/main/connection.hpp"
 #include "duckdb/main/database.hpp"
@@ -13,17 +17,147 @@
 #include "duckdb/parser/parsed_data/create_view_info.hpp"
 #include "duckdb/parser/parsed_data/drop_info.hpp"
 #include "duckdb/planner/binder.hpp"
+#include "duckdb/planner/expression_binder/index_binder.hpp"
 #include "duckdb/planner/parsed_data/bound_create_table_info.hpp"
-#include "duckdb/storage/data_table.hpp"
-#include "duckdb/storage/write_ahead_log.hpp"
 #include "duckdb/storage/storage_manager.hpp"
-#include "duckdb/main/attached_database.hpp"
+#include "duckdb/storage/write_ahead_log.hpp"
+#include "duckdb/common/serializer/memory_stream.hpp"
+#include "duckdb/common/checksum.hpp"
+#include "duckdb/execution/index/index_type_set.hpp"
 #include "duckdb/execution/index/art/art.hpp"
-#include "duckdb/catalog/catalog_entry/duck_index_entry.hpp"
-#include "duckdb/common/serializer/binary_deserializer.hpp"
 
 namespace duckdb {
 
+class ReplayState {
+public:
+	ReplayState(AttachedDatabase &db, ClientContext &context) : db(db), context(context), catalog(db.GetCatalog()) {
+	}
+
+	AttachedDatabase &db;
+	ClientContext &context;
+	Catalog &catalog;
+	optional_ptr<TableCatalogEntry> current_table;
+	MetaBlockPointer checkpoint_id;
+	idx_t wal_version = 1;
+};
+
+class WriteAheadLogDeserializer {
+public:
+	WriteAheadLogDeserializer(ReplayState &state_p, BufferedFileReader &stream_p, bool deserialize_only = false)
+	    : state(state_p), db(state.db), context(state.context), catalog(state.catalog), data(nullptr),
+	      stream(nullptr, 0), deserializer(stream_p), deserialize_only(deserialize_only) {
+	}
+	WriteAheadLogDeserializer(ReplayState &state_p, unique_ptr<data_t[]> data_p, idx_t size,
+	                          bool deserialize_only = false)
+	    : state(state_p), db(state.db), context(state.context), catalog(state.catalog), data(std::move(data_p)),
+	      stream(data.get(), size), deserializer(stream), deserialize_only(deserialize_only) {
+	}
+
+	static WriteAheadLogDeserializer Open(ReplayState &state_p, BufferedFileReader &stream,
+	                                      bool deserialize_only = false) {
+		if (state_p.wal_version == 1) {
+			// old WAL versions do not have checksums
+			return WriteAheadLogDeserializer(state_p, stream, deserialize_only);
+		}
+		if (state_p.wal_version != 2) {
+			throw IOException("Failed to read WAL of version %llu - can only read version 1 and 2",
+			                  state_p.wal_version);
+		}
+		// read the checksum and size
+		auto size = stream.Read<uint64_t>();
+		auto stored_checksum = stream.Read<uint64_t>();
+		auto offset = stream.CurrentOffset();
+		auto file_size = stream.FileSize();
+
+		if (offset + size > file_size) {
+			throw SerializationException(
+			    "Corrupt WAL file: entry size exceeded remaining data in file at byte position %llu "
+			    "(found entry with size %llu bytes, file size %llu bytes)",
+			    offset, size, file_size);
+		}
+
+		// allocate a buffer and read data into the buffer
+		auto buffer = unique_ptr<data_t[]>(new data_t[size]);
+		stream.ReadData(buffer.get(), size);
+
+		// compute and verify the checksum
+		auto computed_checksum = Checksum(buffer.get(), size);
+		if (stored_checksum != computed_checksum) {
+			throw SerializationException(
+			    "Corrupt WAL file: entry at byte position %llu computed checksum %llu does not match "
+			    "stored checksum %llu",
+			    offset, computed_checksum, stored_checksum);
+		}
+		return WriteAheadLogDeserializer(state_p, std::move(buffer), size, deserialize_only);
+	}
+
+	bool ReplayEntry() {
+		deserializer.Begin();
+		auto wal_type = deserializer.ReadProperty<WALType>(100, "wal_type");
+		if (wal_type == WALType::WAL_FLUSH) {
+			deserializer.End();
+			return true;
+		}
+		ReplayEntry(wal_type);
+		deserializer.End();
+		return false;
+	}
+
+	bool DeserializeOnly() {
+		return deserialize_only;
+	}
+
+protected:
+	void ReplayEntry(WALType wal_type);
+
+	void ReplayVersion();
+
+	void ReplayCreateTable();
+	void ReplayDropTable();
+	void ReplayAlter();
+
+	void ReplayCreateView();
+	void ReplayDropView();
+
+	void ReplayCreateSchema();
+	void ReplayDropSchema();
+
+	void ReplayCreateType();
+	void ReplayDropType();
+
+	void ReplayCreateSequence();
+	void ReplayDropSequence();
+	void ReplaySequenceValue();
+
+	void ReplayCreateMacro();
+	void ReplayDropMacro();
+
+	void ReplayCreateTableMacro();
+	void ReplayDropTableMacro();
+
+	void ReplayCreateIndex();
+	void ReplayDropIndex();
+
+	void ReplayUseTable();
+	void ReplayInsert();
+	void ReplayDelete();
+	void ReplayUpdate();
+	void ReplayCheckpoint();
+
+private:
+	ReplayState &state;
+	AttachedDatabase &db;
+	ClientContext &context;
+	Catalog &catalog;
+	unique_ptr<data_t[]> data;
+	MemoryStream stream;
+	BinaryDeserializer deserializer;
+	bool deserialize_only;
+};
+
+//===--------------------------------------------------------------------===//
+// Replay
+//===--------------------------------------------------------------------===//
 bool WriteAheadLog::Replay(AttachedDatabase &database, string &path) {
 	Connection con(database.GetDatabase());
 	auto initial_source = make_uniq<BufferedFileReader>(FileSystem::Get(database), path.c_str());
@@ -37,32 +171,27 @@ bool WriteAheadLog::Replay(AttachedDatabase &database, string &path) {
 	// first deserialize the WAL to look for a checkpoint flag
 	// if there is a checkpoint flag, we might have already flushed the contents of the WAL to disk
 	ReplayState checkpoint_state(database, *con.context);
-	checkpoint_state.deserialize_only = true;
 	try {
 		while (true) {
-			// read the current entry
-			BinaryDeserializer deserializer(*initial_source);
-			deserializer.Begin();
-			auto entry_type = deserializer.ReadProperty<WALType>(100, "wal_type");
-			if (entry_type == WALType::WAL_FLUSH) {
-				deserializer.End();
+			// read the current entry (deserialize only)
+			auto deserializer = WriteAheadLogDeserializer::Open(checkpoint_state, *initial_source, true);
+			if (deserializer.ReplayEntry()) {
 				// check if the file is exhausted
 				if (initial_source->Finished()) {
 					// we finished reading the file: break
 					break;
 				}
-			} else {
-				// replay the entry
-				checkpoint_state.ReplayEntry(entry_type, deserializer);
-				deserializer.End();
 			}
 		}
-	} catch (SerializationException &ex) { // LCOV_EXCL_START
-		                                   // serialization exception - torn WAL
-		                                   // continue reading
-	} catch (std::exception &ex) {
-		Printer::PrintF("Exception in WAL playback during initial read: %s\n", ex.what());
-		return false;
+	} catch (std::exception &ex) { // LCOV_EXCL_START
+		ErrorData error(ex);
+		if (error.Type() == ExceptionType::SERIALIZATION) {
+			// serialization exception - torn WAL
+			// continue reading
+		} else {
+			Printer::PrintF("Exception in WAL playback during initial read: %s\n", error.RawMessage());
+			return false;
+		}
 	} catch (...) {
 		Printer::Print("Unknown Exception in WAL playback during initial read");
 		return false;
@@ -89,11 +218,8 @@ bool WriteAheadLog::Replay(AttachedDatabase &database, string &path) {
 	try {
 		while (true) {
 			// read the current entry
-			BinaryDeserializer deserializer(reader);
-			deserializer.Begin();
-			auto entry_type = deserializer.ReadProperty<WALType>(100, "wal_type");
-			if (entry_type == WALType::WAL_FLUSH) {
-				deserializer.End();
+			auto deserializer = WriteAheadLogDeserializer::Open(state, reader);
+			if (deserializer.ReplayEntry()) {
 				con.Commit();
 				// check if the file is exhausted
 				if (reader.Finished()) {
@@ -101,19 +227,15 @@ bool WriteAheadLog::Replay(AttachedDatabase &database, string &path) {
 					break;
 				}
 				con.BeginTransaction();
-			} else {
-				// replay the entry
-				state.ReplayEntry(entry_type, deserializer);
-				deserializer.End();
 			}
 		}
-	} catch (SerializationException &ex) { // LCOV_EXCL_START
-		// serialization error during WAL replay: rollback
-		con.Rollback();
-	} catch (std::exception &ex) {
-		// FIXME: this should report a proper warning in the connection
-		Printer::PrintF("Exception in WAL playback: %s\n", ex.what());
-		// exception thrown in WAL replay: rollback
+	} catch (std::exception &ex) { // LCOV_EXCL_START
+		ErrorData error(ex);
+		if (error.Type() != ExceptionType::SERIALIZATION) {
+			// FIXME: this should report a proper warning in the connection
+			Printer::PrintF("Exception in WAL playback: %s\n", error.RawMessage());
+			// exception thrown in WAL replay: rollback
+		}
 		con.Rollback();
 	} catch (...) {
 		Printer::Print("Unknown Exception in WAL playback: %s\n");
@@ -126,88 +248,98 @@ bool WriteAheadLog::Replay(AttachedDatabase &database, string &path) {
 //===--------------------------------------------------------------------===//
 // Replay Entries
 //===--------------------------------------------------------------------===//
-void ReplayState::ReplayEntry(WALType entry_type, BinaryDeserializer &deserializer) {
+void WriteAheadLogDeserializer::ReplayEntry(WALType entry_type) {
 	switch (entry_type) {
+	case WALType::WAL_VERSION:
+		ReplayVersion();
+		break;
 	case WALType::CREATE_TABLE:
-		ReplayCreateTable(deserializer);
+		ReplayCreateTable();
 		break;
 	case WALType::DROP_TABLE:
-		ReplayDropTable(deserializer);
+		ReplayDropTable();
 		break;
 	case WALType::ALTER_INFO:
-		ReplayAlter(deserializer);
+		ReplayAlter();
 		break;
 	case WALType::CREATE_VIEW:
-		ReplayCreateView(deserializer);
+		ReplayCreateView();
 		break;
 	case WALType::DROP_VIEW:
-		ReplayDropView(deserializer);
+		ReplayDropView();
 		break;
 	case WALType::CREATE_SCHEMA:
-		ReplayCreateSchema(deserializer);
+		ReplayCreateSchema();
 		break;
 	case WALType::DROP_SCHEMA:
-		ReplayDropSchema(deserializer);
+		ReplayDropSchema();
 		break;
 	case WALType::CREATE_SEQUENCE:
-		ReplayCreateSequence(deserializer);
+		ReplayCreateSequence();
 		break;
 	case WALType::DROP_SEQUENCE:
-		ReplayDropSequence(deserializer);
+		ReplayDropSequence();
 		break;
 	case WALType::SEQUENCE_VALUE:
-		ReplaySequenceValue(deserializer);
+		ReplaySequenceValue();
 		break;
 	case WALType::CREATE_MACRO:
-		ReplayCreateMacro(deserializer);
+		ReplayCreateMacro();
 		break;
 	case WALType::DROP_MACRO:
-		ReplayDropMacro(deserializer);
+		ReplayDropMacro();
 		break;
 	case WALType::CREATE_TABLE_MACRO:
-		ReplayCreateTableMacro(deserializer);
+		ReplayCreateTableMacro();
 		break;
 	case WALType::DROP_TABLE_MACRO:
-		ReplayDropTableMacro(deserializer);
+		ReplayDropTableMacro();
 		break;
 	case WALType::CREATE_INDEX:
-		ReplayCreateIndex(deserializer);
+		ReplayCreateIndex();
 		break;
 	case WALType::DROP_INDEX:
-		ReplayDropIndex(deserializer);
+		ReplayDropIndex();
 		break;
 	case WALType::USE_TABLE:
-		ReplayUseTable(deserializer);
+		ReplayUseTable();
 		break;
 	case WALType::INSERT_TUPLE:
-		ReplayInsert(deserializer);
+		ReplayInsert();
 		break;
 	case WALType::DELETE_TUPLE:
-		ReplayDelete(deserializer);
+		ReplayDelete();
 		break;
 	case WALType::UPDATE_TUPLE:
-		ReplayUpdate(deserializer);
+		ReplayUpdate();
 		break;
 	case WALType::CHECKPOINT:
-		ReplayCheckpoint(deserializer);
+		ReplayCheckpoint();
 		break;
 	case WALType::CREATE_TYPE:
-		ReplayCreateType(deserializer);
+		ReplayCreateType();
 		break;
 	case WALType::DROP_TYPE:
-		ReplayDropType(deserializer);
+		ReplayDropType();
 		break;
 	default:
 		throw InternalException("Invalid WAL entry type!");
 	}
 }
 
+//===--------------------------------------------------------------------===//
+// Replay Version
+//===--------------------------------------------------------------------===//
+void WriteAheadLogDeserializer::ReplayVersion() {
+	state.wal_version = deserializer.ReadProperty<idx_t>(101, "version");
+}
+
 //===--------------------------------------------------------------------===//
 // Replay Table
 //===--------------------------------------------------------------------===//
-void ReplayState::ReplayCreateTable(BinaryDeserializer &deserializer) {
+void WriteAheadLogDeserializer::ReplayCreateTable() {
 	auto info = deserializer.ReadProperty<unique_ptr<CreateInfo>>(101, "table");
-	if (deserialize_only) {
+	if (DeserializeOnly()) {
 		return;
 	}
 	// bind the constraints to the table again
@@ -218,25 +350,23 @@ void ReplayState::ReplayCreateTable(BinaryDeserializer &deserializer) {
 	catalog.CreateTable(context, *bound_info);
 }
 
-void ReplayState::ReplayDropTable(BinaryDeserializer &deserializer) {
-
+void WriteAheadLogDeserializer::ReplayDropTable() {
 	DropInfo info;
 
 	info.type = CatalogType::TABLE_ENTRY;
 	info.schema = deserializer.ReadProperty<string>(101, "schema");
 	info.name = deserializer.ReadProperty<string>(102, "name");
-	if (deserialize_only) {
+	if (DeserializeOnly()) {
 		return;
 	}
 
 	catalog.DropEntry(context, info);
 }
 
-void ReplayState::ReplayAlter(BinaryDeserializer &deserializer) {
-
+void WriteAheadLogDeserializer::ReplayAlter() {
 	auto info = deserializer.ReadProperty<unique_ptr<ParseInfo>>(101, "info");
 	auto &alter_info = info->Cast<AlterInfo>();
-	if (deserialize_only) {
+	if (DeserializeOnly()) {
 		return;
 	}
 	catalog.Alter(context, alter_info);
@@ -245,20 +375,20 @@ void ReplayState::ReplayAlter(BinaryDeserializer &deserializer) {
 //===--------------------------------------------------------------------===//
 // Replay View
 //===--------------------------------------------------------------------===//
-void ReplayState::ReplayCreateView(BinaryDeserializer &deserializer) {
+void WriteAheadLogDeserializer::ReplayCreateView() {
 	auto entry = deserializer.ReadProperty<unique_ptr<CreateInfo>>(101, "view");
-	if (deserialize_only) {
+	if (DeserializeOnly()) {
 		return;
 	}
 	catalog.CreateView(context, entry->Cast<CreateViewInfo>());
 }
 
-void ReplayState::ReplayDropView(BinaryDeserializer &deserializer) {
+void WriteAheadLogDeserializer::ReplayDropView() {
 	DropInfo info;
 	info.type = CatalogType::VIEW_ENTRY;
 	info.schema = deserializer.ReadProperty<string>(101, "schema");
 	info.name = deserializer.ReadProperty<string>(102, "name");
-	if (deserialize_only) {
+	if (DeserializeOnly()) {
 		return;
 	}
 	catalog.DropEntry(context, info);
@@ -267,22 +397,22 @@ void ReplayState::ReplayDropView(BinaryDeserializer &deserializer) {
 //===--------------------------------------------------------------------===//
 // Replay Schema
 //===--------------------------------------------------------------------===//
-void ReplayState::ReplayCreateSchema(BinaryDeserializer &deserializer) {
+void WriteAheadLogDeserializer::ReplayCreateSchema() {
 	CreateSchemaInfo info;
 	info.schema = deserializer.ReadProperty<string>(101, "schema");
-	if (deserialize_only) {
+	if (DeserializeOnly()) {
 		return;
 	}
 
 	catalog.CreateSchema(context, info);
 }
 
-void ReplayState::ReplayDropSchema(BinaryDeserializer &deserializer) {
+void WriteAheadLogDeserializer::ReplayDropSchema() {
 	DropInfo info;
 
 	info.type = CatalogType::SCHEMA_ENTRY;
 	info.name = deserializer.ReadProperty<string>(101, "schema");
-	if (deserialize_only) {
+	if (DeserializeOnly()) {
 		return;
 	}
 
@@ -292,19 +422,19 @@ void ReplayState::ReplayDropSchema(BinaryDeserializer &deserializer) {
 //===--------------------------------------------------------------------===//
 // Replay Custom Type
 //===--------------------------------------------------------------------===//
-void ReplayState::ReplayCreateType(BinaryDeserializer &deserializer) {
+void WriteAheadLogDeserializer::ReplayCreateType() {
 	auto info = deserializer.ReadProperty<unique_ptr<CreateInfo>>(101, "type");
 	info->on_conflict = OnCreateConflict::IGNORE_ON_CONFLICT;
 	catalog.CreateType(context, info->Cast<CreateTypeInfo>());
 }
 
-void ReplayState::ReplayDropType(BinaryDeserializer &deserializer) {
+void WriteAheadLogDeserializer::ReplayDropType() {
 	DropInfo info;
 
 	info.type = CatalogType::TYPE_ENTRY;
 	info.schema = deserializer.ReadProperty<string>(101, "schema");
 	info.name = deserializer.ReadProperty<string>(102, "name");
-	if (deserialize_only) {
+	if (DeserializeOnly()) {
 		return;
 	}
 
@@ -314,62 +444,59 @@ void ReplayState::ReplayDropType(BinaryDeserializer &deserializer) {
 //===--------------------------------------------------------------------===//
 // Replay Sequence
 //===--------------------------------------------------------------------===//
-void ReplayState::ReplayCreateSequence(BinaryDeserializer &deserializer) {
+void WriteAheadLogDeserializer::ReplayCreateSequence() {
 	auto entry = deserializer.ReadProperty<unique_ptr<CreateInfo>>(101, "sequence");
-	if (deserialize_only) {
+	if (DeserializeOnly()) {
 		return;
 	}
 
 	catalog.CreateSequence(context, entry->Cast<CreateSequenceInfo>());
 }
 
-void ReplayState::ReplayDropSequence(BinaryDeserializer &deserializer) {
+void WriteAheadLogDeserializer::ReplayDropSequence() {
 	DropInfo info;
 	info.type = CatalogType::SEQUENCE_ENTRY;
 	info.schema = deserializer.ReadProperty<string>(101, "schema");
 	info.name = deserializer.ReadProperty<string>(102, "name");
-	if (deserialize_only) {
+	if (DeserializeOnly()) {
 		return;
 	}
 
 	catalog.DropEntry(context, info);
 }
 
-void ReplayState::ReplaySequenceValue(BinaryDeserializer &deserializer) {
+void WriteAheadLogDeserializer::ReplaySequenceValue() {
 	auto schema = deserializer.ReadProperty<string>(101, "schema");
 	auto name = deserializer.ReadProperty<string>(102, "name");
 	auto usage_count = deserializer.ReadProperty<uint64_t>(103, "usage_count");
 	auto counter = deserializer.ReadProperty<int64_t>(104, "counter");
-	if (deserialize_only) {
+	if (DeserializeOnly()) {
 		return;
 	}
 
 	// fetch the sequence from the catalog
 	auto &seq = catalog.GetEntry<SequenceCatalogEntry>(context, schema, name);
-	if (usage_count > seq.usage_count) {
-		seq.usage_count = usage_count;
-		seq.counter = counter;
-	}
+	seq.ReplayValue(usage_count, counter);
 }
 
 //===--------------------------------------------------------------------===//
 // Replay Macro
 //===--------------------------------------------------------------------===//
-void ReplayState::ReplayCreateMacro(BinaryDeserializer &deserializer) {
+void WriteAheadLogDeserializer::ReplayCreateMacro() {
 	auto entry = deserializer.ReadProperty<unique_ptr<CreateInfo>>(101, "macro");
-	if (deserialize_only) {
+	if (DeserializeOnly()) {
 		return;
 	}
 
 	catalog.CreateFunction(context, entry->Cast<CreateMacroInfo>());
 }
 
-void ReplayState::ReplayDropMacro(BinaryDeserializer &deserializer) {
+void WriteAheadLogDeserializer::ReplayDropMacro() {
 	DropInfo info;
 	info.type = CatalogType::MACRO_ENTRY;
 	info.schema = deserializer.ReadProperty<string>(101, "schema");
 	info.name = deserializer.ReadProperty<string>(102, "name");
-	if (deserialize_only) {
+	if (DeserializeOnly()) {
 		return;
 	}
 
@@ -379,20 +506,20 @@ void ReplayState::ReplayDropMacro(BinaryDeserializer &deserializer) {
 //===--------------------------------------------------------------------===//
 // Replay Table Macro
 //===--------------------------------------------------------------------===//
-void ReplayState::ReplayCreateTableMacro(BinaryDeserializer &deserializer) {
+void WriteAheadLogDeserializer::ReplayCreateTableMacro() {
 	auto entry = deserializer.ReadProperty<unique_ptr<CreateInfo>>(101, "table_macro");
-	if (deserialize_only) {
+	if (DeserializeOnly()) {
 		return;
 	}
 	catalog.CreateFunction(context, entry->Cast<CreateMacroInfo>());
 }
 
-void ReplayState::ReplayDropTableMacro(BinaryDeserializer &deserializer) {
+void WriteAheadLogDeserializer::ReplayDropTableMacro() {
 	DropInfo info;
 	info.type = CatalogType::TABLE_MACRO_ENTRY;
 	info.schema = deserializer.ReadProperty<string>(101, "schema");
 	info.name = deserializer.ReadProperty<string>(102, "name");
-	if (deserialize_only) {
+	if (DeserializeOnly()) {
 		return;
 	}
 
@@ -402,56 +529,106 @@ void ReplayState::ReplayDropTableMacro(BinaryDeserializer &deserializer) {
 //===--------------------------------------------------------------------===//
 // Replay Index
 //===--------------------------------------------------------------------===//
-void ReplayState::ReplayCreateIndex(BinaryDeserializer &deserializer) {
-	auto info = deserializer.ReadProperty<unique_ptr<CreateInfo>>(101, "index");
-	if (deserialize_only) {
+void WriteAheadLogDeserializer::ReplayCreateIndex() {
+	auto create_info = deserializer.ReadProperty<unique_ptr<CreateInfo>>(101, "index_catalog_entry");
+	auto index_info = deserializer.ReadProperty<IndexStorageInfo>(102, "index_storage_info");
+	D_ASSERT(index_info.IsValid() && !index_info.name.empty());
+
+	auto &storage_manager = db.GetStorageManager();
+	auto &single_file_sm = storage_manager.Cast<SingleFileStorageManager>();
+	auto &block_manager = single_file_sm.block_manager;
+	auto &buffer_manager = block_manager->buffer_manager;
+
+	deserializer.ReadList(103, "index_storage", [&](Deserializer::List &list, idx_t i) {
+		auto &data_info = index_info.allocator_infos[i];
+
+		// read the data into buffer handles and convert them to blocks on disk
+		// then, update the block pointer
+		for (idx_t j = 0; j < data_info.allocation_sizes.size(); j++) {
+
+			// read the data into a buffer handle
+			shared_ptr<BlockHandle> block_handle;
+			buffer_manager.Allocate(MemoryTag::ART_INDEX, Storage::BLOCK_SIZE, false, &block_handle);
+			auto buffer_handle = buffer_manager.Pin(block_handle);
+			auto data_ptr = buffer_handle.Ptr();
+
+			list.ReadElement<bool>(data_ptr, data_info.allocation_sizes[j]);
+
+			// now convert the buffer handle to a persistent block and remember the block id
+			auto block_id = block_manager->GetFreeBlockId();
+			block_manager->ConvertToPersistent(block_id, std::move(block_handle));
+			data_info.block_pointers[j].block_id = block_id;
+		}
+	});
+
+	if (DeserializeOnly()) {
 		return;
 	}
-	auto &index_info = info->Cast<CreateIndexInfo>();
+	auto &info = create_info->Cast<CreateIndexInfo>();
 
-	// get the physical table to which we'll add the index
-	auto &table = catalog.GetEntry<TableCatalogEntry>(context, info->schema, index_info.table);
-	auto &data_table = table.GetStorage();
+	// Ensure the index type exists
+	if (info.index_type.empty()) {
+		info.index_type = ART::TYPE_NAME;
+	}
 
-	// bind the parsed expressions
-	if (index_info.expressions.empty()) {
-		for (auto &parsed_expr : index_info.parsed_expressions) {
-			index_info.expressions.push_back(parsed_expr->Copy());
-		}
+	auto index_type = context.db->config.GetIndexTypes().FindByName(info.index_type);
+	if (!index_type) {
+		throw InternalException("Index type \"%s\" not recognized", info.index_type);
 	}
-	auto binder = Binder::CreateBinder(context);
-	auto expressions = binder->BindCreateIndexExpressions(table, index_info);
-
-	// create the empty index
-	unique_ptr<Index> index;
-	switch (index_info.index_type) {
-	case IndexType::ART: {
-		index = make_uniq<ART>(index_info.column_ids, TableIOManager::Get(data_table), expressions,
-		                       index_info.constraint_type, data_table.db);
-		break;
+
+	// create the index in the catalog
+	auto &table = catalog.GetEntry<TableCatalogEntry>(context, create_info->schema, info.table).Cast<DuckTableEntry>();
+	auto &index = catalog.CreateIndex(context, info)->Cast<DuckIndexEntry>();
+	index.info = make_shared<IndexDataTableInfo>(table.GetStorage().info, index.name);
+
+	// insert the parsed expressions into the index so that we can (de)serialize them during consecutive checkpoints
+	for (auto &parsed_expr : info.parsed_expressions) {
+		index.parsed_expressions.push_back(parsed_expr->Copy());
 	}
-	default:
-		throw InternalException("Unimplemented index type");
+
+	// obtain the parsed expressions of the ART from the index metadata
+	vector<unique_ptr<ParsedExpression>> parsed_expressions;
+	for (auto &parsed_expr : info.parsed_expressions) {
+		parsed_expressions.push_back(parsed_expr->Copy());
 	}
+	D_ASSERT(!parsed_expressions.empty());
 
-	// add the index to the catalog
-	auto &index_entry = catalog.CreateIndex(context, index_info)->Cast<DuckIndexEntry>();
-	index_entry.index = index.get();
-	index_entry.info = data_table.info;
-	for (auto &parsed_expr : index_info.parsed_expressions) {
-		index_entry.parsed_expressions.push_back(parsed_expr->Copy());
+	// add the table to the bind context to bind the parsed expressions
+	auto binder = Binder::CreateBinder(context);
+	vector<LogicalType> column_types;
+	vector<string> column_names;
+	for (auto &col : table.GetColumns().Logical()) {
+		column_types.push_back(col.Type());
+		column_names.push_back(col.Name());
 	}
 
-	// physically add the index to the data table storage
-	data_table.WALAddIndex(context, std::move(index), expressions);
+	// create a binder to bind the parsed expressions
+	vector<column_t> column_ids;
+	binder->bind_context.AddBaseTable(0, info.table, column_names, column_types, column_ids, &table);
+	IndexBinder idx_binder(*binder, context);
+
+	// bind the parsed expressions to create unbound expressions
+	vector<unique_ptr<Expression>> unbound_expressions;
+	unbound_expressions.reserve(parsed_expressions.size());
+	for (auto &expr : parsed_expressions) {
+		unbound_expressions.push_back(idx_binder.Bind(expr));
+	}
+
+	auto &data_table = table.GetStorage();
+
+	CreateIndexInput input(TableIOManager::Get(data_table), data_table.db, info.constraint_type, info.index_name,
+	                       info.column_ids, unbound_expressions, index_info, info.options);
+
+	auto index_instance = index_type->create_instance(input);
+	data_table.info->indexes.AddIndex(std::move(index_instance));
 }
 
-void ReplayState::ReplayDropIndex(BinaryDeserializer &deserializer) {
+void WriteAheadLogDeserializer::ReplayDropIndex() {
 	DropInfo info;
 	info.type = CatalogType::INDEX_ENTRY;
 	info.schema = deserializer.ReadProperty<string>(101, "schema");
 	info.name = deserializer.ReadProperty<string>(102, "name");
-	if (deserialize_only) {
+	if (DeserializeOnly()) {
 		return;
 	}
 
@@ -461,36 +638,36 @@ void ReplayState::ReplayDropIndex(BinaryDeserializer &deserializer) {
 //===--------------------------------------------------------------------===//
 // Replay Data
 //===--------------------------------------------------------------------===//
-void ReplayState::ReplayUseTable(BinaryDeserializer &deserializer) {
+void WriteAheadLogDeserializer::ReplayUseTable() {
 	auto schema_name = deserializer.ReadProperty<string>(101, "schema");
 	auto table_name = deserializer.ReadProperty<string>(102, "table");
-	if (deserialize_only) {
+	if (DeserializeOnly()) {
 		return;
 	}
-	current_table = &catalog.GetEntry<TableCatalogEntry>(context, schema_name, table_name);
+	state.current_table = &catalog.GetEntry<TableCatalogEntry>(context, schema_name, table_name);
 }
 
-void ReplayState::ReplayInsert(BinaryDeserializer &deserializer) {
+void WriteAheadLogDeserializer::ReplayInsert() {
 	DataChunk chunk;
 	deserializer.ReadObject(101, "chunk", [&](Deserializer &object) { chunk.Deserialize(object); });
-	if (deserialize_only) {
+	if (DeserializeOnly()) {
 		return;
 	}
-	if (!current_table) {
-		throw Exception("Corrupt WAL: insert without table");
+	if (!state.current_table) {
+		throw InternalException("Corrupt WAL: insert without table");
 	}
 
 	// append to the current table
-	current_table->GetStorage().LocalAppend(*current_table, context, chunk);
+	state.current_table->GetStorage().LocalAppend(*state.current_table, context, chunk);
 }
 
-void ReplayState::ReplayDelete(BinaryDeserializer &deserializer) {
+void WriteAheadLogDeserializer::ReplayDelete() {
 	DataChunk chunk;
 	deserializer.ReadObject(101, "chunk", [&](Deserializer &object) { chunk.Deserialize(object); });
-	if (deserialize_only) {
+	if (DeserializeOnly()) {
 		return;
 	}
-	if (!current_table) {
+	if (!state.current_table) {
 		throw InternalException("Corrupt WAL: delete without table");
 	}
 
@@ -502,24 +679,24 @@ void ReplayState::ReplayDelete(BinaryDeserializer &deserializer) {
 	// delete the tuples from the current table
 	for (idx_t i = 0; i < chunk.size(); i++) {
 		row_ids[0] = source_ids[i];
-		current_table->GetStorage().Delete(*current_table, context, row_identifiers, 1);
+		state.current_table->GetStorage().Delete(*state.current_table, context, row_identifiers, 1);
 	}
 }
 
-void ReplayState::ReplayUpdate(BinaryDeserializer &deserializer) {
+void WriteAheadLogDeserializer::ReplayUpdate() {
 	auto column_path = deserializer.ReadProperty<vector<column_t>>(101, "column_indexes");
 
 	DataChunk chunk;
 	deserializer.ReadObject(102, "chunk", [&](Deserializer &object) { chunk.Deserialize(object); });
 
-	if (deserialize_only) {
+	if (DeserializeOnly()) {
 		return;
 	}
-	if (!current_table) {
+	if (!state.current_table) {
 		throw InternalException("Corrupt WAL: update without table");
 	}
 
-	if (column_path[0] >= current_table->GetColumns().PhysicalColumnCount()) {
+	if (column_path[0] >= state.current_table->GetColumns().PhysicalColumnCount()) {
 		throw InternalException("Corrupt WAL: column index for update out of bounds");
 	}
 
@@ -528,11 +705,11 @@ void ReplayState::ReplayUpdate(BinaryDeserializer &deserializer) {
 	chunk.data.pop_back();
 
 	// now perform the update
-	current_table->GetStorage().UpdateColumn(*current_table, context, row_ids, column_path, chunk);
+	state.current_table->GetStorage().UpdateColumn(*state.current_table, context, row_ids, column_path, chunk);
 }
 
-void ReplayState::ReplayCheckpoint(BinaryDeserializer &deserializer) {
-	checkpoint_id = deserializer.ReadProperty<MetaBlockPointer>(101, "meta_block");
+void WriteAheadLogDeserializer::ReplayCheckpoint() {
+	state.checkpoint_id = deserializer.ReadProperty<MetaBlockPointer>(101, "meta_block");
 }
 
 } // namespace duckdb