duckdb 0.10.3-dev3.0 → 0.10.3-dev8.0

package/src/duckdb/src/storage/table_index_list.cpp

@@ -1,12 +1,14 @@
 #include "duckdb/storage/table/table_index_list.hpp"
 
-#include "duckdb/storage/data_table.hpp"
 #include "duckdb/common/types/conflict_manager.hpp"
-#include "duckdb/execution/index/unknown_index.hpp"
 #include "duckdb/execution/index/index_type_set.hpp"
-#include "duckdb/storage/table/data_table_info.hpp"
-#include "duckdb/main/database.hpp"
+#include "duckdb/execution/index/unbound_index.hpp"
 #include "duckdb/main/config.hpp"
+#include "duckdb/main/database.hpp"
+#include "duckdb/storage/data_table.hpp"
+#include "duckdb/storage/table/data_table_info.hpp"
+#include "duckdb/catalog/catalog_entry/duck_table_entry.hpp"
+#include "duckdb/planner/expression_binder/index_binder.hpp"
 
 namespace duckdb {
 void TableIndexList::AddIndex(unique_ptr<Index> index) {
@@ -20,8 +22,9 @@ void TableIndexList::RemoveIndex(const string &name) {
 
 	for (idx_t index_idx = 0; index_idx < indexes.size(); index_idx++) {
 		auto &index_entry = indexes[index_idx];
-		if (index_entry->name == name) {
-			indexes.erase(indexes.begin() + index_idx);
+
+		if (index_entry->GetIndexName() == name) {
+			indexes.erase_at(index_idx);
 			break;
 		}
 	}
@@ -32,9 +35,8 @@ void TableIndexList::CommitDrop(const string &name) {
 
 	for (idx_t index_idx = 0; index_idx < indexes.size(); index_idx++) {
 		auto &index_entry = indexes[index_idx];
-		if (index_entry->name == name) {
+		if (index_entry->GetIndexName() == name) {
 			index_entry->CommitDrop();
-			break;
 		}
 	}
 }
@@ -46,7 +48,7 @@ bool TableIndexList::NameIsUnique(const string &name) {
 	for (idx_t index_idx = 0; index_idx < indexes.size(); index_idx++) {
 		auto &index_entry = indexes[index_idx];
 		if (index_entry->IsPrimary() || index_entry->IsForeign() || index_entry->IsUnique()) {
-			if (index_entry->name == name) {
+			if (index_entry->GetIndexName() == name) {
 				return false;
 			}
 		}
@@ -55,38 +57,53 @@ bool TableIndexList::NameIsUnique(const string &name) {
 	return true;
 }
 
-void TableIndexList::InitializeIndexes(ClientContext &context, DataTableInfo &table_info, bool throw_on_failure) {
-	lock_guard<mutex> lock(indexes_lock);
-	for (auto &index : indexes) {
-		if (!index->IsUnknown()) {
-			continue;
-		}
-
-		auto &unknown_index = index->Cast<UnknownIndex>();
-		auto &index_type_name = unknown_index.GetIndexType();
-
-		// Do we know the type of this index now?
-		auto index_type = context.db->config.GetIndexTypes().FindByName(index_type_name);
-		if (!index_type) {
-			if (throw_on_failure) {
-				throw MissingExtensionException(
-				    "Cannot initialize index '%s', unknown index type '%s'. You probably need to load an extension.",
-				    unknown_index.name, index_type_name);
+void TableIndexList::InitializeIndexes(ClientContext &context, DataTableInfo &table_info, const char *index_type) {
+	// Fast path: do we have any unbound indexes?
+	bool needs_binding = false;
+	{
+		lock_guard<mutex> lock(indexes_lock);
+		for (auto &index : indexes) {
+			if (!index->IsBound() && (index_type == nullptr || index->GetIndexType() == index_type)) {
+				needs_binding = true;
+				break;
 			}
-			continue;
 		}
+	}
+	if (!needs_binding) {
+		return;
+	}
 
-		// Swap this with a new index
-		auto &create_info = unknown_index.GetCreateInfo();
-		auto &storage_info = unknown_index.GetStorageInfo();
-
-		CreateIndexInput input(*table_info.table_io_manager, table_info.db, create_info.constraint_type,
-		                       create_info.index_name, create_info.column_ids, unknown_index.unbound_expressions,
-		                       storage_info, create_info.options);
-
-		auto index_instance = index_type->create_instance(input);
+	// Get the table from the catalog so we can add it to the binder
+	auto &catalog = table_info.GetDB().GetCatalog();
+	auto &table =
+	    catalog.GetEntry(context, CatalogType::TABLE_ENTRY, table_info.GetSchemaName(), table_info.GetTableName())
+	        .Cast<DuckTableEntry>();
+	vector<LogicalType> column_types;
+	vector<string> column_names;
+	for (auto &col : table.GetColumns().Logical()) {
+		column_types.push_back(col.Type());
+		column_names.push_back(col.Name());
+	}
 
-		index = std::move(index_instance);
+	lock_guard<mutex> lock(indexes_lock);
+	for (auto &index : indexes) {
+		if (!index->IsBound() && (index_type == nullptr || index->GetIndexType() == index_type)) {
+			// Create a binder to bind this index (we cant reuse this binder for other indexes)
+			auto binder = Binder::CreateBinder(context);
+
+			// Add the table to the binder
+			// We're not interested in the column_ids here, so just pass a dummy vector
+			vector<column_t> dummy_column_ids;
+			binder->bind_context.AddBaseTable(0, table_info.GetTableName(), column_names, column_types,
+			                                  dummy_column_ids, &table);
+
+			// Create an IndexBinder to bind the index
+			IndexBinder idx_binder(*binder, context);
+
+			// Replace the unbound index with a bound index
+			auto bound_idx = idx_binder.BindIndex(index->Cast<UnboundIndex>());
+			index = std::move(bound_idx);
+		}
 	}
 }
 
@@ -127,15 +144,18 @@ void TableIndexList::VerifyForeignKey(const vector<PhysicalIndex> &fk_keys, Data
 	if (!index) {
 		throw InternalException("Internal Foreign Key error: could not find index to verify...");
 	}
+	if (!index->IsBound()) {
+		throw InternalException("Internal Foreign Key error: trying to verify an unbound index...");
+	}
 	conflict_manager.SetIndexCount(1);
-	index->CheckConstraintsForChunk(chunk, conflict_manager);
+	index->Cast<BoundIndex>().CheckConstraintsForChunk(chunk, conflict_manager);
 }
 
 vector<column_t> TableIndexList::GetRequiredColumns() {
 	lock_guard<mutex> lock(indexes_lock);
 	set<column_t> unique_indexes;
 	for (auto &index : indexes) {
-		for (auto col_index : index->column_ids) {
+		for (auto col_index : index->GetColumnIds()) {
 			unique_indexes.insert(col_index);
 		}
 	}
@@ -151,9 +171,16 @@ vector<IndexStorageInfo> TableIndexList::GetStorageInfos() {
 
 	vector<IndexStorageInfo> index_storage_infos;
 	for (auto &index : indexes) {
-		auto index_storage_info = index->GetStorageInfo(false);
-		D_ASSERT(index_storage_info.IsValid() && !index_storage_info.name.empty());
-		index_storage_infos.push_back(index_storage_info);
+		if (index->IsBound()) {
+			auto index_storage_info = index->Cast<BoundIndex>().GetStorageInfo(false);
+			D_ASSERT(index_storage_info.IsValid() && !index_storage_info.name.empty());
+			index_storage_infos.push_back(index_storage_info);
+		} else {
+			// TODO: Will/should this ever happen?
+			auto index_storage_info = index->Cast<UnboundIndex>().GetStorageInfo();
+			D_ASSERT(index_storage_info.IsValid() && !index_storage_info.name.empty());
+			index_storage_infos.push_back(index_storage_info);
+		}
 	}
 	return index_storage_infos;
 }

package/src/duckdb/src/storage/temporary_file_manager.cpp

@@ -8,7 +8,10 @@ namespace duckdb {
 // BlockIndexManager
 //===--------------------------------------------------------------------===//
 
-BlockIndexManager::BlockIndexManager() : max_index(0) {
+BlockIndexManager::BlockIndexManager(TemporaryFileManager &manager) : max_index(0), manager(&manager) {
+}
+
+BlockIndexManager::BlockIndexManager() : max_index(0), manager(nullptr) {
 }
 
 idx_t BlockIndexManager::GetNewBlockIndex() {
@@ -28,11 +31,11 @@ bool BlockIndexManager::RemoveIndex(idx_t index) {
 	// check if we can truncate the file
 
 	// get the max_index in use right now
-	auto max_index_in_use = indexes_in_use.empty() ? 0 : *indexes_in_use.rbegin();
+	auto max_index_in_use = indexes_in_use.empty() ? 0 : *indexes_in_use.rbegin() + 1;
 	if (max_index_in_use < max_index) {
 		// max index in use is lower than the max_index
 		// reduce the max_index
-		max_index = indexes_in_use.empty() ? 0 : max_index_in_use + 1;
+		SetMaxIndex(max_index_in_use);
 		// we can remove any free_indexes that are larger than the current max_index
 		while (!free_indexes.empty()) {
 			auto max_entry = *free_indexes.rbegin();
@@ -54,9 +57,32 @@ bool BlockIndexManager::HasFreeBlocks() {
 	return !free_indexes.empty();
 }
 
+void BlockIndexManager::SetMaxIndex(idx_t new_index) {
+	static constexpr idx_t TEMPFILE_BLOCK_SIZE = Storage::BLOCK_ALLOC_SIZE;
+	if (!manager) {
+		max_index = new_index;
+	} else {
+		auto old = max_index;
+		if (new_index < old) {
+			max_index = new_index;
+			auto difference = old - new_index;
+			auto size_on_disk = difference * TEMPFILE_BLOCK_SIZE;
+			manager->DecreaseSizeOnDisk(size_on_disk);
+		} else if (new_index > old) {
+			auto difference = new_index - old;
+			auto size_on_disk = difference * TEMPFILE_BLOCK_SIZE;
+			manager->IncreaseSizeOnDisk(size_on_disk);
+			// Increase can throw, so this is only updated after it was succesfully updated
+			max_index = new_index;
+		}
+	}
+}
+
 idx_t BlockIndexManager::GetNewBlockIndexInternal() {
 	if (free_indexes.empty()) {
-		return max_index++;
+		auto new_index = max_index;
+		SetMaxIndex(max_index + 1);
+		return new_index;
 	}
 	auto entry = free_indexes.begin();
 	auto index = *entry;
@@ -69,9 +95,10 @@ idx_t BlockIndexManager::GetNewBlockIndexInternal() {
 //===--------------------------------------------------------------------===//
 
 TemporaryFileHandle::TemporaryFileHandle(idx_t temp_file_count, DatabaseInstance &db, const string &temp_directory,
-                                         idx_t index)
+                                         idx_t index, TemporaryFileManager &manager)
     : max_allowed_index((1 << temp_file_count) * MAX_ALLOWED_INDEX_BASE), db(db), file_index(index),
-      path(FileSystem::GetFileSystem(db).JoinPath(temp_directory, "duckdb_temp_storage-" + to_string(index) + ".tmp")) {
+      path(FileSystem::GetFileSystem(db).JoinPath(temp_directory, "duckdb_temp_storage-" + to_string(index) + ".tmp")),
+      index_manager(manager) {
 }
 
 TemporaryFileHandle::TemporaryFileLock::TemporaryFileLock(mutex &mutex) : lock(mutex) {
@@ -106,7 +133,7 @@ void TemporaryFileHandle::EraseBlockIndex(block_id_t block_index) {
 	// remove the block (and potentially truncate the temp file)
 	TemporaryFileLock lock(file_lock);
 	D_ASSERT(handle);
-	RemoveTempBlockIndex(lock, block_index);
+	RemoveTempBlockIndex(lock, NumericCast<idx_t>(block_index));
 }
 
 bool TemporaryFileHandle::DeleteIfEmpty() {
@@ -147,7 +174,7 @@ void TemporaryFileHandle::RemoveTempBlockIndex(TemporaryFileLock &, idx_t index)
 #ifndef WIN32 // this ended up causing issues when sorting
 		auto max_index = index_manager.GetMaxIndex();
 		auto &fs = FileSystem::GetFileSystem(db);
-		fs.Truncate(*handle, GetPositionInFile(max_index + 1));
+		fs.Truncate(*handle, NumericCast<int64_t>(GetPositionInFile(max_index + 1)));
 #endif
 	}
 }
@@ -160,15 +187,15 @@ idx_t TemporaryFileHandle::GetPositionInFile(idx_t index) {
 // TemporaryDirectoryHandle
 //===--------------------------------------------------------------------===//
 
-TemporaryDirectoryHandle::TemporaryDirectoryHandle(DatabaseInstance &db, string path_p)
+TemporaryDirectoryHandle::TemporaryDirectoryHandle(DatabaseInstance &db, string path_p, optional_idx max_swap_space)
     : db(db), temp_directory(std::move(path_p)), temp_file(make_uniq<TemporaryFileManager>(db, temp_directory)) {
 	auto &fs = FileSystem::GetFileSystem(db);
-	if (!temp_directory.empty()) {
-		if (!fs.DirectoryExists(temp_directory)) {
-			fs.CreateDirectory(temp_directory);
-			created_directory = true;
-		}
+	D_ASSERT(!temp_directory.empty());
+	if (!fs.DirectoryExists(temp_directory)) {
+		fs.CreateDirectory(temp_directory);
+		created_directory = true;
 	}
+	temp_file->SetMaxSwapSpace(max_swap_space);
 }
 
 TemporaryDirectoryHandle::~TemporaryDirectoryHandle() {
@@ -224,8 +251,26 @@ bool TemporaryFileIndex::IsValid() const {
 // TemporaryFileManager
 //===--------------------------------------------------------------------===//
 
+static idx_t GetDefaultMax(const string &path) {
+	D_ASSERT(!path.empty());
+	auto disk_space = FileSystem::GetAvailableDiskSpace(path);
+	// Use the available disk space
+	// We have made sure that the file exists before we call this, it shouldn't fail
+	if (!disk_space.IsValid()) {
+		// But if it does (i.e because the system call is not implemented)
+		// we don't cap the available swap space
+		return DConstants::INVALID_INDEX - 1;
+	}
+	// Only use 90% of the available disk space
+	return static_cast<idx_t>(static_cast<double>(disk_space.GetIndex()) * 0.9);
+}
+
 TemporaryFileManager::TemporaryFileManager(DatabaseInstance &db, const string &temp_directory_p)
-    : db(db), temp_directory(temp_directory_p) {
+    : db(db), temp_directory(temp_directory_p), size_on_disk(0), max_swap_space(0) {
+}
+
+TemporaryFileManager::~TemporaryFileManager() {
+	files.clear();
 }
 
 TemporaryFileManager::TemporaryManagerLock::TemporaryManagerLock(mutex &mutex) : lock(mutex) {
@@ -250,7 +295,7 @@ void TemporaryFileManager::WriteTemporaryBuffer(block_id_t block_id, FileBuffer
 		if (!handle) {
 			// no existing handle to write to; we need to create & open a new file
 			auto new_file_index = index_manager.GetNewBlockIndex();
-			auto new_file = make_uniq<TemporaryFileHandle>(files.size(), db, temp_directory, new_file_index);
+			auto new_file = make_uniq<TemporaryFileHandle>(files.size(), db, temp_directory, new_file_index, *this);
 			handle = new_file.get();
 			files[new_file_index] = std::move(new_file);
 
@@ -269,6 +314,55 @@ bool TemporaryFileManager::HasTemporaryBuffer(block_id_t block_id) {
 	return used_blocks.find(block_id) != used_blocks.end();
 }
 
+idx_t TemporaryFileManager::GetTotalUsedSpaceInBytes() {
+	return size_on_disk.load();
+}
+
+optional_idx TemporaryFileManager::GetMaxSwapSpace() const {
+	return max_swap_space;
+}
+
+void TemporaryFileManager::SetMaxSwapSpace(optional_idx limit) {
+	idx_t new_limit;
+	if (limit.IsValid()) {
+		new_limit = limit.GetIndex();
+	} else {
+		new_limit = GetDefaultMax(temp_directory);
+	}
+
+	auto current_size_on_disk = size_on_disk.load();
+	if (current_size_on_disk > new_limit) {
+		auto used = StringUtil::BytesToHumanReadableString(current_size_on_disk);
+		auto max = StringUtil::BytesToHumanReadableString(new_limit);
+		throw OutOfMemoryException(
+		    R"(failed to adjust the 'max_temp_directory_size', currently used space (%s) exceeds the new limit (%s)
+Please increase the limit or destroy the buffers stored in the temp directory by e.g removing temporary tables.
+To get usage information of the temp_directory, use 'CALL duckdb_temporary_files();'
+		)",
+		    used, max);
+	}
+	max_swap_space = new_limit;
+}
+
+void TemporaryFileManager::IncreaseSizeOnDisk(idx_t bytes) {
+	auto current_size_on_disk = size_on_disk.load();
+	if (current_size_on_disk + bytes > max_swap_space) {
+		auto used = StringUtil::BytesToHumanReadableString(current_size_on_disk);
+		auto max = StringUtil::BytesToHumanReadableString(max_swap_space);
+		auto data_size = StringUtil::BytesToHumanReadableString(bytes);
+		throw OutOfMemoryException(R"(failed to offload data block of size %s (%s/%s used).
+This limit was set by the 'max_temp_directory_size' setting.
+By default, this setting utilizes the available disk space on the drive where the 'temp_directory' is located.
+You can adjust this setting, by using (for example) PRAGMA max_temp_directory_size='10GiB')",
+		                           data_size, used, max);
+	}
+	size_on_disk += bytes;
+}
+
+void TemporaryFileManager::DecreaseSizeOnDisk(idx_t bytes) {
+	size_on_disk -= bytes;
+}
+
 unique_ptr<FileBuffer> TemporaryFileManager::ReadTemporaryBuffer(block_id_t id,
                                                                  unique_ptr<FileBuffer> reusable_buffer) {
 	TemporaryFileIndex index;
@@ -310,7 +404,7 @@ void TemporaryFileManager::EraseUsedBlock(TemporaryManagerLock &lock, block_id_t
 		throw InternalException("EraseUsedBlock - Block %llu not found in used blocks", id);
 	}
 	used_blocks.erase(entry);
-	handle->EraseBlockIndex(index.block_index);
+	handle->EraseBlockIndex(NumericCast<block_id_t>(index.block_index));
 	if (handle->DeleteIfEmpty()) {
 		EraseFileHandle(lock, index.file_index);
 	}

package/src/duckdb/src/storage/temporary_memory_manager.cpp

@@ -45,9 +45,9 @@ void TemporaryMemoryManager::UpdateConfiguration(ClientContext &context) {
 	auto &buffer_manager = BufferManager::GetBufferManager(context);
 	auto &task_scheduler = TaskScheduler::GetScheduler(context);
 
-	memory_limit = MAXIMUM_MEMORY_LIMIT_RATIO * double(buffer_manager.GetMaxMemory());
+	memory_limit = NumericCast<idx_t>(MAXIMUM_MEMORY_LIMIT_RATIO * static_cast<double>(buffer_manager.GetMaxMemory()));
 	has_temporary_directory = buffer_manager.HasTemporaryDirectory();
-	num_threads = task_scheduler.NumberOfThreads();
+	num_threads = NumericCast<idx_t>(task_scheduler.NumberOfThreads());
 	query_max_memory = buffer_manager.GetQueryMaxMemory();
 }
 
@@ -92,14 +92,14 @@ void TemporaryMemoryManager::UpdateState(ClientContext &context, TemporaryMemory
 		// 3. MAXIMUM_FREE_MEMORY_RATIO * free memory
 		auto upper_bound = MinValue<idx_t>(temporary_memory_state.remaining_size, query_max_memory);
 		auto free_memory = memory_limit - (reservation - temporary_memory_state.reservation);
-		upper_bound = MinValue<idx_t>(upper_bound, MAXIMUM_FREE_MEMORY_RATIO * free_memory);
+		upper_bound = MinValue<idx_t>(upper_bound, NumericCast<idx_t>(MAXIMUM_FREE_MEMORY_RATIO * free_memory));
 
 		if (remaining_size > memory_limit) {
 			// We're processing more data than fits in memory, so we must further limit memory usage.
 			// The upper bound for the reservation of this state is now also the minimum of:
 			// 3. The ratio of the remaining size of this state and the total remaining size * memory limit
 			auto ratio_of_remaining = double(temporary_memory_state.remaining_size) / double(remaining_size);
-			upper_bound = MinValue<idx_t>(upper_bound, ratio_of_remaining * memory_limit);
+			upper_bound = MinValue<idx_t>(upper_bound, NumericCast<idx_t>(ratio_of_remaining * memory_limit));
 		}
 
 		SetReservation(temporary_memory_state, MaxValue<idx_t>(lower_bound, upper_bound));

package/src/duckdb/src/storage/wal_replay.cpp

@@ -25,6 +25,9 @@
 #include "duckdb/common/checksum.hpp"
 #include "duckdb/execution/index/index_type_set.hpp"
 #include "duckdb/execution/index/art/art.hpp"
+#include "duckdb/main/config.hpp"
+#include "duckdb/storage/table/delete_state.hpp"
+#include "duckdb/transaction/meta_transaction.hpp"
 
 namespace duckdb {
 
@@ -160,6 +163,7 @@ private:
 //===--------------------------------------------------------------------===//
 bool WriteAheadLog::Replay(AttachedDatabase &database, unique_ptr<FileHandle> handle) {
 	Connection con(database.GetDatabase());
+	auto wal_path = handle->GetPath();
 	BufferedFileReader reader(FileSystem::Get(database), std::move(handle));
 	if (reader.Finished()) {
 		// WAL is empty
@@ -167,7 +171,9 @@ bool WriteAheadLog::Replay(AttachedDatabase &database, unique_ptr<FileHandle> ha
 	}
 
 	con.BeginTransaction();
+	MetaTransaction::Get(*con.context).ModifyDatabase(database);
 
+	auto &config = DBConfig::GetConfig(database.GetDatabase());
 	// first deserialize the WAL to look for a checkpoint flag
 	// if there is a checkpoint flag, we might have already flushed the contents of the WAL to disk
 	ReplayState checkpoint_state(database, *con.context);
@@ -185,16 +191,10 @@ bool WriteAheadLog::Replay(AttachedDatabase &database, unique_ptr<FileHandle> ha
 		}
 	} catch (std::exception &ex) { // LCOV_EXCL_START
 		ErrorData error(ex);
-		if (error.Type() == ExceptionType::SERIALIZATION) {
-			// serialization exception - torn WAL
-			// continue reading
-		} else {
-			Printer::PrintF("Exception in WAL playback during initial read: %s\n", error.RawMessage());
-			return false;
+		// ignore serialization exceptions - they signal a torn WAL
+		if (error.Type() != ExceptionType::SERIALIZATION) {
+			error.Throw("Failure while replaying WAL file \"" + wal_path + "\": ");
 		}
-	} catch (...) {
-		Printer::Print("Unknown Exception in WAL playback during initial read");
-		return false;
 	} // LCOV_EXCL_STOP
 	if (checkpoint_state.checkpoint_id.IsValid()) {
 		// there is a checkpoint flag: check if we need to deserialize the WAL
@@ -215,7 +215,6 @@ bool WriteAheadLog::Replay(AttachedDatabase &database, unique_ptr<FileHandle> ha
 	// replay the WAL
 	// note that everything is wrapped inside a try/catch block here
 	// there can be errors in WAL replay because of a corrupt WAL file
-	// in this case we should throw a warning but startup anyway
 	try {
 		while (true) {
 			// read the current entry
@@ -228,20 +227,23 @@ bool WriteAheadLog::Replay(AttachedDatabase &database, unique_ptr<FileHandle> ha
 					break;
 				}
 				con.BeginTransaction();
+				MetaTransaction::Get(*con.context).ModifyDatabase(database);
 			}
 		}
 	} catch (std::exception &ex) { // LCOV_EXCL_START
+		// exception thrown in WAL replay: rollback
+		con.Query("ROLLBACK");
 		ErrorData error(ex);
-		if (error.Type() != ExceptionType::SERIALIZATION) {
-			// FIXME: this should report a proper warning in the connection
-			Printer::PrintF("Exception in WAL playback: %s\n", error.RawMessage());
-			// exception thrown in WAL replay: rollback
+		// serialization failure means a truncated WAL
+		// these failures are ignored unless abort_on_wal_failure is true
+		// other failures always result in an error
+		if (config.options.abort_on_wal_failure || error.Type() != ExceptionType::SERIALIZATION) {
+			error.Throw("Failure while replaying WAL file \"" + wal_path + "\": ");
 		}
-		con.Rollback();
 	} catch (...) {
-		Printer::Print("Unknown Exception in WAL playback: %s\n");
 		// exception thrown in WAL replay: rollback
-		con.Rollback();
+		con.Query("ROLLBACK");
+		throw;
 	} // LCOV_EXCL_STOP
 	return false;
 }
@@ -346,7 +348,7 @@ void WriteAheadLogDeserializer::ReplayCreateTable() {
 	// bind the constraints to the table again
 	auto binder = Binder::CreateBinder(context);
 	auto &schema = catalog.GetSchema(context, info->schema);
-	auto bound_info = binder->BindCreateTableInfo(std::move(info), schema);
+	auto bound_info = Binder::BindCreateTableCheckpoint(std::move(info), schema);
 
 	catalog.CreateTable(context, *bound_info);
 }
@@ -580,7 +582,7 @@ void WriteAheadLogDeserializer::ReplayCreateIndex() {
 	// create the index in the catalog
 	auto &table = catalog.GetEntry<TableCatalogEntry>(context, create_info->schema, info.table).Cast<DuckTableEntry>();
 	auto &index = catalog.CreateIndex(context, info)->Cast<DuckIndexEntry>();
-	index.info = make_shared<IndexDataTableInfo>(table.GetStorage().info, index.name);
+	index.info = make_shared_ptr<IndexDataTableInfo>(table.GetStorage().GetDataTableInfo(), index.name);
 
 	// insert the parsed expressions into the index so that we can (de)serialize them during consecutive checkpoints
 	for (auto &parsed_expr : info.parsed_expressions) {
@@ -621,7 +623,7 @@ void WriteAheadLogDeserializer::ReplayCreateIndex() {
 	                       info.column_ids, unbound_expressions, index_info, info.options);
 
 	auto index_instance = index_type->create_instance(input);
-	data_table.info->indexes.AddIndex(std::move(index_instance));
+	data_table.AddIndex(std::move(index_instance));
 }
 
 void WriteAheadLogDeserializer::ReplayDropIndex() {
@@ -659,7 +661,9 @@ void WriteAheadLogDeserializer::ReplayInsert() {
 	}
 
 	// append to the current table
-	state.current_table->GetStorage().LocalAppend(*state.current_table, context, chunk);
+	// we don't do any constraint verification here
+	vector<unique_ptr<BoundConstraint>> bound_constraints;
+	state.current_table->GetStorage().LocalAppend(*state.current_table, context, chunk, bound_constraints);
 }
 
 void WriteAheadLogDeserializer::ReplayDelete() {
@@ -678,9 +682,10 @@ void WriteAheadLogDeserializer::ReplayDelete() {
 
 	auto source_ids = FlatVector::GetData<row_t>(chunk.data[0]);
 	// delete the tuples from the current table
+	TableDeleteState delete_state;
 	for (idx_t i = 0; i < chunk.size(); i++) {
 		row_ids[0] = source_ids[i];
-		state.current_table->GetStorage().Delete(*state.current_table, context, row_identifiers, 1);
+		state.current_table->GetStorage().Delete(delete_state, context, row_identifiers, 1);
 	}
 }