duckdb 0.7.2-dev329.0 → 0.7.2-dev3294.0

package/src/duckdb/src/storage/compression/fsst.cpp

@@ -80,7 +80,7 @@ struct FSSTAnalyzeState : public AnalyzeState {
 	idx_t count;
 
 	StringHeap fsst_string_heap;
-	std::vector<string_t> fsst_strings;
+	vector<string_t> fsst_strings;
 	size_t fsst_string_total_size;
 
 	RandomEngine random_engine;
@@ -90,7 +90,7 @@ struct FSSTAnalyzeState : public AnalyzeState {
 };
 
 unique_ptr<AnalyzeState> FSSTStorage::StringInitAnalyze(ColumnData &col_data, PhysicalType type) {
-	return make_unique<FSSTAnalyzeState>();
+	return make_uniq<FSSTAnalyzeState>();
 }
 
 bool FSSTStorage::StringAnalyze(AnalyzeState &state_p, Vector &input, idx_t count) {
@@ -151,18 +151,18 @@ idx_t FSSTStorage::StringFinalAnalyze(AnalyzeState &state_p) {
 
 	size_t output_buffer_size = 7 + 2 * state.fsst_string_total_size; // size as specified in fsst.h
 
-	std::vector<size_t> fsst_string_sizes;
-	std::vector<unsigned char *> fsst_string_ptrs;
+	vector<size_t> fsst_string_sizes;
+	vector<unsigned char *> fsst_string_ptrs;
 	for (auto &str : state.fsst_strings) {
 		fsst_string_sizes.push_back(str.GetSize());
-		fsst_string_ptrs.push_back((unsigned char *)str.GetDataUnsafe());
+		fsst_string_ptrs.push_back((unsigned char *)str.GetData());
 	}
 
 	state.fsst_encoder = duckdb_fsst_create(string_count, &fsst_string_sizes[0], &fsst_string_ptrs[0], 0);
 
 	// TODO: do we really need to encode to get a size estimate?
-	auto compressed_ptrs = std::vector<unsigned char *>(string_count, nullptr);
-	auto compressed_sizes = std::vector<size_t>(string_count, 0);
+	auto compressed_ptrs = vector<unsigned char *>(string_count, nullptr);
+	auto compressed_sizes = vector<size_t>(string_count, 0);
 	unique_ptr<unsigned char[]> compressed_buffer(new unsigned char[output_buffer_size]);
 
 	auto res =
@@ -199,10 +199,8 @@ idx_t FSSTStorage::StringFinalAnalyze(AnalyzeState &state_p) {
 
 class FSSTCompressionState : public CompressionState {
 public:
-	explicit FSSTCompressionState(ColumnDataCheckpointer &checkpointer) : checkpointer(checkpointer) {
-		auto &db = checkpointer.GetDatabase();
-		auto &config = DBConfig::GetConfig(db);
-		function = config.GetCompressionFunction(CompressionType::COMPRESSION_FSST, PhysicalType::VARCHAR);
+	explicit FSSTCompressionState(ColumnDataCheckpointer &checkpointer)
+	    : checkpointer(checkpointer), function(checkpointer.GetCompressionFunction(CompressionType::COMPRESSION_FSST)) {
 		CreateEmptySegment(checkpointer.GetRowGroup().start);
 	}
 
@@ -372,7 +370,7 @@ public:
 	}
 
 	ColumnDataCheckpointer &checkpointer;
-	CompressionFunction *function;
+	CompressionFunction &function;
 
 	// State regarding current segment
 	unique_ptr<ColumnSegment> current_segment;
@@ -381,7 +379,7 @@ public:
 	data_ptr_t current_end_ptr;
 
 	// Buffers and map for current segment
-	std::vector<uint32_t> index_buffer;
+	vector<uint32_t> index_buffer;
 
 	size_t max_compressed_string_length;
 	bitpacking_width_t current_width;
@@ -395,7 +393,7 @@ public:
 unique_ptr<CompressionState> FSSTStorage::InitCompression(ColumnDataCheckpointer &checkpointer,
                                                           unique_ptr<AnalyzeState> analyze_state_p) {
 	auto analyze_state = static_cast<FSSTAnalyzeState *>(analyze_state_p.get());
-	auto compression_state = make_unique<FSSTCompressionState>(checkpointer);
+	auto compression_state = make_uniq<FSSTCompressionState>(checkpointer);
 
 	if (analyze_state->fsst_encoder == nullptr) {
 		throw InternalException("No encoder found during FSST compression");
@@ -433,7 +431,7 @@ void FSSTStorage::Compress(CompressionState &state_p, Vector &scan_vector, idx_t
 		total_count++;
 		total_size += data[idx].GetSize();
 		sizes_in.push_back(data[idx].GetSize());
-		strings_in.push_back((unsigned char *)data[idx].GetDataUnsafe());
+		strings_in.push_back((unsigned char *)data[idx].GetData());
 	}
 
 	// Only Nulls or empty strings in this vector, nothing to compress
@@ -518,7 +516,7 @@ struct FSSTScanState : public StringScanState {
 };
 
 unique_ptr<SegmentScanState> FSSTStorage::StringInitScan(ColumnSegment &segment) {
-	auto state = make_unique<FSSTScanState>();
+	auto state = make_uniq<FSSTScanState>();
 	auto &buffer_manager = BufferManager::GetBufferManager(segment.db);
 	state->handle = buffer_manager.Pin(segment.block);
 	auto base_ptr = state->handle.Ptr() + segment.GetBlockOffset();
@@ -671,7 +669,7 @@ void FSSTStorage::StringFetchRow(ColumnSegment &segment, ColumnFetchState &state
 		    segment, dict, result, base_ptr, delta_decode_buffer[offsets.unused_delta_decoded_values], string_length);
 
 		result_data[result_idx] = FSSTPrimitives::DecompressValue(
-		    (void *)&decoder, result, (unsigned char *)compressed_string.GetDataUnsafe(), compressed_string.GetSize());
+		    (void *)&decoder, result, (unsigned char *)compressed_string.GetData(), compressed_string.GetSize());
 	} else {
 		// There's no fsst symtable, this only happens for empty strings or nulls, we can just emit an empty string
 		result_data[result_idx] = string_t(nullptr, 0);

package/src/duckdb/src/storage/compression/numeric_constant.cpp

@@ -1,10 +1,10 @@
 #include "duckdb/function/compression/compression.hpp"
-#include "duckdb/storage/buffer_manager.hpp"
 #include "duckdb/common/types/vector.hpp"
 
 #include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/function/compression_function.hpp"
 #include "duckdb/storage/segment/uncompressed.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
 
 namespace duckdb {
 
@@ -33,7 +33,7 @@ void ConstantFillFunction(ColumnSegment &segment, Vector &result, idx_t start_id
 	auto &nstats = segment.stats.statistics;
 
 	auto data = FlatVector::GetData<T>(result);
-	auto constant_value = NumericStats::GetMinUnsafe<T>(nstats);
+	auto constant_value = NumericStats::GetMin<T>(nstats);
 	for (idx_t i = 0; i < count; i++) {
 		data[start_idx + i] = constant_value;
 	}
@@ -71,7 +71,7 @@ void ConstantScanFunction(ColumnSegment &segment, ColumnScanState &state, idx_t
 	auto &nstats = segment.stats.statistics;
 
 	auto data = FlatVector::GetData<T>(result);
-	data[0] = NumericStats::GetMinUnsafe<T>(nstats);
+	data[0] = NumericStats::GetMin<T>(nstats);
 	result.SetVectorType(VectorType::CONSTANT_VECTOR);
 }
 

package/src/duckdb/src/storage/compression/rle.cpp

@@ -6,6 +6,7 @@
 #include "duckdb/storage/table/column_data_checkpointer.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
 #include "duckdb/common/types/null_value.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
 #include <functional>
 
 namespace duckdb {
@@ -88,7 +89,7 @@ struct RLEAnalyzeState : public AnalyzeState {
 
 template <class T>
 unique_ptr<AnalyzeState> RLEInitAnalyze(ColumnData &col_data, PhysicalType type) {
-	return make_unique<RLEAnalyzeState<T>>();
+	return make_uniq<RLEAnalyzeState<T>>();
 }
 
 template <class T>
@@ -135,11 +136,9 @@ struct RLECompressState : public CompressionState {
 		return max_vector_count * STANDARD_VECTOR_SIZE;
 	}
 
-	explicit RLECompressState(ColumnDataCheckpointer &checkpointer_p) : checkpointer(checkpointer_p) {
-		auto &db = checkpointer.GetDatabase();
-		auto &type = checkpointer.GetType();
-		auto &config = DBConfig::GetConfig(db);
-		function = config.GetCompressionFunction(CompressionType::COMPRESSION_RLE, type.InternalType());
+	explicit RLECompressState(ColumnDataCheckpointer &checkpointer_p)
+	    : checkpointer(checkpointer_p),
+	      function(checkpointer.GetCompressionFunction(CompressionType::COMPRESSION_RLE)) {
 		CreateEmptySegment(checkpointer.GetRowGroup().start);
 
 		state.dataptr = (void *)this;
@@ -213,7 +212,7 @@ struct RLECompressState : public CompressionState {
 	}
 
 	ColumnDataCheckpointer &checkpointer;
-	CompressionFunction *function;
+	CompressionFunction &function;
 	unique_ptr<ColumnSegment> current_segment;
 	BufferHandle handle;
 
@@ -224,7 +223,7 @@ struct RLECompressState : public CompressionState {
 
 template <class T, bool WRITE_STATISTICS>
 unique_ptr<CompressionState> RLEInitCompression(ColumnDataCheckpointer &checkpointer, unique_ptr<AnalyzeState> state) {
-	return make_unique<RLECompressState<T, WRITE_STATISTICS>>(checkpointer);
+	return make_uniq<RLECompressState<T, WRITE_STATISTICS>>(checkpointer);
 }
 
 template <class T, bool WRITE_STATISTICS>
@@ -281,7 +280,7 @@ struct RLEScanState : public SegmentScanState {
 
 template <class T>
 unique_ptr<SegmentScanState> RLEInitScan(ColumnSegment &segment) {
-	auto result = make_unique<RLEScanState<T>>(segment);
+	auto result = make_uniq<RLEScanState<T>>(segment);
 	return std::move(result);
 }
 

package/src/duckdb/src/storage/compression/string_uncompressed.cpp

@@ -29,7 +29,7 @@ struct StringAnalyzeState : public AnalyzeState {
 };
 
 unique_ptr<AnalyzeState> UncompressedStringStorage::StringInitAnalyze(ColumnData &col_data, PhysicalType type) {
-	return make_unique<StringAnalyzeState>();
+	return make_uniq<StringAnalyzeState>();
 }
 
 bool UncompressedStringStorage::StringAnalyze(AnalyzeState &state_p, Vector &input, idx_t count) {
@@ -61,7 +61,7 @@ idx_t UncompressedStringStorage::StringFinalAnalyze(AnalyzeState &state_p) {
 // Scan
 //===--------------------------------------------------------------------===//
 unique_ptr<SegmentScanState> UncompressedStringStorage::StringInitScan(ColumnSegment &segment) {
-	auto result = make_unique<StringScanState>();
+	auto result = make_uniq<StringScanState>();
 	auto &buffer_manager = BufferManager::GetBufferManager(segment.db);
 	result->handle = buffer_manager.Pin(segment.block);
 	return std::move(result);
@@ -152,7 +152,7 @@ unique_ptr<CompressedSegmentState> UncompressedStringStorage::StringInitSegment(
 		dictionary.end = segment.SegmentSize();
 		SetDictionary(segment, handle, dictionary);
 	}
-	return make_unique<UncompressedStringSegmentState>();
+	return make_uniq<UncompressedStringSegmentState>();
 }
 
 idx_t UncompressedStringStorage::FinalizeAppend(ColumnSegment &segment, SegmentStatistics &stats) {
@@ -246,7 +246,7 @@ void UncompressedStringStorage::WriteStringMemory(ColumnSegment &segment, string
 		// string does not fit, allocate space for it
 		// create a new string block
 		idx_t alloc_size = MaxValue<idx_t>(total_length, Storage::BLOCK_SIZE);
-		auto new_block = make_unique<StringBlock>();
+		auto new_block = make_uniq<StringBlock>();
 		new_block->offset = 0;
 		new_block->size = alloc_size;
 		// allocate an in-memory buffer for it
@@ -267,7 +267,7 @@ void UncompressedStringStorage::WriteStringMemory(ColumnSegment &segment, string
 	auto ptr = handle.Ptr() + state.head->offset;
 	Store<uint32_t>(string.GetSize(), ptr);
 	ptr += sizeof(uint32_t);
-	memcpy(ptr, string.GetDataUnsafe(), string.GetSize());
+	memcpy(ptr, string.GetData(), string.GetSize());
 	state.head->offset += total_length;
 }
 
@@ -292,13 +292,13 @@ string_t UncompressedStringStorage::ReadOverflowString(ColumnSegment &segment, V
 		offset += 2 * sizeof(uint32_t);
 
 		data_ptr_t decompression_ptr;
-		std::unique_ptr<data_t[]> decompression_buffer;
+		unique_ptr<data_t[]> decompression_buffer;
 
 		// If string is in single block we decompress straight from it, else we copy first
 		if (remaining <= Storage::BLOCK_SIZE - sizeof(block_id_t) - offset) {
 			decompression_ptr = handle.Ptr() + offset;
 		} else {
-			decompression_buffer = std::unique_ptr<data_t[]>(new data_t[compressed_size]);
+			decompression_buffer = unique_ptr<data_t[]>(new data_t[compressed_size]);
 			auto target_ptr = decompression_buffer.get();
 
 			// now append the string to the single buffer

package/src/duckdb/src/storage/compression/validity_uncompressed.cpp

@@ -6,7 +6,7 @@
 #include "duckdb/common/types/null_value.hpp"
 #include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/function/compression_function.hpp"
-#include "duckdb/main/config.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
 
 namespace duckdb {
 
@@ -184,7 +184,7 @@ struct ValidityAnalyzeState : public AnalyzeState {
 };
 
 unique_ptr<AnalyzeState> ValidityInitAnalyze(ColumnData &col_data, PhysicalType type) {
-	return make_unique<ValidityAnalyzeState>();
+	return make_uniq<ValidityAnalyzeState>();
 }
 
 bool ValidityAnalyze(AnalyzeState &state_p, Vector &input, idx_t count) {
@@ -207,7 +207,7 @@ struct ValidityScanState : public SegmentScanState {
 };
 
 unique_ptr<SegmentScanState> ValidityInitScan(ColumnSegment &segment) {
-	auto result = make_unique<ValidityScanState>();
+	auto result = make_uniq<ValidityScanState>();
 	auto &buffer_manager = BufferManager::GetBufferManager(segment.db);
 	result->handle = buffer_manager.Pin(segment.block);
 	result->block_id = segment.block->BlockId();
@@ -395,7 +395,7 @@ void ValidityFetchRow(ColumnSegment &segment, ColumnFetchState &state, row_t row
 static unique_ptr<CompressionAppendState> ValidityInitAppend(ColumnSegment &segment) {
 	auto &buffer_manager = BufferManager::GetBufferManager(segment.db);
 	auto handle = buffer_manager.Pin(segment.block);
-	return make_unique<CompressionAppendState>(std::move(handle));
+	return make_uniq<CompressionAppendState>(std::move(handle));
 }
 
 unique_ptr<CompressedSegmentState> ValidityInitSegment(ColumnSegment &segment, block_id_t block_id) {

package/src/duckdb/src/storage/data_table.cpp

@@ -5,7 +5,6 @@
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/helper.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
-#include "duckdb/common/sort/sort.hpp"
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/parser/constraints/list.hpp"
@@ -23,6 +22,8 @@
 #include "duckdb/main/attached_database.hpp"
 #include "duckdb/common/types/conflict_manager.hpp"
 #include "duckdb/common/types/constraint_conflict_info.hpp"
+#include "duckdb/storage/table/append_state.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
 
 namespace duckdb {
 
@@ -45,7 +46,7 @@ DataTable::DataTable(AttachedDatabase &db, shared_ptr<TableIOManager> table_io_m
 	auto types = GetTypes();
 	this->row_groups =
 	    make_shared<RowGroupCollection>(info, TableIOManager::Get(*this).GetBlockManagerForRowData(), types, 0);
-	if (data && !data->row_groups.empty()) {
+	if (data && data->row_group_count > 0) {
 		this->row_groups->Initialize(*data);
 	} else {
 		this->row_groups->InitializeEmpty();
@@ -68,7 +69,7 @@ DataTable::DataTable(ClientContext &context, DataTable &parent, ColumnDefinition
 
 	// also add this column to client local storage
 	auto &local_storage = LocalStorage::Get(context, db);
-	local_storage.AddColumn(&parent, this, new_column, default_value);
+	local_storage.AddColumn(parent, *this, new_column, default_value);
 
 	// this table replaces the previous table, hence the parent is no longer the root DataTable
 	parent.is_root = false;
@@ -113,7 +114,7 @@ DataTable::DataTable(ClientContext &context, DataTable &parent, idx_t removed_co
 
 	// scan the original table, and fill the new column with the transformed value
 	auto &local_storage = LocalStorage::Get(context, db);
-	local_storage.DropColumn(&parent, this, removed_column);
+	local_storage.DropColumn(parent, *this, removed_column);
 
 	// this table replaces the previous table, hence the parent is no longer the root DataTable
 	parent.is_root = false;
@@ -133,7 +134,7 @@ DataTable::DataTable(ClientContext &context, DataTable &parent, unique_ptr<Bound
 
 	// Get the local data ownership from old dt
 	auto &local_storage = LocalStorage::Get(context, db);
-	local_storage.MoveStorage(&parent, this);
+	local_storage.MoveStorage(parent, *this);
 	// this table replaces the previous table, hence the parent is no longer the root DataTable
 	parent.is_root = false;
 }
@@ -165,7 +166,7 @@ DataTable::DataTable(ClientContext &context, DataTable &parent, idx_t changed_id
 
 	// scan the original table, and fill the new column with the transformed value
 	auto &local_storage = LocalStorage::Get(context, db);
-	local_storage.ChangeType(&parent, this, changed_idx, target_type, bound_columns, cast_expr);
+	local_storage.ChangeType(parent, *this, changed_idx, target_type, bound_columns, cast_expr);
 
 	// this table replaces the previous table, hence the parent is no longer the root DataTable
 	parent.is_root = false;
@@ -196,7 +197,7 @@ void DataTable::InitializeScan(DuckTransaction &transaction, TableScanState &sta
                                TableFilterSet *table_filters) {
 	InitializeScan(state, column_ids, table_filters);
 	auto &local_storage = LocalStorage::Get(transaction);
-	local_storage.InitializeScan(this, state.local_state, table_filters);
+	local_storage.InitializeScan(*this, state.local_state, table_filters);
 }
 
 void DataTable::InitializeScanWithOffset(TableScanState &state, const vector<column_t> &column_ids, idx_t start_row,
@@ -218,7 +219,7 @@ void DataTable::InitializeParallelScan(ClientContext &context, ParallelTableScan
 	row_groups->InitializeParallelScan(state.scan_state);
 
 	auto &local_storage = LocalStorage::Get(context, db);
-	local_storage.InitializeParallelScan(this, state.local_state);
+	local_storage.InitializeParallelScan(*this, state.local_state);
 }
 
 bool DataTable::NextParallelScan(ClientContext &context, ParallelTableScanState &state, TableScanState &scan_state) {
@@ -227,7 +228,7 @@ bool DataTable::NextParallelScan(ClientContext &context, ParallelTableScanState
 	}
 	scan_state.table_state.batch_index = state.scan_state.batch_index;
 	auto &local_storage = LocalStorage::Get(context, db);
-	if (local_storage.NextParallelScan(context, this, state.local_state, scan_state.local_state)) {
+	if (local_storage.NextParallelScan(context, *this, state.local_state, scan_state.local_state)) {
 		return true;
 	} else {
 		// finished all scans: no more scans remaining
@@ -359,13 +360,13 @@ idx_t LocateErrorIndex(bool is_append, const ManagedSelection &matches) {
 	return failed_index;
 }
 
-[[noreturn]] static void ThrowForeignKeyConstraintError(idx_t failed_index, bool is_append, Index *index,
+[[noreturn]] static void ThrowForeignKeyConstraintError(idx_t failed_index, bool is_append, Index &index,
                                                         DataChunk &input) {
 	auto verify_type = is_append ? VerifyExistenceType::APPEND_FK : VerifyExistenceType::DELETE_FK;
 
 	D_ASSERT(failed_index != DConstants::INVALID_INDEX);
-	D_ASSERT(index->type == IndexType::ART);
-	auto &art_index = (ART &)*index;
+	D_ASSERT(index.type == IndexType::ART);
+	auto &art_index = index.Cast<ART>();
 	auto key_name = art_index.GenerateErrorKeyName(input, failed_index);
 	auto exception_msg = art_index.GenerateConstraintErrorMessage(verify_type, key_name);
 	throw ConstraintException(exception_msg);
@@ -396,14 +397,11 @@ void DataTable::VerifyForeignKeyConstraint(const BoundForeignKeyConstraint &bfk,
 		dst_keys_ptr = &bfk.info.fk_keys;
 	}
 
-	auto table_entry_ptr =
+	auto &table_entry_ptr =
 	    Catalog::GetEntry<TableCatalogEntry>(context, INVALID_CATALOG, bfk.info.schema, bfk.info.table);
-	if (table_entry_ptr == nullptr) {
-		throw InternalException("Can't find table \"%s\" in foreign key constraint", bfk.info.table);
-	}
 	// make the data chunk to check
 	vector<LogicalType> types;
-	for (auto &col : table_entry_ptr->GetColumns().Physical()) {
+	for (auto &col : table_entry_ptr.GetColumns().Physical()) {
 		types.emplace_back(col.Type());
 	}
 	DataChunk dst_chunk;
@@ -412,7 +410,7 @@ void DataTable::VerifyForeignKeyConstraint(const BoundForeignKeyConstraint &bfk,
 		dst_chunk.data[(*dst_keys_ptr)[i].index].Reference(chunk.data[(*src_keys_ptr)[i].index]);
 	}
 	dst_chunk.SetCardinality(chunk.size());
-	auto data_table = table_entry_ptr->GetStoragePtr();
+	auto &data_table = table_entry_ptr.GetStorage();
 
 	idx_t count = dst_chunk.size();
 	if (count <= 0) {
@@ -427,7 +425,7 @@ void DataTable::VerifyForeignKeyConstraint(const BoundForeignKeyConstraint &bfk,
 	regular_conflicts.SetMode(ConflictManagerMode::SCAN);
 	transaction_conflicts.SetMode(ConflictManagerMode::SCAN);
 
-	data_table->info->indexes.VerifyForeignKey(*dst_keys_ptr, dst_chunk, regular_conflicts);
+	data_table.info->indexes.VerifyForeignKey(*dst_keys_ptr, dst_chunk, regular_conflicts);
 	regular_conflicts.Finalize();
 	auto &regular_matches = regular_conflicts.Conflicts();
 	// check whether or not the chunk can be inserted or deleted into the referenced table' transaction local storage
@@ -451,12 +449,12 @@ void DataTable::VerifyForeignKeyConstraint(const BoundForeignKeyConstraint &bfk,
 	}
 
 	// Some error occurred, and we likely want to throw
-	Index *index;
-	Index *transaction_index;
+	optional_ptr<Index> index;
+	optional_ptr<Index> transaction_index;
 
 	auto fk_type = is_append ? ForeignKeyType::FK_TYPE_PRIMARY_KEY_TABLE : ForeignKeyType::FK_TYPE_FOREIGN_KEY_TABLE;
 	// check whether or not the chunk can be inserted or deleted into the referenced table' storage
-	index = data_table->info->indexes.FindForeignKeyIndex(*dst_keys_ptr, fk_type);
+	index = data_table.info->indexes.FindForeignKeyIndex(*dst_keys_ptr, fk_type);
 	if (transaction_check) {
 		auto &transact_index = local_storage.GetIndexes(data_table);
 		// check whether or not the chunk can be inserted or deleted into the referenced table' storage
@@ -468,7 +466,7 @@ void DataTable::VerifyForeignKeyConstraint(const BoundForeignKeyConstraint &bfk,
 		D_ASSERT(error);
 		auto failed_index = LocateErrorIndex(is_append, regular_matches);
 		D_ASSERT(failed_index != DConstants::INVALID_INDEX);
-		ThrowForeignKeyConstraintError(failed_index, is_append, index, dst_chunk);
+		ThrowForeignKeyConstraintError(failed_index, is_append, *index, dst_chunk);
 	}
 	if (transaction_error && error && is_append) {
 		// When we want to do an append, we only throw if the foreign key does not exist in both transaction and local
@@ -494,20 +492,20 @@ void DataTable::VerifyForeignKeyConstraint(const BoundForeignKeyConstraint &bfk,
 			// We don't throw, every value was present in either regular or transaction storage
 			return;
 		}
-		ThrowForeignKeyConstraintError(failed_index, true, index, dst_chunk);
+		ThrowForeignKeyConstraintError(failed_index, true, *index, dst_chunk);
 	}
 	if (!is_append && transaction_check) {
 		auto &transaction_matches = transaction_conflicts.Conflicts();
 		if (error) {
 			auto failed_index = LocateErrorIndex(false, regular_matches);
 			D_ASSERT(failed_index != DConstants::INVALID_INDEX);
-			ThrowForeignKeyConstraintError(failed_index, false, index, dst_chunk);
+			ThrowForeignKeyConstraintError(failed_index, false, *index, dst_chunk);
 		} else {
 			D_ASSERT(transaction_error);
 			D_ASSERT(transaction_matches.Count() != DConstants::INVALID_INDEX);
 			auto failed_index = LocateErrorIndex(false, transaction_matches);
 			D_ASSERT(failed_index != DConstants::INVALID_INDEX);
-			ThrowForeignKeyConstraintError(failed_index, false, transaction_index, dst_chunk);
+			ThrowForeignKeyConstraintError(failed_index, false, *transaction_index, dst_chunk);
 		}
 	}
 }
@@ -634,7 +632,7 @@ void DataTable::InitializeLocalAppend(LocalAppendState &state, ClientContext &co
 		throw TransactionException("Transaction conflict: adding entries to a table that has been altered!");
 	}
 	auto &local_storage = LocalStorage::Get(context, db);
-	local_storage.InitializeAppend(state, this);
+	local_storage.InitializeAppend(state, *this);
 }
 
 void DataTable::LocalAppend(LocalAppendState &state, TableCatalogEntry &table, ClientContext &context, DataChunk &chunk,
@@ -662,14 +660,19 @@ void DataTable::FinalizeLocalAppend(LocalAppendState &state) {
 	LocalStorage::FinalizeAppend(state);
 }
 
-OptimisticDataWriter *DataTable::CreateOptimisticWriter(ClientContext &context) {
+OptimisticDataWriter &DataTable::CreateOptimisticWriter(ClientContext &context) {
 	auto &local_storage = LocalStorage::Get(context, db);
-	return local_storage.CreateOptimisticWriter(this);
+	return local_storage.CreateOptimisticWriter(*this);
+}
+
+void DataTable::FinalizeOptimisticWriter(ClientContext &context, OptimisticDataWriter &writer) {
+	auto &local_storage = LocalStorage::Get(context, db);
+	local_storage.FinalizeOptimisticWriter(*this, writer);
 }
 
 void DataTable::LocalMerge(ClientContext &context, RowGroupCollection &collection) {
 	auto &local_storage = LocalStorage::Get(context, db);
-	local_storage.LocalMerge(this, collection);
+	local_storage.LocalMerge(*this, collection);
 }
 
 void DataTable::LocalAppend(TableCatalogEntry &table, ClientContext &context, DataChunk &chunk) {
@@ -728,8 +731,7 @@ void DataTable::ScanTableSegment(idx_t row_start, idx_t count, const std::functi
 	CreateIndexScanState state;
 
 	InitializeScanWithOffset(state, column_ids, row_start, row_start + count);
-	auto row_start_aligned = state.table_state.row_group_state.row_group->start +
-	                         state.table_state.row_group_state.vector_index * STANDARD_VECTOR_SIZE;
+	auto row_start_aligned = state.table_state.row_group->start + state.table_state.vector_index * STANDARD_VECTOR_SIZE;
 
 	idx_t current_row = row_start_aligned;
 	while (current_row < end) {
@@ -819,9 +821,10 @@ void DataTable::RevertAppend(idx_t start_row, idx_t count) {
 //===--------------------------------------------------------------------===//
 // Indexes
 //===--------------------------------------------------------------------===//
-bool DataTable::AppendToIndexes(TableIndexList &indexes, DataChunk &chunk, row_t row_start) {
+PreservedError DataTable::AppendToIndexes(TableIndexList &indexes, DataChunk &chunk, row_t row_start) {
+	PreservedError error;
 	if (indexes.Empty()) {
-		return true;
+		return error;
 	}
 	// first generate the vector of row identifiers
 	Vector row_identifiers(LogicalType::ROW_TYPE);
@@ -832,11 +835,13 @@ bool DataTable::AppendToIndexes(TableIndexList &indexes, DataChunk &chunk, row_t
 	// now append the entries to the indices
 	indexes.Scan([&](Index &index) {
 		try {
-			if (!index.Append(chunk, row_identifiers)) {
-				append_failed = true;
-				return true;
-			}
-		} catch (...) {
+			error = index.Append(chunk, row_identifiers);
+		} catch (Exception &ex) {
+			error = PreservedError(ex);
+		} catch (std::exception &ex) {
+			error = PreservedError(ex);
+		}
+		if (error) {
 			append_failed = true;
 			return true;
 		}
@@ -850,12 +855,11 @@ bool DataTable::AppendToIndexes(TableIndexList &indexes, DataChunk &chunk, row_t
 		for (auto *index : already_appended) {
 			index->Delete(chunk, row_identifiers);
 		}
-		return false;
 	}
-	return true;
+	return error;
 }
 
-bool DataTable::AppendToIndexes(DataChunk &chunk, row_t row_start) {
+PreservedError DataTable::AppendToIndexes(DataChunk &chunk, row_t row_start) {
 	D_ASSERT(is_root);
 	return AppendToIndexes(info->indexes, chunk, row_start);
 }
@@ -979,10 +983,10 @@ idx_t DataTable::Delete(TableCatalogEntry &table, ClientContext &context, Vector
 			// transaction-local delete
 			if (has_delete_constraints) {
 				// perform the constraint verification
-				local_storage.FetchChunk(this, offset_ids, current_count, col_ids, verify_chunk, fetch_state);
+				local_storage.FetchChunk(*this, offset_ids, current_count, col_ids, verify_chunk, fetch_state);
 				VerifyDeleteConstraints(table, context, verify_chunk);
 			}
-			delete_count += local_storage.Delete(this, offset_ids, current_count);
+			delete_count += local_storage.Delete(*this, offset_ids, current_count);
 		} else {
 			// regular table delete
 			if (has_delete_constraints) {
@@ -990,7 +994,7 @@ idx_t DataTable::Delete(TableCatalogEntry &table, ClientContext &context, Vector
 				Fetch(transaction, verify_chunk, col_ids, offset_ids, current_count, fetch_state);
 				VerifyDeleteConstraints(table, context, verify_chunk);
 			}
-			delete_count += row_groups->Delete(transaction, this, ids + current_offset, current_count);
+			delete_count += row_groups->Delete(transaction, *this, ids + current_offset, current_count);
 		}
 	}
 	return delete_count;
@@ -1111,7 +1115,7 @@ void DataTable::Update(TableCatalogEntry &table, ClientContext &context, Vector
 	if (first_id >= MAX_ROW_ID) {
 		// update is in transaction-local storage: push update into local storage
 		auto &local_storage = LocalStorage::Get(context, db);
-		local_storage.Update(this, row_ids, column_ids, updates);
+		local_storage.Update(*this, row_ids, column_ids, updates);
 		return;
 	}
 
@@ -1163,9 +1167,7 @@ void DataTable::WALAddIndex(ClientContext &context, unique_ptr<Index> index,
 
 	auto &allocator = Allocator::Get(db);
 
-	DataChunk result;
-	result.Initialize(allocator, index->logical_types);
-
+	// intermediate holds scanned chunks of the underlying data to create the index
 	DataChunk intermediate;
 	vector<LogicalType> intermediate_types;
 	auto column_ids = index->column_ids;
@@ -1177,6 +1179,10 @@ void DataTable::WALAddIndex(ClientContext &context, unique_ptr<Index> index,
 	intermediate_types.emplace_back(LogicalType::ROW_TYPE);
 	intermediate.Initialize(allocator, intermediate_types);
 
+	// holds the result of executing the index expression on the intermediate chunks
+	DataChunk result;
+	result.Initialize(allocator, index->logical_types);
+
 	// initialize an index scan
 	CreateIndexScanState state;
 	InitializeWALCreateIndexScan(state, column_ids);
@@ -1204,12 +1210,13 @@ void DataTable::WALAddIndex(ClientContext &context, unique_ptr<Index> index,
 			index->ExecuteExpressions(intermediate, result);
 
 			// insert into the index
-			if (!index->Insert(lock, result, intermediate.data[intermediate.ColumnCount() - 1])) {
-				throw InternalException("Error during WAL replay. Can't create unique index, table contains "
-				                        "duplicate data on indexed column(s).");
+			auto error = index->Insert(lock, result, intermediate.data[intermediate.ColumnCount() - 1]);
+			if (error) {
+				throw InternalException("Error during WAL replay: %s", error.Message());
 			}
 		}
 	}
+
 	info->indexes.AddIndex(std::move(index));
 }