duckdb 0.8.2-dev4314.0 → 0.8.2-dev4424.0

package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp

@@ -36,11 +36,12 @@ class Vector;
 struct ColumnCheckpointState;
 struct RowGroupPointer;
 struct TransactionData;
-struct VersionNode;
 class CollectionScanState;
 class TableFilterSet;
 struct ColumnFetchState;
 struct RowGroupAppendState;
+class MetadataManager;
+class RowVersionManager;
 
 struct RowGroupWriteData {
 	vector<unique_ptr<ColumnCheckpointState>> states;
@@ -50,11 +51,6 @@ struct RowGroupWriteData {
 class RowGroup : public SegmentBase<RowGroup> {
 public:
 	friend class ColumnData;
-	friend class VersionDeleteState;
-
-public:
-	static constexpr const idx_t ROW_GROUP_SIZE = STANDARD_ROW_GROUPS_SIZE;
-	static constexpr const idx_t ROW_GROUP_VECTOR_COUNT = ROW_GROUP_SIZE / STANDARD_VECTOR_SIZE;
 
 public:
 	RowGroup(RowGroupCollection &collection, idx_t start, idx_t count);
@@ -65,7 +61,7 @@ private:
 	//! The RowGroupCollection this row-group is a part of
 	reference<RowGroupCollection> collection;
 	//! The version info of the row_group (inserted and deleted tuple info)
-	shared_ptr<VersionNode> version_info;
+	shared_ptr<RowVersionManager> version_info;
 	//! The column data of the row_group
 	vector<shared_ptr<ColumnData>> columns;
 
@@ -145,12 +141,17 @@ public:
 
 	void NextVector(CollectionScanState &state);
 
+	idx_t DeleteRows(idx_t vector_idx, transaction_t transaction_id, row_t rows[], idx_t count);
+	RowVersionManager &GetOrCreateVersionInfo();
+
 	// Serialization
 	static void Serialize(RowGroupPointer &pointer, Serializer &serializer);
 	static RowGroupPointer Deserialize(Deserializer &deserializer);
 
 private:
-	ChunkInfo *GetChunkInfo(idx_t vector_idx);
+	shared_ptr<RowVersionManager> &GetVersionInfo();
+	shared_ptr<RowVersionManager> &GetOrCreateVersionInfoPtr();
+
 	ColumnData &GetColumn(storage_t c);
 	idx_t GetColumnCount() const;
 	vector<shared_ptr<ColumnData>> &GetColumns();
@@ -158,18 +159,17 @@ private:
 	template <TableScanType TYPE>
 	void TemplatedScan(TransactionData transaction, CollectionScanState &state, DataChunk &result);
 
+	vector<MetaBlockPointer> CheckpointDeletes(MetadataManager &manager);
+
+	bool HasUnloadedDeletes() const;
+
 private:
 	mutex row_group_lock;
 	mutex stats_lock;
 	vector<MetaBlockPointer> column_pointers;
 	unique_ptr<atomic<bool>[]> is_loaded;
-};
-
-struct VersionNode {
-	unique_ptr<ChunkInfo> info[RowGroup::ROW_GROUP_VECTOR_COUNT];
-
-	void SetStart(idx_t start);
-	idx_t GetCommittedDeletedCount(idx_t count);
+	vector<MetaBlockPointer> deletes_pointers;
+	atomic<bool> deletes_is_loaded;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/row_version_manager.hpp

@@ -0,0 +1,59 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/table/row_version_manager.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/vector_size.hpp"
+#include "duckdb/storage/table/chunk_info.hpp"
+#include "duckdb/storage/storage_info.hpp"
+#include "duckdb/common/mutex.hpp"
+
+namespace duckdb {
+
+class MetadataManager;
+struct MetaBlockPointer;
+
+class RowVersionManager {
+public:
+	explicit RowVersionManager(idx_t start);
+
+	idx_t GetStart() {
+		return start;
+	}
+	void SetStart(idx_t start);
+	idx_t GetCommittedDeletedCount(idx_t count);
+
+	idx_t GetSelVector(TransactionData transaction, idx_t vector_idx, SelectionVector &sel_vector, idx_t max_count);
+	idx_t GetCommittedSelVector(transaction_t start_time, transaction_t transaction_id, idx_t vector_idx,
+	                            SelectionVector &sel_vector, idx_t max_count);
+	bool Fetch(TransactionData transaction, idx_t row);
+
+	void AppendVersionInfo(TransactionData transaction, idx_t count, idx_t row_group_start, idx_t row_group_end);
+	void CommitAppend(transaction_t commit_id, idx_t row_group_start, idx_t count);
+	void RevertAppend(idx_t start_row);
+
+	idx_t DeleteRows(idx_t vector_idx, transaction_t transaction_id, row_t rows[], idx_t count);
+	void CommitDelete(idx_t vector_idx, transaction_t commit_id, row_t rows[], idx_t count);
+
+	vector<MetaBlockPointer> Checkpoint(MetadataManager &manager);
+	static shared_ptr<RowVersionManager> Deserialize(MetaBlockPointer delete_pointer, MetadataManager &manager,
+	                                                 idx_t start);
+
+private:
+	mutex version_lock;
+	idx_t start;
+	unique_ptr<ChunkInfo> vector_info[Storage::ROW_GROUP_VECTOR_COUNT];
+	bool has_changes;
+	vector<MetaBlockPointer> storage_pointers;
+
+private:
+	optional_ptr<ChunkInfo> GetChunkInfo(idx_t vector_idx);
+	ChunkVectorInfo &GetVectorInfo(idx_t vector_idx);
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/update_segment.hpp

@@ -101,7 +101,7 @@ struct UpdateNodeData {
 };
 
 struct UpdateNode {
-	unique_ptr<UpdateNodeData> info[RowGroup::ROW_GROUP_VECTOR_COUNT];
+	unique_ptr<UpdateNodeData> info[Storage::ROW_GROUP_VECTOR_COUNT];
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/transaction/commit_state.hpp

@@ -23,7 +23,7 @@ struct UpdateInfo;
 
 class CommitState {
 public:
-	explicit CommitState(ClientContext &context, transaction_t commit_id, optional_ptr<WriteAheadLog> log = nullptr);
+	explicit CommitState(transaction_t commit_id, optional_ptr<WriteAheadLog> log = nullptr);
 
 	optional_ptr<WriteAheadLog> log;
 	transaction_t commit_id;
@@ -35,9 +35,6 @@ public:
 	unique_ptr<DataChunk> delete_chunk;
 	unique_ptr<DataChunk> update_chunk;
 
-private:
-	ClientContext &context;
-
 public:
 	template <bool HAS_LOG>
 	void CommitEntry(UndoFlags type, data_ptr_t data);
@@ -49,8 +46,6 @@ private:
 	void WriteCatalogEntry(CatalogEntry &entry, data_ptr_t extra_data);
 	void WriteDelete(DeleteInfo &info);
 	void WriteUpdate(UpdateInfo &info);
-
-	void AppendRowId(row_t rowid);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/transaction/delete_info.hpp

@@ -11,12 +11,13 @@
 #include "duckdb/common/constants.hpp"
 
 namespace duckdb {
-class ChunkVectorInfo;
 class DataTable;
+class RowVersionManager;
 
 struct DeleteInfo {
 	DataTable *table;
-	ChunkVectorInfo *vinfo;
+	RowVersionManager *version_info;
+	idx_t vector_idx;
 	idx_t count;
 	idx_t base_row;
 	row_t rows[1];

package/src/duckdb/src/include/duckdb/transaction/duck_transaction.hpp

@@ -11,12 +11,13 @@
 #include "duckdb/transaction/transaction.hpp"
 
 namespace duckdb {
+class RowVersionManager;
 
 class DuckTransaction : public Transaction {
 public:
 	DuckTransaction(TransactionManager &manager, ClientContext &context, transaction_t start_time,
 	                transaction_t transaction_id);
-	~DuckTransaction();
+	~DuckTransaction() override;
 
 	//! The start timestamp of this transaction
 	transaction_t start_time;
@@ -49,7 +50,8 @@ public:
 
 	bool ChangesMade();
 
-	void PushDelete(DataTable &table, ChunkVectorInfo *vinfo, row_t rows[], idx_t count, idx_t base_row);
+	void PushDelete(DataTable &table, RowVersionManager &info, idx_t vector_idx, row_t rows[], idx_t count,
+	                idx_t base_row);
 	void PushAppend(DataTable &table, idx_t row_start, idx_t row_count);
 	UpdateInfo *CreateUpdateInfo(idx_t type_size, idx_t entries);
 

package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp

@@ -88,7 +88,7 @@ private:
 class LocalStorage {
 public:
 	// Threshold to merge row groups instead of appending
-	static constexpr const idx_t MERGE_THRESHOLD = RowGroup::ROW_GROUP_SIZE;
+	static constexpr const idx_t MERGE_THRESHOLD = Storage::ROW_GROUP_SIZE;
 
 public:
 	struct CommitState {

package/src/duckdb/src/include/duckdb/transaction/undo_buffer.hpp

@@ -48,7 +48,6 @@ public:
 	void Rollback() noexcept;
 
 private:
-	ClientContext &context;
 	ArenaAllocator allocator;
 
 private:

package/src/duckdb/src/main/settings/settings.cpp

@@ -726,17 +726,12 @@ Value ForceCompressionSetting::GetSetting(ClientContext &context) {
 //===--------------------------------------------------------------------===//
 void ForceBitpackingModeSetting::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) {
 	auto mode_str = StringUtil::Lower(input.ToString());
-	if (mode_str == "none") {
-		config.options.force_bitpacking_mode = BitpackingMode::AUTO;
-	} else {
-		auto mode = BitpackingModeFromString(mode_str);
-		if (mode == BitpackingMode::AUTO) {
-			throw ParserException(
-			    "Unrecognized option for force_bitpacking_mode, expected none, constant, constant_delta, "
-			    "delta_for, or for");
-		}
-		config.options.force_bitpacking_mode = mode;
+	auto mode = BitpackingModeFromString(mode_str);
+	if (mode == BitpackingMode::INVALID) {
+		throw ParserException("Unrecognized option for force_bitpacking_mode, expected none, constant, constant_delta, "
+		                      "delta_for, or for");
 	}
+	config.options.force_bitpacking_mode = mode;
 }
 
 void ForceBitpackingModeSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) {

package/src/duckdb/src/optimizer/statistics/expression/propagate_cast.cpp

@@ -22,6 +22,20 @@ static unique_ptr<BaseStatistics> StatisticsOperationsNumericNumericCast(const B
 }
 
 static unique_ptr<BaseStatistics> StatisticsNumericCastSwitch(const BaseStatistics &input, const LogicalType &target) {
+	//	Downcasting timestamps to times is not a truncation operation
+	switch (target.id()) {
+	case LogicalTypeId::TIME:
+		switch (input.GetType().id()) {
+		case LogicalTypeId::TIMESTAMP:
+		case LogicalTypeId::TIMESTAMP_TZ:
+			return nullptr;
+		default:
+			break;
+		}
+	default:
+		break;
+	}
+
 	switch (target.InternalType()) {
 	case PhysicalType::INT8:
 	case PhysicalType::INT16:

package/src/duckdb/src/storage/checkpoint/table_data_writer.cpp

@@ -66,7 +66,6 @@ void SingleFileTableDataWriter::FinalizeTable(TableStatistics &&global_stats, Da
 		row_group_serializer.End();
 	}
 
-	// TODO: Serialize this:
 	auto index_pointers = info->indexes.SerializeIndexes(table_data_writer);
 
 	// Now begin the metadata as a unit

package/src/duckdb/src/storage/checkpoint_manager.cpp

@@ -363,63 +363,64 @@ void CheckpointWriter::WriteIndex(IndexCatalogEntry &index_catalog, Serializer &
 
 void CheckpointReader::ReadIndex(ClientContext &context, Deserializer &deserializer) {
 
-	// Deserialize the index metadata
-	auto info = deserializer.ReadProperty<unique_ptr<CreateInfo>>(100, "index");
-	auto &index_info = info->Cast<CreateIndexInfo>();
-
-	// Create the index in the catalog
-	auto &schema_catalog = catalog.GetSchema(context, info->schema);
-	auto &table_catalog =
-	    catalog.GetEntry(context, CatalogType::TABLE_ENTRY, info->schema, index_info.table).Cast<DuckTableEntry>();
-	auto &index_catalog = schema_catalog.CreateIndex(context, index_info, table_catalog)->Cast<DuckIndexEntry>();
-	index_catalog.info = table_catalog.GetStorage().info;
-
-	// We deserialize the index lazily, i.e., we do not need to load any node information
+	// deserialize the index create info
+	auto create_info = deserializer.ReadProperty<unique_ptr<CreateInfo>>(100, "index");
+	auto &info = create_info->Cast<CreateIndexInfo>();
+
+	// create the index in the catalog
+	auto &schema = catalog.GetSchema(context, create_info->schema);
+	auto &table =
+	    catalog.GetEntry(context, CatalogType::TABLE_ENTRY, create_info->schema, info.table).Cast<DuckTableEntry>();
+
+	auto &index = schema.CreateIndex(context, info, table)->Cast<DuckIndexEntry>();
+
+	index.info = table.GetStorage().info;
+	// insert the parsed expressions into the stored index so that we correctly (de)serialize it during consecutive
+	// checkpoints
+	for (auto &parsed_expr : info.parsed_expressions) {
+		index.parsed_expressions.push_back(parsed_expr->Copy());
+	}
+
+	// we deserialize the index lazily, i.e., we do not need to load any node information
 	// except the root block pointer
-	auto index_block_pointer = deserializer.ReadProperty<BlockPointer>(101, "root_block_pointer");
+	auto root_block_pointer = deserializer.ReadProperty<BlockPointer>(101, "root_block_pointer");
 
-	// obtain the expressions of the ART from the index metadata
-	vector<unique_ptr<Expression>> unbound_expressions;
+	// obtain the parsed expressions of the ART from the index metadata
 	vector<unique_ptr<ParsedExpression>> parsed_expressions;
-	for (auto &p_exp : index_info.parsed_expressions) {
-		parsed_expressions.push_back(p_exp->Copy());
+	for (auto &parsed_expr : info.parsed_expressions) {
+		parsed_expressions.push_back(parsed_expr->Copy());
 	}
+	D_ASSERT(!parsed_expressions.empty());
 
-	// bind the parsed expressions
-	// add the table to the bind context
+	// add the table to the bind context to bind the parsed expressions
 	auto binder = Binder::CreateBinder(context);
 	vector<LogicalType> column_types;
 	vector<string> column_names;
-	for (auto &col : table_catalog.GetColumns().Logical()) {
+	for (auto &col : table.GetColumns().Logical()) {
 		column_types.push_back(col.Type());
 		column_names.push_back(col.Name());
 	}
+
+	// create a binder to bind the parsed expressions
 	vector<column_t> column_ids;
-	binder->bind_context.AddBaseTable(0, index_info.table, column_names, column_types, column_ids, &table_catalog);
+	binder->bind_context.AddBaseTable(0, info.table, column_names, column_types, column_ids, &table);
 	IndexBinder idx_binder(*binder, context);
+
+	// bind the parsed expressions to create unbound expressions
+	vector<unique_ptr<Expression>> unbound_expressions;
 	unbound_expressions.reserve(parsed_expressions.size());
 	for (auto &expr : parsed_expressions) {
 		unbound_expressions.push_back(idx_binder.Bind(expr));
 	}
 
-	if (parsed_expressions.empty()) {
-		// this is a PK/FK index: we create the necessary bound column ref expressions
-		unbound_expressions.reserve(index_info.column_ids.size());
-		for (idx_t key_nr = 0; key_nr < index_info.column_ids.size(); key_nr++) {
-			auto &col = table_catalog.GetColumn(LogicalIndex(index_info.column_ids[key_nr]));
-			unbound_expressions.push_back(
-			    make_uniq<BoundColumnRefExpression>(col.GetName(), col.GetType(), ColumnBinding(0, key_nr)));
-		}
-	}
-
 	// create the index and add it to the storage
-	switch (index_info.index_type) {
+	switch (info.index_type) {
 	case IndexType::ART: {
-		auto &storage = table_catalog.GetStorage();
-		auto art = make_uniq<ART>(index_info.column_ids, TableIOManager::Get(storage), std::move(unbound_expressions),
-		                          index_info.constraint_type, storage.db, nullptr, index_block_pointer);
+		auto &storage = table.GetStorage();
+		auto art = make_uniq<ART>(info.column_ids, TableIOManager::Get(storage), std::move(unbound_expressions),
+		                          info.constraint_type, storage.db, nullptr, root_block_pointer);
 
-		index_catalog.index = art.get();
+		index.index = art.get();
 		storage.info->indexes.AddIndex(std::move(art));
 	} break;
 	default:

package/src/duckdb/src/storage/compression/bitpacking.cpp

@@ -23,8 +23,7 @@ static constexpr const idx_t BITPACKING_METADATA_GROUP_SIZE = STANDARD_VECTOR_SI
 
 BitpackingMode BitpackingModeFromString(const string &str) {
 	auto mode = StringUtil::Lower(str);
-
-	if (mode == "auto") {
+	if (mode == "auto" || mode == "none") {
 		return BitpackingMode::AUTO;
 	} else if (mode == "constant") {
 		return BitpackingMode::CONSTANT;
@@ -35,21 +34,21 @@ BitpackingMode BitpackingModeFromString(const string &str) {
 	} else if (mode == "for") {
 		return BitpackingMode::FOR;
 	} else {
-		return BitpackingMode::AUTO;
+		return BitpackingMode::INVALID;
 	}
 }
 
 string BitpackingModeToString(const BitpackingMode &mode) {
 	switch (mode) {
-	case (BitpackingMode::AUTO):
+	case BitpackingMode::AUTO:
 		return "auto";
-	case (BitpackingMode::CONSTANT):
+	case BitpackingMode::CONSTANT:
 		return "constant";
-	case (BitpackingMode::CONSTANT_DELTA):
+	case BitpackingMode::CONSTANT_DELTA:
 		return "constant_delta";
-	case (BitpackingMode::DELTA_FOR):
+	case BitpackingMode::DELTA_FOR:
 		return "delta_for";
-	case (BitpackingMode::FOR):
+	case BitpackingMode::FOR:
 		return "for";
 	default:
 		throw NotImplementedException("Unknown bitpacking mode: " + to_string((uint8_t)mode) + "\n");
@@ -161,7 +160,7 @@ public:
 		// Don't delta encoding 1 value makes no sense
 		if (compression_buffer_idx < 2) {
 			return;
-		};
+		}
 
 		// TODO: handle NULLS here?
 		// Currently we cannot handle nulls because we would need an additional step of patching for this.
@@ -686,48 +685,57 @@ public:
 	}
 
 	void Skip(ColumnSegment &segment, idx_t skip_count) {
-		while (skip_count > 0) {
-			if (current_group_offset + skip_count < BITPACKING_METADATA_GROUP_SIZE) {
-				// Skipping Delta FOR requires a bit of decoding to figure out the new delta
-				if (current_group.mode == BitpackingMode::DELTA_FOR) {
-					// if current_group_offset points into the middle of a
-					// BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE, we need to scan a few
-					// values before current_group_offset to align with the algorithm groups
-					idx_t extra_count = current_group_offset % BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE;
-
-					// Calculate total offset and count to bitunpack
-					idx_t base_decompress_count = BitpackingPrimitives::RoundUpToAlgorithmGroupSize(skip_count);
-					idx_t decompress_count = base_decompress_count + extra_count;
-					idx_t decompress_offset = current_group_offset - extra_count;
-					bool skip_sign_extension = true;
-
-					BitpackingPrimitives::UnPackBuffer<T>(data_ptr_cast(decompression_buffer),
-					                                      current_group_ptr + decompress_offset, decompress_count,
-					                                      current_width, skip_sign_extension);
-
-					ApplyFrameOfReference<T_S>(reinterpret_cast<T_S *>(decompression_buffer + extra_count),
-					                           current_frame_of_reference, skip_count);
-					DeltaDecode<T_S>(reinterpret_cast<T_S *>(decompression_buffer + extra_count),
-					                 static_cast<T_S>(current_delta_offset), skip_count);
-					current_delta_offset = decompression_buffer[extra_count + skip_count - 1];
-
-					current_group_offset += skip_count;
-				} else {
-					current_group_offset += skip_count;
-				}
-				break;
-			} else {
-				auto left_in_this_group = BITPACKING_METADATA_GROUP_SIZE - current_group_offset;
-				auto number_of_groups_to_skip = (skip_count - left_in_this_group) / BITPACKING_METADATA_GROUP_SIZE;
-
-				current_group_offset = 0;
-				bitpacking_metadata_ptr -= number_of_groups_to_skip * sizeof(bitpacking_metadata_encoded_t);
+		bool skip_sign_extend = true;
 
+		idx_t skipped = 0;
+		while (skipped < skip_count) {
+			// Exhausted this metadata group, move pointers to next group and load metadata for next group.
+			if (current_group_offset >= BITPACKING_METADATA_GROUP_SIZE) {
 				LoadNextGroup();
+			}
 
-				skip_count -= left_in_this_group;
-				skip_count -= number_of_groups_to_skip * BITPACKING_METADATA_GROUP_SIZE;
+			idx_t offset_in_compression_group =
+			    current_group_offset % BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE;
+
+			if (current_group.mode == BitpackingMode::CONSTANT) {
+				idx_t remaining = skip_count - skipped;
+				idx_t to_skip = MinValue(remaining, BITPACKING_METADATA_GROUP_SIZE - current_group_offset);
+				skipped += to_skip;
+				current_group_offset += to_skip;
+				continue;
+			}
+			if (current_group.mode == BitpackingMode::CONSTANT_DELTA) {
+				idx_t remaining = skip_count - skipped;
+				idx_t to_skip = MinValue(remaining, BITPACKING_METADATA_GROUP_SIZE - current_group_offset);
+				skipped += to_skip;
+				current_group_offset += to_skip;
+				continue;
+			}
+			D_ASSERT(current_group.mode == BitpackingMode::FOR || current_group.mode == BitpackingMode::DELTA_FOR);
+
+			idx_t to_skip =
+			    MinValue<idx_t>(skip_count - skipped,
+			                    BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE - offset_in_compression_group);
+			// Calculate start of compression algorithm group
+			if (current_group.mode == BitpackingMode::DELTA_FOR) {
+				data_ptr_t current_position_ptr = current_group_ptr + current_group_offset * current_width / 8;
+				data_ptr_t decompression_group_start_pointer =
+				    current_position_ptr - offset_in_compression_group * current_width / 8;
+
+				BitpackingPrimitives::UnPackBlock<T>(data_ptr_cast(decompression_buffer),
+				                                     decompression_group_start_pointer, current_width,
+				                                     skip_sign_extend);
+
+				T *decompression_ptr = decompression_buffer + offset_in_compression_group;
+				ApplyFrameOfReference<T_S>(reinterpret_cast<T_S *>(decompression_ptr),
+				                           static_cast<T_S>(current_frame_of_reference), to_skip);
+				DeltaDecode<T_S>(reinterpret_cast<T_S *>(decompression_ptr), static_cast<T_S>(current_delta_offset),
+				                 to_skip);
+				current_delta_offset = decompression_ptr[to_skip - 1];
 			}
+
+			skipped += to_skip;
+			current_group_offset += to_skip;
 		}
 	}
 
@@ -757,7 +765,6 @@ void BitpackingScanPartial(ColumnSegment &segment, ColumnScanState &state, idx_t
 	bool skip_sign_extend = true;
 
 	idx_t scanned = 0;
-
 	while (scanned < scan_count) {
 		// Exhausted this metadata group, move pointers to next group and load metadata for next group.
 		if (scan_state.current_group_offset >= BITPACKING_METADATA_GROUP_SIZE) {

package/src/duckdb/src/storage/data_table.cpp

@@ -208,7 +208,7 @@ void DataTable::InitializeScanWithOffset(TableScanState &state, const vector<col
 }
 
 idx_t DataTable::MaxThreads(ClientContext &context) {
-	idx_t parallel_scan_vector_count = RowGroup::ROW_GROUP_VECTOR_COUNT;
+	idx_t parallel_scan_vector_count = Storage::ROW_GROUP_VECTOR_COUNT;
 	if (ClientConfig::GetConfig(context).verify_parallelism) {
 		parallel_scan_vector_count = 1;
 	}

package/src/duckdb/src/storage/local_storage.cpp

@@ -102,7 +102,7 @@ void LocalTableStorage::WriteNewRowGroup() {
 }
 
 void LocalTableStorage::FlushBlocks() {
-	if (!merged_storage && row_groups->GetTotalRows() > RowGroup::ROW_GROUP_SIZE) {
+	if (!merged_storage && row_groups->GetTotalRows() > Storage::ROW_GROUP_SIZE) {
 		optimistic_writer.WriteLastRowGroup(*row_groups);
 	}
 	optimistic_writer.FinalFlush();
@@ -159,7 +159,7 @@ void LocalTableStorage::AppendToIndexes(DuckTransaction &transaction, TableAppen
 		    AppendToIndexes(transaction, *row_groups, table.info->indexes, table.GetTypes(), append_state.current_row);
 	}
 	if (error) {
-		// need to revert the append
+		// need to revert all appended row ids
 		row_t current_row = append_state.row_start;
 		// remove the data from the indexes, if there are any indexes
 		row_groups->Scan(transaction, [&](DataChunk &chunk) -> bool {
@@ -184,6 +184,13 @@ void LocalTableStorage::AppendToIndexes(DuckTransaction &transaction, TableAppen
 		if (append_to_table) {
 			table.RevertAppendInternal(append_state.row_start, append_count);
 		}
+
+		// we need to vacuum the indexes to remove any buffers that are now empty
+		// due to reverting the appends
+		table.info->indexes.Scan([&](Index &index) {
+			index.Vacuum();
+			return false;
+		});
 		error.Throw();
 	}
 }

package/src/duckdb/src/storage/metadata/metadata_manager.cpp

@@ -34,6 +34,12 @@ MetadataHandle MetadataManager::AllocateHandle() {
 	MetadataPointer pointer;
 	pointer.block_index = free_block;
 	auto &block = blocks[free_block];
+	if (block.block->BlockId() < MAXIMUM_BLOCK) {
+		// this block is a disk-backed block, yet we are planning to write to it
+		// we need to convert it into a transient block before we can write to it
+		ConvertToTransient(block);
+		D_ASSERT(block.block->BlockId() >= MAXIMUM_BLOCK);
+	}
 	D_ASSERT(!block.free_blocks.empty());
 	pointer.index = block.free_blocks.back();
 	// mark the block as used
@@ -54,6 +60,23 @@ MetadataHandle MetadataManager::Pin(MetadataPointer pointer) {
 	return handle;
 }
 
+void MetadataManager::ConvertToTransient(MetadataBlock &block) {
+	// pin the old block
+	auto old_buffer = buffer_manager.Pin(block.block);
+
+	// allocate a new transient block to replace it
+	shared_ptr<BlockHandle> new_block;
+	auto new_buffer = buffer_manager.Allocate(Storage::BLOCK_SIZE, false, &new_block);
+
+	// copy the data to the transient block
+	memcpy(new_buffer.Ptr(), old_buffer.Ptr(), Storage::BLOCK_SIZE);
+
+	block.block = std::move(new_block);
+
+	// unregister the old block
+	block_manager.UnregisterBlock(block.block_id, false);
+}
+
 block_id_t MetadataManager::AllocateNewBlock() {
 	auto new_block_id = GetNextBlockId();
 
@@ -91,11 +114,11 @@ MetaBlockPointer MetadataManager::GetDiskPointer(MetadataPointer pointer, uint32
 	return MetaBlockPointer(block_pointer, offset);
 }
 
-block_id_t MetaBlockPointer::GetBlockId() {
+block_id_t MetaBlockPointer::GetBlockId() const {
 	return block_id_t(block_pointer & ~(idx_t(0xFF) << 56ULL));
 }
 
-uint32_t MetaBlockPointer::GetBlockIndex() {
+uint32_t MetaBlockPointer::GetBlockIndex() const {
 	return block_pointer >> 56ULL;
 }
 
@@ -262,6 +285,22 @@ void MetadataManager::MarkBlocksAsModified() {
 	}
 }
 
+void MetadataManager::ClearModifiedBlocks(const vector<MetaBlockPointer> &pointers) {
+	for (auto &pointer : pointers) {
+		auto block_id = pointer.GetBlockId();
+		auto block_index = pointer.GetBlockIndex();
+		auto entry = modified_blocks.find(block_id);
+		if (entry == modified_blocks.end()) {
+			throw InternalException("ClearModifiedBlocks - Block id %llu not found in modified_blocks", block_id);
+		}
+		auto &modified_list = entry->second;
+		// verify the block has been modified
+		D_ASSERT(modified_list && (1ULL << block_index));
+		// unset the bit
+		modified_list &= ~(1ULL << block_index);
+	}
+}
+
 block_id_t MetadataManager::GetNextBlockId() {
 	return block_manager.GetFreeBlockId();
 }