duckdb 0.8.2-dev4314.0 → 0.8.2-dev4376.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.8.2-dev4314.0",
+  "version": "0.8.2-dev4376.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/extension/parquet/parquet_extension.cpp

@@ -118,7 +118,7 @@ struct ParquetWriteBindData : public TableFunctionData {
 	vector<LogicalType> sql_types;
 	vector<string> column_names;
 	duckdb_parquet::format::CompressionCodec::type codec = duckdb_parquet::format::CompressionCodec::SNAPPY;
-	idx_t row_group_size = RowGroup::ROW_GROUP_SIZE;
+	idx_t row_group_size = Storage::ROW_GROUP_SIZE;
 
 	//! If row_group_size_bytes is not set, we default to row_group_size * BYTES_PER_ROW
 	static constexpr const idx_t BYTES_PER_ROW = 1024;

package/src/duckdb/src/common/enum_util.cpp

@@ -551,6 +551,8 @@ BindingMode EnumUtil::FromString<BindingMode>(const char *value) {
 template<>
 const char* EnumUtil::ToChars<BitpackingMode>(BitpackingMode value) {
 	switch(value) {
+	case BitpackingMode::INVALID:
+		return "INVALID";
 	case BitpackingMode::AUTO:
 		return "AUTO";
 	case BitpackingMode::CONSTANT:
@@ -568,6 +570,9 @@ const char* EnumUtil::ToChars<BitpackingMode>(BitpackingMode value) {
 
 template<>
 BitpackingMode EnumUtil::FromString<BitpackingMode>(const char *value) {
+	if (StringUtil::Equals(value, "INVALID")) {
+		return BitpackingMode::INVALID;
+	}
 	if (StringUtil::Equals(value, "AUTO")) {
 		return BitpackingMode::AUTO;
 	}

package/src/duckdb/src/common/file_buffer.cpp

@@ -5,7 +5,7 @@
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/file_system.hpp"
 #include "duckdb/common/helper.hpp"
-
+#include "duckdb/storage/storage_info.hpp"
 #include <cstring>
 
 namespace duckdb {

package/src/duckdb/src/common/types/validity_mask.cpp

@@ -1,4 +1,7 @@
 #include "duckdb/common/types/validity_mask.hpp"
+#include "duckdb/common/limits.hpp"
+#include "duckdb/common/serializer/write_stream.hpp"
+#include "duckdb/common/serializer/read_stream.hpp"
 
 namespace duckdb {
 
@@ -173,4 +176,57 @@ void ValidityMask::SliceInPlace(const ValidityMask &other, idx_t target_offset,
 #endif
 }
 
+enum class ValiditySerialization : uint8_t { BITMASK = 0, VALID_VALUES = 1, INVALID_VALUES = 2 };
+
+void ValidityMask::Write(WriteStream &writer, idx_t count) {
+	auto valid_values = CountValid(count);
+	auto invalid_values = count - valid_values;
+	auto bitmask_bytes = ValidityMask::ValidityMaskSize(count);
+	auto need_u32 = count >= NumericLimits<uint16_t>::Maximum();
+	auto bytes_per_value = need_u32 ? sizeof(uint32_t) : sizeof(uint16_t);
+	auto valid_value_size = bytes_per_value * valid_values + sizeof(uint32_t);
+	auto invalid_value_size = bytes_per_value * invalid_values + sizeof(uint32_t);
+	if (valid_value_size < bitmask_bytes || invalid_value_size < bitmask_bytes) {
+		auto serialize_valid = valid_value_size < invalid_value_size;
+		// serialize (in)valid value indexes as [COUNT][V0][V1][...][VN]
+		auto flag = serialize_valid ? ValiditySerialization::VALID_VALUES : ValiditySerialization::INVALID_VALUES;
+		writer.Write(flag);
+		writer.Write<uint32_t>(MinValue<uint32_t>(valid_values, invalid_values));
+		for (idx_t i = 0; i < count; i++) {
+			if (RowIsValid(i) == serialize_valid) {
+				if (need_u32) {
+					writer.Write<uint32_t>(i);
+				} else {
+					writer.Write<uint16_t>(i);
+				}
+			}
+		}
+	} else {
+		// serialize the entire bitmask
+		writer.Write(ValiditySerialization::BITMASK);
+		writer.WriteData(const_data_ptr_cast(GetData()), bitmask_bytes);
+	}
+}
+
+void ValidityMask::Read(ReadStream &reader, idx_t count) {
+	Initialize(count);
+	// deserialize the storage type
+	auto flag = reader.Read<ValiditySerialization>();
+	if (flag == ValiditySerialization::BITMASK) {
+		// deserialize the bitmask
+		reader.ReadData(data_ptr_cast(GetData()), ValidityMask::ValidityMaskSize(count));
+		return;
+	}
+	auto is_u32 = count >= NumericLimits<uint16_t>::Maximum();
+	auto is_valid = flag == ValiditySerialization::VALID_VALUES;
+	auto serialize_count = reader.Read<uint32_t>();
+	if (is_valid) {
+		SetAllInvalid(count);
+	}
+	for (idx_t i = 0; i < serialize_count; i++) {
+		idx_t index = is_u32 ? reader.Read<uint32_t>() : reader.Read<uint16_t>();
+		Set(index, is_valid);
+	}
+}
+
 } // namespace duckdb

package/src/duckdb/src/execution/operator/persistent/physical_batch_insert.cpp

@@ -194,7 +194,7 @@ public:
 		}
 		auto new_count = current_collection->GetTotalRows();
 		auto batch_type =
-		    new_count < RowGroup::ROW_GROUP_SIZE ? RowGroupBatchType::NOT_FLUSHED : RowGroupBatchType::FLUSHED;
+		    new_count < Storage::ROW_GROUP_SIZE ? RowGroupBatchType::NOT_FLUSHED : RowGroupBatchType::FLUSHED;
 		if (batch_type == RowGroupBatchType::FLUSHED && writer) {
 			writer->WriteLastRowGroup(*current_collection);
 		}

package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp

@@ -482,7 +482,7 @@ SinkCombineResultType PhysicalInsert::Combine(ExecutionContext &context, Operato
 
 	lock_guard<mutex> lock(gstate.lock);
 	gstate.insert_count += append_count;
-	if (append_count < RowGroup::ROW_GROUP_SIZE) {
+	if (append_count < Storage::ROW_GROUP_SIZE) {
 		// we have few rows - append to the local storage directly
 		auto &table = gstate.table;
 		auto &storage = table.GetStorage();

package/src/duckdb/src/function/table/arrow_conversion.cpp

@@ -837,7 +837,15 @@ void ArrowTableFunction::ArrowToDuckDB(ArrowScanLocalState &scan_state, const ar
 			throw InvalidInputException("arrow_scan: array length mismatch");
 		}
 		// Make sure this Vector keeps the Arrow chunk alive in case we can zero-copy the data
-		output.data[idx].GetBuffer()->SetAuxiliaryData(make_uniq<ArrowAuxiliaryData>(scan_state.chunk));
+		if (scan_state.arrow_owned_data.find(idx) == scan_state.arrow_owned_data.end()) {
+			auto arrow_data = make_shared<ArrowArrayWrapper>();
+			arrow_data->arrow_array = scan_state.chunk->arrow_array;
+			scan_state.chunk->arrow_array.release = nullptr;
+			scan_state.arrow_owned_data[idx] = arrow_data;
+		}
+
+		output.data[idx].GetBuffer()->SetAuxiliaryData(make_uniq<ArrowAuxiliaryData>(scan_state.arrow_owned_data[idx]));
+
 		D_ASSERT(arrow_convert_data.find(col_idx) != arrow_convert_data.end());
 		auto &arrow_type = *arrow_convert_data.at(col_idx);
 		if (array.dictionary) {

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.8.2-dev4314"
+#define DUCKDB_VERSION "0.8.2-dev4376"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "84a109bbee"
+#define DUCKDB_SOURCE_ID "312b995450"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/common/constants.hpp

@@ -58,21 +58,6 @@ struct DConstants {
 	static constexpr const idx_t INVALID_INDEX = idx_t(-1);
 };
 
-struct Storage {
-	//! The size of a hard disk sector, only really needed for Direct IO
-	constexpr static int SECTOR_SIZE = 4096;
-	//! Block header size for blocks written to the storage
-	constexpr static int BLOCK_HEADER_SIZE = sizeof(uint64_t);
-	// Size of a memory slot managed by the StorageManager. This is the quantum of allocation for Blocks on DuckDB. We
-	// default to 256KB. (1 << 18)
-	constexpr static int BLOCK_ALLOC_SIZE = 262144;
-	//! The actual memory space that is available within the blocks
-	constexpr static int BLOCK_SIZE = BLOCK_ALLOC_SIZE - BLOCK_HEADER_SIZE;
-	//! The size of the headers. This should be small and written more or less atomically by the hard disk. We default
-	//! to the page size, which is 4KB. (1 << 12)
-	constexpr static int FILE_HEADER_SIZE = 4096;
-};
-
 struct LogicalIndex {
 	explicit LogicalIndex(idx_t index) : index(index) {
 	}

package/src/duckdb/src/include/duckdb/common/serializer/memory_stream.hpp

@@ -1,7 +1,7 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/common/serializer/buffer_stream.hpp
+// duckdb/common/serializer/memory_stream.hpp
 //
 //
 //===----------------------------------------------------------------------===//

package/src/duckdb/src/include/duckdb/common/types/validity_mask.hpp

@@ -332,6 +332,9 @@ public:
 	DUCKDB_API string ToString(idx_t count) const;
 
 	DUCKDB_API static bool IsAligned(idx_t count);
+
+	void Write(WriteStream &writer, idx_t count);
+	void Read(ReadStream &reader, idx_t count);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/function/table/arrow.hpp

@@ -67,6 +67,9 @@ struct ArrowScanLocalState : public LocalTableFunctionState {
 
 	unique_ptr<ArrowArrayStreamWrapper> stream;
 	shared_ptr<ArrowArrayWrapper> chunk;
+	// This vector hold the Arrow Vectors owned by DuckDB to allow for zero-copy
+	// Note that only DuckDB can release these vectors
+	unordered_map<idx_t, shared_ptr<ArrowArrayWrapper>> arrow_owned_data;
 	idx_t chunk_offset = 0;
 	idx_t batch_index = 0;
 	vector<column_t> column_ids;

package/src/duckdb/src/include/duckdb/storage/block.hpp

@@ -52,11 +52,11 @@ struct MetaBlockPointer {
 	idx_t block_pointer;
 	uint32_t offset;
 
-	bool IsValid() {
+	bool IsValid() const {
 		return block_pointer != DConstants::INVALID_INDEX;
 	}
-	block_id_t GetBlockId();
-	uint32_t GetBlockIndex();
+	block_id_t GetBlockId() const;
+	uint32_t GetBlockIndex() const;
 
 	void Serialize(Serializer &serializer) const;
 	static MetaBlockPointer Deserialize(Deserializer &source);

package/src/duckdb/src/include/duckdb/storage/compression/bitpacking.hpp

@@ -12,14 +12,7 @@
 
 namespace duckdb {
 
-enum class BitpackingMode : uint8_t {
-	AUTO,
-
-	CONSTANT,
-	CONSTANT_DELTA,
-	DELTA_FOR,
-	FOR
-};
+enum class BitpackingMode : uint8_t { INVALID, AUTO, CONSTANT, CONSTANT_DELTA, DELTA_FOR, FOR };
 
 BitpackingMode BitpackingModeFromString(const string &str);
 string BitpackingModeToString(const BitpackingMode &mode);

package/src/duckdb/src/include/duckdb/storage/data_pointer.hpp

@@ -40,8 +40,8 @@ struct RowGroupPointer {
 	uint64_t tuple_count;
 	//! The data pointers of the column segments stored in the row group
 	vector<MetaBlockPointer> data_pointers;
-	//! The versions information of the row group (if any)
-	shared_ptr<VersionNode> versions;
+	//! Data pointers to the delete information of the row group (if any)
+	vector<MetaBlockPointer> deletes_pointers;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/metadata/metadata_manager.hpp

@@ -64,6 +64,7 @@ public:
 	void Flush();
 
 	void MarkBlocksAsModified();
+	void ClearModifiedBlocks(const vector<MetaBlockPointer> &pointers);
 
 	idx_t BlockCount();
 
@@ -82,6 +83,7 @@ protected:
 
 	void AddBlock(MetadataBlock new_block, bool if_exists = false);
 	void AddAndRegisterBlock(MetadataBlock block);
+	void ConvertToTransient(MetadataBlock &block);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/metadata/metadata_reader.hpp

@@ -18,6 +18,7 @@ enum class BlockReaderType { EXISTING_BLOCKS, REGISTER_BLOCKS };
 class MetadataReader : public ReadStream {
 public:
 	MetadataReader(MetadataManager &manager, MetaBlockPointer pointer,
+	               optional_ptr<vector<MetaBlockPointer>> read_pointers = nullptr,
 	               BlockReaderType type = BlockReaderType::EXISTING_BLOCKS);
 	MetadataReader(MetadataManager &manager, BlockPointer pointer);
 	~MetadataReader() override;
@@ -46,6 +47,7 @@ private:
 	MetadataHandle block;
 	MetadataPointer next_pointer;
 	bool has_next_block;
+	optional_ptr<vector<MetaBlockPointer>> read_pointers;
 	idx_t index;
 	idx_t offset;
 	idx_t next_offset;

package/src/duckdb/src/include/duckdb/storage/metadata/metadata_writer.hpp

@@ -15,10 +15,10 @@ namespace duckdb {
 
 class MetadataWriter : public WriteStream {
 public:
+	explicit MetadataWriter(MetadataManager &manager,
+	                        optional_ptr<vector<MetaBlockPointer>> written_pointers = nullptr);
 	MetadataWriter(const MetadataWriter &) = delete;
 	MetadataWriter &operator=(const MetadataWriter &) = delete;
-
-	explicit MetadataWriter(MetadataManager &manager);
 	~MetadataWriter() override;
 
 public:
@@ -27,6 +27,9 @@ public:
 
 	BlockPointer GetBlockPointer();
 	MetaBlockPointer GetMetaBlockPointer();
+	MetadataManager &GetManager() {
+		return manager;
+	}
 
 protected:
 	virtual MetadataHandle NextHandle();
@@ -41,6 +44,7 @@ private:
 	MetadataManager &manager;
 	MetadataHandle block;
 	MetadataPointer current_pointer;
+	optional_ptr<vector<MetaBlockPointer>> written_pointers;
 	idx_t capacity;
 	idx_t offset;
 };

package/src/duckdb/src/include/duckdb/storage/storage_info.hpp

@@ -23,6 +23,25 @@ struct FileHandle;
 #error Row group size should be cleanly divisible by vector size
 #endif
 
+struct Storage {
+	//! The size of a hard disk sector, only really needed for Direct IO
+	constexpr static int SECTOR_SIZE = 4096;
+	//! Block header size for blocks written to the storage
+	constexpr static int BLOCK_HEADER_SIZE = sizeof(uint64_t);
+	// Size of a memory slot managed by the StorageManager. This is the quantum of allocation for Blocks on DuckDB. We
+	// default to 256KB. (1 << 18)
+	constexpr static int BLOCK_ALLOC_SIZE = 262144;
+	//! The actual memory space that is available within the blocks
+	constexpr static int BLOCK_SIZE = BLOCK_ALLOC_SIZE - BLOCK_HEADER_SIZE;
+	//! The size of the headers. This should be small and written more or less atomically by the hard disk. We default
+	//! to the page size, which is 4KB. (1 << 12)
+	constexpr static int FILE_HEADER_SIZE = 4096;
+	//! The number of rows per row group (must be a multiple of the vector size)
+	constexpr static const idx_t ROW_GROUP_SIZE = STANDARD_ROW_GROUPS_SIZE;
+	//! The number of vectors per row group
+	constexpr static const idx_t ROW_GROUP_VECTOR_COUNT = ROW_GROUP_SIZE / STANDARD_VECTOR_SIZE;
+};
+
 //! The version number of the database storage format
 extern const uint64_t VERSION_NUMBER;
 

package/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp

@@ -46,8 +46,10 @@ public:
 	virtual void CommitAppend(transaction_t commit_id, idx_t start, idx_t end) = 0;
 	virtual idx_t GetCommittedDeletedCount(idx_t max_count) = 0;
 
-	virtual void Serialize(Serializer &serializer) const = 0;
-	static unique_ptr<ChunkInfo> Deserialize(Deserializer &deserializer);
+	virtual bool HasDeletes() const = 0;
+
+	virtual void Write(WriteStream &writer) const;
+	static unique_ptr<ChunkInfo> Read(ReadStream &reader);
 
 public:
 	template <class TARGET>
@@ -74,8 +76,8 @@ public:
 public:
 	explicit ChunkConstantInfo(idx_t start);
 
-	atomic<transaction_t> insert_id;
-	atomic<transaction_t> delete_id;
+	transaction_t insert_id;
+	transaction_t delete_id;
 
 public:
 	idx_t GetSelVector(TransactionData transaction, SelectionVector &sel_vector, idx_t max_count) override;
@@ -85,8 +87,10 @@ public:
 	void CommitAppend(transaction_t commit_id, idx_t start, idx_t end) override;
 	idx_t GetCommittedDeletedCount(idx_t max_count) override;
 
-	void Serialize(Serializer &serializer) const override;
-	static unique_ptr<ChunkInfo> Deserialize(Deserializer &deserializer);
+	bool HasDeletes() const override;
+
+	void Write(WriteStream &writer) const override;
+	static unique_ptr<ChunkInfo> Read(ReadStream &reader);
 
 private:
 	template <class OP>
@@ -102,13 +106,13 @@ public:
 	explicit ChunkVectorInfo(idx_t start);
 
 	//! The transaction ids of the transactions that inserted the tuples (if any)
-	atomic<transaction_t> inserted[STANDARD_VECTOR_SIZE];
-	atomic<transaction_t> insert_id;
-	atomic<bool> same_inserted_id;
+	transaction_t inserted[STANDARD_VECTOR_SIZE];
+	transaction_t insert_id;
+	bool same_inserted_id;
 
 	//! The transaction ids of the transactions that deleted the tuples (if any)
-	atomic<transaction_t> deleted[STANDARD_VECTOR_SIZE];
-	atomic<bool> any_deleted;
+	transaction_t deleted[STANDARD_VECTOR_SIZE];
+	bool any_deleted;
 
 public:
 	idx_t GetSelVector(transaction_t start_time, transaction_t transaction_id, SelectionVector &sel_vector,
@@ -130,8 +134,10 @@ public:
 	idx_t Delete(transaction_t transaction_id, row_t rows[], idx_t count);
 	void CommitDelete(transaction_t commit_id, row_t rows[], idx_t count);
 
-	void Serialize(Serializer &serializer) const override;
-	static unique_ptr<ChunkInfo> Deserialize(Deserializer &deserializer);
+	bool HasDeletes() const override;
+
+	void Write(WriteStream &writer) const override;
+	static unique_ptr<ChunkInfo> Read(ReadStream &reader);
 
 private:
 	template <class OP>

package/src/duckdb/src/include/duckdb/storage/table/column_data.hpp

@@ -151,7 +151,7 @@ protected:
 	void AppendTransientSegment(SegmentLock &l, idx_t start_row);
 
 	//! Scans a base vector from the column
-	idx_t ScanVector(ColumnScanState &state, Vector &result, idx_t remaining);
+	idx_t ScanVector(ColumnScanState &state, Vector &result, idx_t remaining, bool has_updates);
 	//! Scans a vector from the column merged with any potential updates
 	//! If ALLOW_UPDATES is set to false, the function will instead throw an exception if any updates are found
 	template <bool SCAN_COMMITTED, bool ALLOW_UPDATES>

package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp

@@ -36,11 +36,12 @@ class Vector;
 struct ColumnCheckpointState;
 struct RowGroupPointer;
 struct TransactionData;
-struct VersionNode;
 class CollectionScanState;
 class TableFilterSet;
 struct ColumnFetchState;
 struct RowGroupAppendState;
+class MetadataManager;
+class RowVersionManager;
 
 struct RowGroupWriteData {
 	vector<unique_ptr<ColumnCheckpointState>> states;
@@ -50,11 +51,6 @@ struct RowGroupWriteData {
 class RowGroup : public SegmentBase<RowGroup> {
 public:
 	friend class ColumnData;
-	friend class VersionDeleteState;
-
-public:
-	static constexpr const idx_t ROW_GROUP_SIZE = STANDARD_ROW_GROUPS_SIZE;
-	static constexpr const idx_t ROW_GROUP_VECTOR_COUNT = ROW_GROUP_SIZE / STANDARD_VECTOR_SIZE;
 
 public:
 	RowGroup(RowGroupCollection &collection, idx_t start, idx_t count);
@@ -65,7 +61,7 @@ private:
 	//! The RowGroupCollection this row-group is a part of
 	reference<RowGroupCollection> collection;
 	//! The version info of the row_group (inserted and deleted tuple info)
-	shared_ptr<VersionNode> version_info;
+	shared_ptr<RowVersionManager> version_info;
 	//! The column data of the row_group
 	vector<shared_ptr<ColumnData>> columns;
 
@@ -145,12 +141,17 @@ public:
 
 	void NextVector(CollectionScanState &state);
 
+	idx_t DeleteRows(idx_t vector_idx, transaction_t transaction_id, row_t rows[], idx_t count);
+	RowVersionManager &GetOrCreateVersionInfo();
+
 	// Serialization
 	static void Serialize(RowGroupPointer &pointer, Serializer &serializer);
 	static RowGroupPointer Deserialize(Deserializer &deserializer);
 
 private:
-	ChunkInfo *GetChunkInfo(idx_t vector_idx);
+	shared_ptr<RowVersionManager> &GetVersionInfo();
+	shared_ptr<RowVersionManager> &GetOrCreateVersionInfoPtr();
+
 	ColumnData &GetColumn(storage_t c);
 	idx_t GetColumnCount() const;
 	vector<shared_ptr<ColumnData>> &GetColumns();
@@ -158,18 +159,17 @@ private:
 	template <TableScanType TYPE>
 	void TemplatedScan(TransactionData transaction, CollectionScanState &state, DataChunk &result);
 
+	vector<MetaBlockPointer> CheckpointDeletes(MetadataManager &manager);
+
+	bool HasUnloadedDeletes() const;
+
 private:
 	mutex row_group_lock;
 	mutex stats_lock;
 	vector<MetaBlockPointer> column_pointers;
 	unique_ptr<atomic<bool>[]> is_loaded;
-};
-
-struct VersionNode {
-	unique_ptr<ChunkInfo> info[RowGroup::ROW_GROUP_VECTOR_COUNT];
-
-	void SetStart(idx_t start);
-	idx_t GetCommittedDeletedCount(idx_t count);
+	vector<MetaBlockPointer> deletes_pointers;
+	atomic<bool> deletes_is_loaded;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/row_version_manager.hpp

@@ -0,0 +1,59 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/table/row_version_manager.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/vector_size.hpp"
+#include "duckdb/storage/table/chunk_info.hpp"
+#include "duckdb/storage/storage_info.hpp"
+#include "duckdb/common/mutex.hpp"
+
+namespace duckdb {
+
+class MetadataManager;
+struct MetaBlockPointer;
+
+class RowVersionManager {
+public:
+	explicit RowVersionManager(idx_t start);
+
+	idx_t GetStart() {
+		return start;
+	}
+	void SetStart(idx_t start);
+	idx_t GetCommittedDeletedCount(idx_t count);
+
+	idx_t GetSelVector(TransactionData transaction, idx_t vector_idx, SelectionVector &sel_vector, idx_t max_count);
+	idx_t GetCommittedSelVector(transaction_t start_time, transaction_t transaction_id, idx_t vector_idx,
+	                            SelectionVector &sel_vector, idx_t max_count);
+	bool Fetch(TransactionData transaction, idx_t row);
+
+	void AppendVersionInfo(TransactionData transaction, idx_t count, idx_t row_group_start, idx_t row_group_end);
+	void CommitAppend(transaction_t commit_id, idx_t row_group_start, idx_t count);
+	void RevertAppend(idx_t start_row);
+
+	idx_t DeleteRows(idx_t vector_idx, transaction_t transaction_id, row_t rows[], idx_t count);
+	void CommitDelete(idx_t vector_idx, transaction_t commit_id, row_t rows[], idx_t count);
+
+	vector<MetaBlockPointer> Checkpoint(MetadataManager &manager);
+	static shared_ptr<RowVersionManager> Deserialize(MetaBlockPointer delete_pointer, MetadataManager &manager,
+	                                                 idx_t start);
+
+private:
+	mutex version_lock;
+	idx_t start;
+	unique_ptr<ChunkInfo> vector_info[Storage::ROW_GROUP_VECTOR_COUNT];
+	bool has_changes;
+	vector<MetaBlockPointer> storage_pointers;
+
+private:
+	optional_ptr<ChunkInfo> GetChunkInfo(idx_t vector_idx);
+	ChunkVectorInfo &GetVectorInfo(idx_t vector_idx);
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/update_segment.hpp

@@ -101,7 +101,7 @@ struct UpdateNodeData {
 };
 
 struct UpdateNode {
-	unique_ptr<UpdateNodeData> info[RowGroup::ROW_GROUP_VECTOR_COUNT];
+	unique_ptr<UpdateNodeData> info[Storage::ROW_GROUP_VECTOR_COUNT];
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/transaction/commit_state.hpp

@@ -23,7 +23,7 @@ struct UpdateInfo;
 
 class CommitState {
 public:
-	explicit CommitState(ClientContext &context, transaction_t commit_id, optional_ptr<WriteAheadLog> log = nullptr);
+	explicit CommitState(transaction_t commit_id, optional_ptr<WriteAheadLog> log = nullptr);
 
 	optional_ptr<WriteAheadLog> log;
 	transaction_t commit_id;
@@ -35,9 +35,6 @@ public:
 	unique_ptr<DataChunk> delete_chunk;
 	unique_ptr<DataChunk> update_chunk;
 
-private:
-	ClientContext &context;
-
 public:
 	template <bool HAS_LOG>
 	void CommitEntry(UndoFlags type, data_ptr_t data);
@@ -49,8 +46,6 @@ private:
 	void WriteCatalogEntry(CatalogEntry &entry, data_ptr_t extra_data);
 	void WriteDelete(DeleteInfo &info);
 	void WriteUpdate(UpdateInfo &info);
-
-	void AppendRowId(row_t rowid);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/transaction/delete_info.hpp

@@ -11,12 +11,13 @@
 #include "duckdb/common/constants.hpp"
 
 namespace duckdb {
-class ChunkVectorInfo;
 class DataTable;
+class RowVersionManager;
 
 struct DeleteInfo {
 	DataTable *table;
-	ChunkVectorInfo *vinfo;
+	RowVersionManager *version_info;
+	idx_t vector_idx;
 	idx_t count;
 	idx_t base_row;
 	row_t rows[1];

package/src/duckdb/src/include/duckdb/transaction/duck_transaction.hpp

@@ -11,12 +11,13 @@
 #include "duckdb/transaction/transaction.hpp"
 
 namespace duckdb {
+class RowVersionManager;
 
 class DuckTransaction : public Transaction {
 public:
 	DuckTransaction(TransactionManager &manager, ClientContext &context, transaction_t start_time,
 	                transaction_t transaction_id);
-	~DuckTransaction();
+	~DuckTransaction() override;
 
 	//! The start timestamp of this transaction
 	transaction_t start_time;
@@ -49,7 +50,8 @@ public:
 
 	bool ChangesMade();
 
-	void PushDelete(DataTable &table, ChunkVectorInfo *vinfo, row_t rows[], idx_t count, idx_t base_row);
+	void PushDelete(DataTable &table, RowVersionManager &info, idx_t vector_idx, row_t rows[], idx_t count,
+	                idx_t base_row);
 	void PushAppend(DataTable &table, idx_t row_start, idx_t row_count);
 	UpdateInfo *CreateUpdateInfo(idx_t type_size, idx_t entries);
 

package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp

@@ -88,7 +88,7 @@ private:
 class LocalStorage {
 public:
 	// Threshold to merge row groups instead of appending
-	static constexpr const idx_t MERGE_THRESHOLD = RowGroup::ROW_GROUP_SIZE;
+	static constexpr const idx_t MERGE_THRESHOLD = Storage::ROW_GROUP_SIZE;
 
 public:
 	struct CommitState {