duckdb 1.0.1-dev22.0 → 1.0.1-dev27.0

package/src/duckdb/src/include/duckdb/storage/storage_manager.hpp

@@ -23,6 +23,7 @@ class CheckpointWriter;
 class DatabaseInstance;
 class TransactionManager;
 class TableCatalogEntry;
+struct PersistentCollectionData;
 
 class StorageCommitState {
 public:
@@ -31,8 +32,18 @@ public:
 	virtual ~StorageCommitState() {
 	}
 
+	//! Revert the commit
+	virtual void RevertCommit() = 0;
 	// Make the commit persistent
 	virtual void FlushCommit() = 0;
+
+	virtual void AddRowGroupData(DataTable &table, idx_t start_index, idx_t count,
+	                             unique_ptr<PersistentCollectionData> row_group_data) = 0;
+	virtual optional_ptr<PersistentCollectionData> GetRowGroupData(DataTable &table, idx_t start_index,
+	                                                               idx_t &count) = 0;
+	virtual bool HasRowGroupData() {
+		return false;
+	}
 };
 
 struct CheckpointOptions {
@@ -57,8 +68,10 @@ public:
 	static StorageManager &Get(AttachedDatabase &db);
 	static StorageManager &Get(Catalog &catalog);
 
-	//! Initialize a database or load an existing database from the given path
-	void Initialize();
+	//! Initialize a database or load an existing database from the database file path. The block_alloc_size is
+	//! either set, or invalid. If invalid, then DuckDB defaults to the default_block_alloc_size (DBConfig),
+	//! or the file's block allocation size, if it is an existing database.
+	void Initialize(const optional_idx block_alloc_size);
 
 	DatabaseInstance &GetDatabase();
 	AttachedDatabase &GetAttached() {
@@ -66,7 +79,7 @@ public:
 	}
 
 	//! Gets the size of the WAL, or zero, if there is no WAL.
-	int64_t GetWALSize();
+	idx_t GetWALSize();
 	//! Gets the WAL of the StorageManager, or nullptr, if there is no WAL.
 	optional_ptr<WriteAheadLog> GetWAL();
 	//! Deletes the WAL file, and resets the unique pointer.
@@ -84,15 +97,16 @@ public:
 	bool InMemory();
 
 	virtual bool AutomaticCheckpoint(idx_t estimated_wal_bytes) = 0;
-	virtual unique_ptr<StorageCommitState> GenStorageCommitState(Transaction &transaction, bool checkpoint) = 0;
+	virtual unique_ptr<StorageCommitState> GenStorageCommitState(WriteAheadLog &wal) = 0;
 	virtual bool IsCheckpointClean(MetaBlockPointer checkpoint_id) = 0;
 	virtual void CreateCheckpoint(CheckpointOptions options = CheckpointOptions()) = 0;
 	virtual DatabaseSize GetDatabaseSize() = 0;
 	virtual vector<MetadataBlockInfo> GetMetadataInfo() = 0;
 	virtual shared_ptr<TableIOManager> GetTableIOManager(BoundCreateTableInfo *info) = 0;
+	virtual BlockManager &GetBlockManager() = 0;
 
 protected:
-	virtual void LoadDatabase() = 0;
+	virtual void LoadDatabase(const optional_idx block_alloc_size) = 0;
 
 protected:
 	//! The database this storage manager belongs to
@@ -123,6 +137,7 @@ public:
 //! Stores database in a single file.
 class SingleFileStorageManager : public StorageManager {
 public:
+	SingleFileStorageManager() = delete;
 	SingleFileStorageManager(AttachedDatabase &db, string path, bool read_only);
 
 	//! The BlockManager to read/store meta information and data in blocks
@@ -132,14 +147,15 @@ public:
 
 public:
 	bool AutomaticCheckpoint(idx_t estimated_wal_bytes) override;
-	unique_ptr<StorageCommitState> GenStorageCommitState(Transaction &transaction, bool checkpoint) override;
+	unique_ptr<StorageCommitState> GenStorageCommitState(WriteAheadLog &wal) override;
 	bool IsCheckpointClean(MetaBlockPointer checkpoint_id) override;
 	void CreateCheckpoint(CheckpointOptions options) override;
 	DatabaseSize GetDatabaseSize() override;
 	vector<MetadataBlockInfo> GetMetadataInfo() override;
 	shared_ptr<TableIOManager> GetTableIOManager(BoundCreateTableInfo *info) override;
+	BlockManager &GetBlockManager() override;
 
 protected:
-	void LoadDatabase() override;
+	void LoadDatabase(const optional_idx block_alloc_size) override;
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/string_uncompressed.hpp

@@ -1,5 +1,14 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/string_uncompressed.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
 #pragma once
 
+#include "duckdb/common/likely.hpp"
 #include "duckdb/common/types/null_value.hpp"
 #include "duckdb/common/types/vector.hpp"
 #include "duckdb/common/vector_size.hpp"
@@ -9,22 +18,21 @@
 #include "duckdb/storage/buffer_manager.hpp"
 #include "duckdb/storage/checkpoint/string_checkpoint_state.hpp"
 #include "duckdb/storage/segment/uncompressed.hpp"
-#include "duckdb/storage/table/scan_state.hpp"
 #include "duckdb/storage/string_uncompressed.hpp"
 #include "duckdb/storage/table/append_state.hpp"
 #include "duckdb/storage/table/column_segment.hpp"
-#include "duckdb/common/likely.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
 
 namespace duckdb {
 struct StringDictionaryContainer {
 	//! The size of the dictionary
 	uint32_t size;
-	//! The end of the dictionary (typically Storage::BLOCK_SIZE)
+	//! The end of the dictionary, which defaults to the block size.
 	uint32_t end;
 
-	void Verify() {
-		D_ASSERT(size <= Storage::BLOCK_SIZE);
-		D_ASSERT(end <= Storage::BLOCK_SIZE);
+	void Verify(const idx_t block_size) {
+		D_ASSERT(size <= block_size);
+		D_ASSERT(end <= block_size);
 		D_ASSERT(size <= end);
 	}
 };
@@ -43,8 +51,6 @@ public:
 	static constexpr idx_t BIG_STRING_MARKER_BASE_SIZE = sizeof(block_id_t) + sizeof(int32_t);
 	//! The marker size of the big string
 	static constexpr idx_t BIG_STRING_MARKER_SIZE = BIG_STRING_MARKER_BASE_SIZE;
-	//! The size below which the segment is compacted on flushing
-	static constexpr size_t COMPACTION_FLUSH_LIMIT = (size_t)Storage::BLOCK_SIZE / 5 * 4;
 
 public:
 	static unique_ptr<AnalyzeState> StringInitAnalyze(ColumnData &col_data, PhysicalType type);
@@ -111,7 +117,7 @@ public:
 			auto end = handle.Ptr() + *dictionary_end;
 
 #ifdef DEBUG
-			GetDictionary(segment, handle).Verify();
+			GetDictionary(segment, handle).Verify(segment.GetBlockManager().GetBlockSize());
 #endif
 			// Unknown string, continue
 			// non-null value, check if we can fit it within the block
@@ -120,7 +126,8 @@ public:
 			// determine whether or not we have space in the block for this string
 			bool use_overflow_block = false;
 			idx_t required_space = string_length;
-			if (DUCKDB_UNLIKELY(required_space >= StringUncompressed::STRING_BLOCK_LIMIT)) {
+			if (DUCKDB_UNLIKELY(required_space >=
+			                    StringUncompressed::GetStringBlockLimit(segment.GetBlockManager().GetBlockSize()))) {
 				// string exceeds block limit, store in overflow block and only write a marker here
 				required_space = BIG_STRING_MARKER_SIZE;
 				use_overflow_block = true;
@@ -150,7 +157,8 @@ public:
 				// place the dictionary offset into the set of vectors
 				// note: for overflow strings we write negative value
 
-				D_ASSERT(*dictionary_size <= int32_t(Storage::BLOCK_SIZE));
+				// dictionary_size is an uint32_t value, so we can cast up.
+				D_ASSERT(NumericCast<idx_t>(*dictionary_size) <= segment.GetBlockManager().GetBlockSize());
 				result_data[target_idx] = -NumericCast<int32_t>((*dictionary_size));
 			} else {
 				// string fits in block, append to dictionary and increment dictionary position
@@ -161,13 +169,14 @@ public:
 				// now write the actual string data into the dictionary
 				memcpy(dict_pos, source_data[source_idx].GetData(), string_length);
 
-				// place the dictionary offset into the set of vectors
-				D_ASSERT(*dictionary_size <= int32_t(Storage::BLOCK_SIZE));
+				// dictionary_size is an uint32_t value, so we can cast up.
+				D_ASSERT(NumericCast<idx_t>(*dictionary_size) <= segment.GetBlockManager().GetBlockSize());
+				// Place the dictionary offset into the set of vectors.
 				result_data[target_idx] = NumericCast<int32_t>(*dictionary_size);
 			}
-			D_ASSERT(RemainingSpace(segment, handle) <= Storage::BLOCK_SIZE);
+			D_ASSERT(RemainingSpace(segment, handle) <= segment.GetBlockManager().GetBlockSize());
 #ifdef DEBUG
-			GetDictionary(segment, handle).Verify();
+			GetDictionary(segment, handle).Verify(segment.GetBlockManager().GetBlockSize());
 #endif
 		}
 		segment.count += count;
@@ -193,10 +202,10 @@ public:
 	static void WriteStringMarker(data_ptr_t target, block_id_t block_id, int32_t offset);
 	static void ReadStringMarker(data_ptr_t target, block_id_t &block_id, int32_t &offset);
 
-	static string_location_t FetchStringLocation(StringDictionaryContainer dict, data_ptr_t baseptr,
-	                                             int32_t dict_offset);
+	static string_location_t FetchStringLocation(StringDictionaryContainer dict, data_ptr_t base_ptr,
+	                                             int32_t dict_offset, const idx_t block_size);
 	static string_t FetchStringFromDict(ColumnSegment &segment, StringDictionaryContainer dict, Vector &result,
-	                                    data_ptr_t baseptr, int32_t dict_offset, uint32_t string_length);
+	                                    data_ptr_t base_ptr, int32_t dict_offset, uint32_t string_length);
 	static string_t FetchString(ColumnSegment &segment, StringDictionaryContainer dict, Vector &result,
 	                            data_ptr_t baseptr, string_location_t location, uint32_t string_length);
 

package/src/duckdb/src/include/duckdb/storage/table/append_state.hpp

@@ -9,12 +9,13 @@
 #pragma once
 
 #include "duckdb/common/common.hpp"
-#include "duckdb/storage/storage_lock.hpp"
-#include "duckdb/storage/buffer/buffer_handle.hpp"
 #include "duckdb/common/vector.hpp"
 #include "duckdb/function/compression_function.hpp"
-#include "duckdb/transaction/transaction_data.hpp"
 #include "duckdb/planner/bound_constraint.hpp"
+#include "duckdb/storage/buffer/buffer_handle.hpp"
+#include "duckdb/storage/storage_lock.hpp"
+#include "duckdb/storage/table/table_statistics.hpp"
+#include "duckdb/transaction/transaction_data.hpp"
 
 namespace duckdb {
 class ColumnSegment;
@@ -69,6 +70,8 @@ struct TableAppendState {
 	RowGroup *start_row_group;
 	//! The transaction data
 	TransactionData transaction;
+	//! Table statistics
+	TableStatistics stats;
 };
 
 struct ConstraintState {

package/src/duckdb/src/include/duckdb/storage/table/array_column_data.hpp

@@ -26,8 +26,9 @@ public:
 
 public:
 	void SetStart(idx_t new_start) override;
-	bool CheckZonemap(ColumnScanState &state, TableFilter &filter) override;
+	FilterPropagateResult CheckZonemap(ColumnScanState &state, TableFilter &filter) override;
 
+	void InitializePrefetch(PrefetchState &prefetch_state, ColumnScanState &scan_state, idx_t rows) override;
 	void InitializeScan(ColumnScanState &state) override;
 	void InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx) override;
 
@@ -57,7 +58,9 @@ public:
 	                                                        PartialBlockManager &partial_block_manager) override;
 	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, ColumnCheckpointInfo &info) override;
 
-	void DeserializeColumn(Deserializer &source, BaseStatistics &target_stats) override;
+	bool IsPersistent() override;
+	PersistentColumnData Serialize() override;
+	void InitializeColumn(PersistentColumnData &column_data, BaseStatistics &target_stats) override;
 
 	void GetColumnSegmentInfo(duckdb::idx_t row_group_index, vector<duckdb::idx_t> col_path,
 	                          vector<duckdb::ColumnSegmentInfo> &result) override;

package/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp

@@ -45,6 +45,7 @@ public:
 	virtual bool Fetch(TransactionData transaction, row_t row) = 0;
 	virtual void CommitAppend(transaction_t commit_id, idx_t start, idx_t end) = 0;
 	virtual idx_t GetCommittedDeletedCount(idx_t max_count) = 0;
+	virtual bool Cleanup(transaction_t lowest_transaction, unique_ptr<ChunkInfo> &result) const;
 
 	virtual bool HasDeletes() const = 0;
 
@@ -86,6 +87,7 @@ public:
 	bool Fetch(TransactionData transaction, row_t row) override;
 	void CommitAppend(transaction_t commit_id, idx_t start, idx_t end) override;
 	idx_t GetCommittedDeletedCount(idx_t max_count) override;
+	bool Cleanup(transaction_t lowest_transaction, unique_ptr<ChunkInfo> &result) const override;
 
 	bool HasDeletes() const override;
 
@@ -122,6 +124,7 @@ public:
 	                            SelectionVector &sel_vector, idx_t max_count) override;
 	bool Fetch(TransactionData transaction, row_t row) override;
 	void CommitAppend(transaction_t commit_id, idx_t start, idx_t end) override;
+	bool Cleanup(transaction_t lowest_transaction, unique_ptr<ChunkInfo> &result) const override;
 	idx_t GetCommittedDeletedCount(idx_t max_count) override;
 
 	void Append(idx_t start, idx_t end, transaction_t commit_id);

package/src/duckdb/src/include/duckdb/storage/table/column_checkpoint_state.hpp

@@ -41,7 +41,11 @@ public:
 	virtual unique_ptr<BaseStatistics> GetStatistics();
 
 	virtual void FlushSegment(unique_ptr<ColumnSegment> segment, idx_t segment_size);
-	virtual void WriteDataPointers(RowGroupWriter &writer, Serializer &serializer);
+	virtual PersistentColumnData ToPersistentData();
+
+	PartialBlockManager &GetPartialBlockManager() {
+		return partial_block_manager;
+	}
 
 public:
 	template <class TARGET>

package/src/duckdb/src/include/duckdb/storage/table/column_data.hpp

@@ -17,6 +17,7 @@
 #include "duckdb/storage/table/column_segment_tree.hpp"
 #include "duckdb/common/mutex.hpp"
 #include "duckdb/common/enums/scan_vector_type.hpp"
+#include "duckdb/common/serializer/serialization_traits.hpp"
 
 namespace duckdb {
 class ColumnData;
@@ -26,11 +27,12 @@ class RowGroup;
 class RowGroupWriter;
 class TableDataWriter;
 class TableStorageInfo;
-struct TransactionData;
-struct TableScanOptions;
-
 struct DataTableInfo;
+struct PrefetchState;
 struct RowGroupWriteInfo;
+struct TableScanOptions;
+struct TransactionData;
+struct PersistentColumnData;
 
 struct ColumnCheckpointInfo {
 	ColumnCheckpointInfo(RowGroupWriteInfo &info, idx_t column_idx) : info(info), column_idx(column_idx) {
@@ -67,7 +69,7 @@ public:
 	optional_ptr<ColumnData> parent;
 
 public:
-	virtual bool CheckZonemap(ColumnScanState &state, TableFilter &filter) = 0;
+	virtual FilterPropagateResult CheckZonemap(ColumnScanState &state, TableFilter &filter);
 
 	BlockManager &GetBlockManager() {
 		return block_manager;
@@ -88,6 +90,8 @@ public:
 	//! Whether or not we can scan an entire vector
 	virtual ScanVectorType GetVectorScanType(ColumnScanState &state, idx_t scan_count);
 
+	//! Initialize prefetch state with required I/O data for the next N rows
+	virtual void InitializePrefetch(PrefetchState &prefetch_state, ColumnScanState &scan_state, idx_t rows);
 	//! Initialize a scan of the column
 	virtual void InitializeScan(ColumnScanState &state);
 	//! Initialize a scan starting at the specified offset
@@ -102,6 +106,7 @@ public:
 
 	virtual void ScanCommittedRange(idx_t row_group_start, idx_t offset_in_row_group, idx_t count, Vector &result);
 	virtual idx_t ScanCount(ColumnScanState &state, Vector &result, idx_t count);
+
 	//! Select
 	virtual void Select(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
 	                    SelectionVector &sel, idx_t &count, const TableFilter &filter);
@@ -144,14 +149,19 @@ public:
 	virtual void CheckpointScan(ColumnSegment &segment, ColumnScanState &state, idx_t row_group_start, idx_t count,
 	                            Vector &scan_vector);
 
-	virtual void DeserializeColumn(Deserializer &deserializer, BaseStatistics &target_stats);
+	virtual bool IsPersistent();
+	vector<DataPointer> GetDataPointers();
+
+	virtual PersistentColumnData Serialize();
+	void InitializeColumn(PersistentColumnData &column_data);
+	virtual void InitializeColumn(PersistentColumnData &column_data, BaseStatistics &target_stats);
 	static shared_ptr<ColumnData> Deserialize(BlockManager &block_manager, DataTableInfo &info, idx_t column_index,
 	                                          idx_t start_row, ReadStream &source, const LogicalType &type);
 
 	virtual void GetColumnSegmentInfo(idx_t row_group_index, vector<idx_t> col_path, vector<ColumnSegmentInfo> &result);
 	virtual void Verify(RowGroup &parent);
 
-	bool CheckZonemap(TableFilter &filter);
+	FilterPropagateResult CheckZonemap(TableFilter &filter);
 
 	static shared_ptr<ColumnData> CreateColumn(BlockManager &block_manager, DataTableInfo &info, idx_t column_index,
 	                                           idx_t start_row, const LogicalType &type,
@@ -200,4 +210,65 @@ protected:
 	idx_t allocation_size;
 };
 
+struct PersistentColumnData {
+	explicit PersistentColumnData(PhysicalType physical_type);
+	PersistentColumnData(PhysicalType physical_type, vector<DataPointer> pointers);
+	// disable copy constructors
+	PersistentColumnData(const PersistentColumnData &other) = delete;
+	PersistentColumnData &operator=(const PersistentColumnData &) = delete;
+	//! enable move constructors
+	PersistentColumnData(PersistentColumnData &&other) noexcept = default;
+	PersistentColumnData &operator=(PersistentColumnData &&) = default;
+	~PersistentColumnData();
+
+	PhysicalType physical_type;
+	vector<DataPointer> pointers;
+	vector<PersistentColumnData> child_columns;
+	bool has_updates = false;
+
+	void Serialize(Serializer &serializer) const;
+	static PersistentColumnData Deserialize(Deserializer &deserializer);
+	void DeserializeField(Deserializer &deserializer, field_id_t field_idx, const char *field_name,
+	                      const LogicalType &type);
+	bool HasUpdates() const;
+};
+
+struct PersistentRowGroupData {
+	explicit PersistentRowGroupData(vector<LogicalType> types);
+	PersistentRowGroupData() = default;
+	// disable copy constructors
+	PersistentRowGroupData(const PersistentRowGroupData &other) = delete;
+	PersistentRowGroupData &operator=(const PersistentRowGroupData &) = delete;
+	//! enable move constructors
+	PersistentRowGroupData(PersistentRowGroupData &&other) noexcept = default;
+	PersistentRowGroupData &operator=(PersistentRowGroupData &&) = default;
+	~PersistentRowGroupData() = default;
+
+	vector<LogicalType> types;
+	vector<PersistentColumnData> column_data;
+	idx_t start;
+	idx_t count;
+
+	void Serialize(Serializer &serializer) const;
+	static PersistentRowGroupData Deserialize(Deserializer &deserializer);
+	bool HasUpdates() const;
+};
+
+struct PersistentCollectionData {
+	PersistentCollectionData() = default;
+	// disable copy constructors
+	PersistentCollectionData(const PersistentCollectionData &other) = delete;
+	PersistentCollectionData &operator=(const PersistentCollectionData &) = delete;
+	//! enable move constructors
+	PersistentCollectionData(PersistentCollectionData &&other) noexcept = default;
+	PersistentCollectionData &operator=(PersistentCollectionData &&) = default;
+	~PersistentCollectionData() = default;
+
+	vector<PersistentRowGroupData> row_group_data;
+
+	void Serialize(Serializer &serializer) const;
+	static PersistentCollectionData Deserialize(Deserializer &deserializer);
+	bool HasUpdates() const;
+};
+
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/column_segment.hpp

@@ -8,16 +8,16 @@
 
 #pragma once
 
-#include "duckdb/storage/block.hpp"
+#include "duckdb/common/enums/scan_vector_type.hpp"
 #include "duckdb/common/types.hpp"
 #include "duckdb/common/types/vector.hpp"
+#include "duckdb/function/compression_function.hpp"
+#include "duckdb/storage/block.hpp"
+#include "duckdb/storage/buffer/block_handle.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
 #include "duckdb/storage/statistics/segment_statistics.hpp"
 #include "duckdb/storage/storage_lock.hpp"
-#include "duckdb/function/compression_function.hpp"
 #include "duckdb/storage/table/segment_base.hpp"
-#include "duckdb/storage/buffer/block_handle.hpp"
-#include "duckdb/common/enums/scan_vector_type.hpp"
 
 namespace duckdb {
 class ColumnSegment;
@@ -31,6 +31,7 @@ class TableFilter;
 struct ColumnFetchState;
 struct ColumnScanState;
 struct ColumnAppendState;
+struct PrefetchState;
 
 enum class ColumnSegmentType : uint8_t { TRANSIENT, PERSISTENT };
 //! TableFilter represents a filter pushed down into the table scan.
@@ -59,10 +60,12 @@ public:
 	                                                         idx_t start, idx_t count, CompressionType compression_type,
 	                                                         BaseStatistics statistics,
 	                                                         unique_ptr<ColumnSegmentState> segment_state);
-	static unique_ptr<ColumnSegment> CreateTransientSegment(DatabaseInstance &db, const LogicalType &type, idx_t start,
-	                                                        idx_t segment_size = Storage::BLOCK_SIZE);
+	static unique_ptr<ColumnSegment> CreateTransientSegment(DatabaseInstance &db, const LogicalType &type,
+	                                                        const idx_t start, const idx_t segment_size,
+	                                                        const idx_t block_size);
 
 public:
+	void InitializePrefetch(PrefetchState &prefetch_state, ColumnScanState &scan_state);
 	void InitializeScan(ColumnScanState &state);
 	//! Scan one vector from this segment
 	void Scan(ColumnScanState &state, idx_t scan_count, Vector &result, idx_t result_offset, ScanVectorType scan_type);
@@ -97,12 +100,17 @@ public:
 	//! Updates pointers to refer to the given block and offset. This is only used
 	//! when sharing a block among segments. This is invoked only AFTER the block is written.
 	void MarkAsPersistent(shared_ptr<BlockHandle> block, uint32_t offset_in_block);
+	//! Gets a data pointer from a persistent column segment
+	DataPointer GetDataPointer();
 
 	block_id_t GetBlockId() {
 		D_ASSERT(segment_type == ColumnSegmentType::PERSISTENT);
 		return block_id;
 	}
 
+	//! Returns the block manager handling this segment. For transient segments, this might be the temporary block
+	//! manager. Later, we possibly convert this (transient) segment to a persistent segment. In that case, there
+	//! exists another block manager handling the ColumnData, of which this segment is a part.
 	BlockManager &GetBlockManager() const {
 		return block->block_manager;
 	}
@@ -125,11 +133,15 @@ public:
 	void CommitDropSegment();
 
 public:
-	ColumnSegment(DatabaseInstance &db, shared_ptr<BlockHandle> block, LogicalType type, ColumnSegmentType segment_type,
-	              idx_t start, idx_t count, CompressionFunction &function, BaseStatistics statistics,
-	              block_id_t block_id, idx_t offset, idx_t segment_size,
-	              unique_ptr<ColumnSegmentState> segment_state = nullptr);
-	ColumnSegment(ColumnSegment &other, idx_t start);
+	//! Construct a column segment.
+	ColumnSegment(DatabaseInstance &db, shared_ptr<BlockHandle> block, const LogicalType &type,
+	              const ColumnSegmentType segment_type, const idx_t start, const idx_t count,
+	              CompressionFunction &function_p, BaseStatistics statistics, const block_id_t block_id_p,
+	              const idx_t offset, const idx_t segment_size_p,
+	              unique_ptr<ColumnSegmentState> segment_state_p = nullptr);
+	//! Construct a column segment from another column segment.
+	//! The other column segment becomes invalid (std::move).
+	ColumnSegment(ColumnSegment &other, const idx_t start);
 
 private:
 	void Scan(ColumnScanState &state, idx_t scan_count, Vector &result);

package/src/duckdb/src/include/duckdb/storage/table/data_table_info.hpp

@@ -44,6 +44,9 @@ public:
 	const vector<IndexStorageInfo> &GetIndexStorageInfo() const {
 		return index_storage_infos;
 	}
+	unique_ptr<StorageLockKey> GetSharedLock() {
+		return checkpoint_lock.GetSharedLock();
+	}
 
 	string GetSchemaName();
 	string GetTableName();

package/src/duckdb/src/include/duckdb/storage/table/list_column_data.hpp

@@ -26,8 +26,9 @@ public:
 
 public:
 	void SetStart(idx_t new_start) override;
-	bool CheckZonemap(ColumnScanState &state, TableFilter &filter) override;
+	FilterPropagateResult CheckZonemap(ColumnScanState &state, TableFilter &filter) override;
 
+	void InitializePrefetch(PrefetchState &prefetch_state, ColumnScanState &scan_state, idx_t rows) override;
 	void InitializeScan(ColumnScanState &state) override;
 	void InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx) override;
 
@@ -57,7 +58,9 @@ public:
 	                                                        PartialBlockManager &partial_block_manager) override;
 	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, ColumnCheckpointInfo &info) override;
 
-	void DeserializeColumn(Deserializer &deserializer, BaseStatistics &target_stats) override;
+	bool IsPersistent() override;
+	PersistentColumnData Serialize() override;
+	void InitializeColumn(PersistentColumnData &column_data, BaseStatistics &target_stats) override;
 
 	void GetColumnSegmentInfo(duckdb::idx_t row_group_index, vector<duckdb::idx_t> col_path,
 	                          vector<duckdb::ColumnSegmentInfo> &result) override;

package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp

@@ -35,6 +35,8 @@ class TableStatistics;
 struct ColumnSegmentInfo;
 class Vector;
 struct ColumnCheckpointState;
+struct PersistentColumnData;
+struct PersistentRowGroupData;
 struct RowGroupPointer;
 struct TransactionData;
 class CollectionScanState;
@@ -43,6 +45,8 @@ struct ColumnFetchState;
 struct RowGroupAppendState;
 class MetadataManager;
 class RowVersionManager;
+class ScanFilterInfo;
+class StorageCommitState;
 
 struct RowGroupWriteInfo {
 	RowGroupWriteInfo(PartialBlockManager &manager, const vector<CompressionType> &compression_types,
@@ -67,13 +71,16 @@ public:
 public:
 	RowGroup(RowGroupCollection &collection, idx_t start, idx_t count);
 	RowGroup(RowGroupCollection &collection, RowGroupPointer pointer);
+	RowGroup(RowGroupCollection &collection, PersistentRowGroupData &data);
 	~RowGroup();
 
 private:
 	//! The RowGroupCollection this row-group is a part of
 	reference<RowGroupCollection> collection;
 	//! The version info of the row_group (inserted and deleted tuple info)
-	shared_ptr<RowVersionManager> version_info;
+	atomic<optional_ptr<RowVersionManager>> version_info;
+	//! The owned version info of the row_group (inserted and deleted tuple info)
+	shared_ptr<RowVersionManager> owned_version_info;
 	//! The column data of the row_group
 	vector<shared_ptr<ColumnData>> columns;
 
@@ -102,7 +109,7 @@ public:
 	bool InitializeScanWithOffset(CollectionScanState &state, idx_t vector_offset);
 	//! Checks the given set of table filters against the row-group statistics. Returns false if the entire row group
 	//! can be skipped.
-	bool CheckZonemap(TableFilterSet &filters, const vector<column_t> &column_ids);
+	bool CheckZonemap(ScanFilterInfo &filters);
 	//! Checks the given set of table filters against the per-segment statistics. Returns false if any segments were
 	//! skipped.
 	bool CheckZonemapSegments(CollectionScanState &state);
@@ -125,6 +132,8 @@ public:
 	void CommitAppend(transaction_t commit_id, idx_t start, idx_t count);
 	//! Revert a previous append made by RowGroup::AppendVersionInfo
 	void RevertAppend(idx_t start);
+	//! Clean up append states that can either be compressed or deleted
+	void CleanupAppend(transaction_t lowest_transaction, idx_t start, idx_t count);
 
 	//! Delete the given set of rows in the version manager
 	idx_t Delete(TransactionData transaction, DataTable &table, row_t *row_ids, idx_t count);
@@ -134,6 +143,8 @@ public:
 	idx_t GetCommittedRowCount();
 	RowGroupWriteData WriteToDisk(RowGroupWriter &writer);
 	RowGroupPointer Checkpoint(RowGroupWriteData write_data, RowGroupWriter &writer, TableStatistics &global_stats);
+	bool IsPersistent() const;
+	PersistentRowGroupData SerializeRowGroupInfo() const;
 
 	void InitializeAppend(RowGroupAppendState &append_state);
 	void Append(RowGroupAppendState &append_state, DataChunk &chunk, idx_t append_count);
@@ -147,6 +158,7 @@ public:
 
 	void MergeStatistics(idx_t column_idx, const BaseStatistics &other);
 	void MergeIntoStatistics(idx_t column_idx, BaseStatistics &other);
+	void MergeIntoStatistics(TableStatistics &other);
 	unique_ptr<BaseStatistics> GetStatistics(idx_t column_idx);
 
 	void GetColumnSegmentInfo(idx_t row_group_index, vector<ColumnSegmentInfo> &result);
@@ -167,8 +179,10 @@ public:
 	static RowGroupPointer Deserialize(Deserializer &deserializer);
 
 private:
-	shared_ptr<RowVersionManager> &GetVersionInfo();
-	shared_ptr<RowVersionManager> &GetOrCreateVersionInfoPtr();
+	optional_ptr<RowVersionManager> GetVersionInfo();
+	shared_ptr<RowVersionManager> GetOrCreateVersionInfoPtr();
+	shared_ptr<RowVersionManager> GetOrCreateVersionInfoInternal();
+	void SetVersionInfo(shared_ptr<RowVersionManager> version);
 
 	ColumnData &GetColumn(storage_t c);
 	idx_t GetColumnCount() const;

package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp

@@ -27,10 +27,12 @@ struct TableAppendState;
 class DuckTransaction;
 class BoundConstraint;
 class RowGroupSegmentTree;
+class StorageCommitState;
 struct ColumnSegmentInfo;
 class MetadataManager;
 struct VacuumState;
 struct CollectionCheckpointState;
+struct PersistentCollectionData;
 
 class RowGroupCollection {
 public:
@@ -41,6 +43,7 @@ public:
 	idx_t GetTotalRows() const;
 	Allocator &GetAllocator() const;
 
+	void Initialize(PersistentCollectionData &data);
 	void Initialize(PersistentTableData &data);
 	void InitializeEmpty();
 
@@ -78,8 +81,11 @@ public:
 	void FinalizeAppend(TransactionData transaction, TableAppendState &state);
 	void CommitAppend(transaction_t commit_id, idx_t row_start, idx_t count);
 	void RevertAppendInternal(idx_t start_row);
+	void CleanupAppend(transaction_t lowest_transaction, idx_t start, idx_t count);
 
-	void MergeStorage(RowGroupCollection &data);
+	void MergeStorage(RowGroupCollection &data, optional_ptr<DataTable> table,
+	                  optional_ptr<StorageCommitState> commit_state);
+	bool IsPersistent() const;
 
 	void RemoveFromIndexes(TableIndexList &indexes, Vector &row_identifiers, idx_t count);