duckdb 0.8.2-dev87.0 → 0.9.1-dev0.0

package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp

@@ -20,6 +20,7 @@
 namespace duckdb {
 
 class DatabaseInstance;
+struct MetadataHandle;
 
 struct StorageManagerOptions {
 	bool read_only = false;
@@ -45,7 +46,7 @@ public:
 	//! Return the next free block id
 	block_id_t GetFreeBlockId() override;
 	//! Returns whether or not a specified block is the root block
-	bool IsRootBlock(block_id_t root) override;
+	bool IsRootBlock(MetaBlockPointer root) override;
 	//! Mark a block as free (immediately re-writeable)
 	void MarkBlockAsFree(block_id_t block_id) override;
 	//! Mark a block as modified (re-writeable after a checkpoint)
@@ -53,13 +54,15 @@ public:
 	//! Increase the reference count of a block. The block should hold at least one reference
 	void IncreaseBlockReferenceCount(block_id_t block_id) override;
 	//! Return the meta block id
-	block_id_t GetMetaBlock() override;
+	idx_t GetMetaBlock() override;
 	//! Read the content of the block from disk
 	void Read(Block &block) override;
 	//! Write the given block to disk
 	void Write(FileBuffer &block, block_id_t block_id) override;
 	//! Write the header to disk, this is the final step of the checkpointing process
 	void WriteHeader(DatabaseHeader header) override;
+	//! Truncate the underlying database file after a checkpoint
+	void Truncate() override;
 
 	//! Returns the number of total blocks
 	idx_t TotalBlocks() override;
@@ -76,7 +79,7 @@ private:
 	void ChecksumAndWrite(FileBuffer &handle, uint64_t location) const;
 
 	//! Return the blocks to which we will write the free list and modified blocks
-	vector<block_id_t> GetFreeListBlocks();
+	vector<MetadataHandle> GetFreeListBlocks();
 
 private:
 	AttachedDatabase &db;
@@ -97,11 +100,11 @@ private:
 	//! The list of blocks that will be added to the free list
 	unordered_set<block_id_t> modified_blocks;
 	//! The current meta block id
-	block_id_t meta_block;
+	idx_t meta_block;
 	//! The current maximum block id, this id will be given away first after the free_list runs out
 	block_id_t max_block;
 	//! The block id where the free list can be found
-	block_id_t free_list_id;
+	idx_t free_list_id;
 	//! The current header iteration count
 	uint64_t iteration_count;
 	//! The storage manager options

package/src/duckdb/src/include/duckdb/storage/statistics/base_statistics.hpp

@@ -21,8 +21,7 @@ struct SelectionVector;
 
 class Serializer;
 class Deserializer;
-class FieldWriter;
-class FieldReader;
+
 class Vector;
 struct UnifiedVectorFormat;
 
@@ -93,11 +92,7 @@ public:
 	void CopyBase(const BaseStatistics &orig);
 
 	void Serialize(Serializer &serializer) const;
-	void Serialize(FieldWriter &writer) const;
-
-	idx_t GetDistinctCount();
-
-	static BaseStatistics Deserialize(Deserializer &source, LogicalType type);
+	static BaseStatistics Deserialize(Deserializer &deserializer);
 
 	//! Verify that a vector does not violate the statistics
 	void Verify(Vector &vector, const SelectionVector &sel, idx_t count) const;
@@ -105,6 +100,7 @@ public:
 
 	string ToString() const;
 
+	idx_t GetDistinctCount();
 	static BaseStatistics FromConstant(const Value &input);
 
 private:
@@ -118,7 +114,6 @@ private:
 
 	static BaseStatistics CreateUnknownType(LogicalType type);
 	static BaseStatistics CreateEmptyType(LogicalType type);
-	static BaseStatistics DeserializeType(FieldReader &reader, LogicalType type);
 	static BaseStatistics FromConstantType(const Value &input);
 
 private:

package/src/duckdb/src/include/duckdb/storage/statistics/column_statistics.hpp

@@ -12,6 +12,7 @@
 #include "duckdb/storage/statistics/distinct_statistics.hpp"
 
 namespace duckdb {
+class Serializer;
 
 class ColumnStatistics {
 public:
@@ -32,8 +33,9 @@ public:
 	void SetDistinct(unique_ptr<DistinctStatistics> distinct_stats);
 
 	shared_ptr<ColumnStatistics> Copy() const;
+
 	void Serialize(Serializer &serializer) const;
-	static shared_ptr<ColumnStatistics> Deserialize(Deserializer &source, const LogicalType &type);
+	static shared_ptr<ColumnStatistics> Deserialize(Deserializer &source);
 
 private:
 	BaseStatistics stats;

package/src/duckdb/src/include/duckdb/storage/statistics/distinct_statistics.hpp

@@ -13,9 +13,9 @@
 #include "duckdb/storage/statistics/base_statistics.hpp"
 
 namespace duckdb {
+class Vector;
 class Serializer;
 class Deserializer;
-class Vector;
 
 class DistinctStatistics {
 public:
@@ -34,12 +34,6 @@ public:
 
 	unique_ptr<DistinctStatistics> Copy() const;
 
-	void Serialize(Serializer &serializer) const;
-	void Serialize(FieldWriter &writer) const;
-
-	static unique_ptr<DistinctStatistics> Deserialize(Deserializer &source);
-	static unique_ptr<DistinctStatistics> Deserialize(FieldReader &reader);
-
 	void Update(Vector &update, idx_t count, bool sample = true);
 	void Update(UnifiedVectorFormat &update_data, const LogicalType &ptype, idx_t count, bool sample = true);
 
@@ -48,6 +42,9 @@ public:
 
 	static bool TypeIsSupported(const LogicalType &type);
 
+	void Serialize(Serializer &serializer) const;
+	static unique_ptr<DistinctStatistics> Deserialize(Deserializer &deserializer);
+
 private:
 	//! For distinct statistics we sample the input to speed up insertions
 	static constexpr const double SAMPLE_RATE = 0.1;

package/src/duckdb/src/include/duckdb/storage/statistics/list_stats.hpp

@@ -14,8 +14,6 @@
 
 namespace duckdb {
 class BaseStatistics;
-class FieldWriter;
-class FieldReader;
 struct SelectionVector;
 class Vector;
 
@@ -28,8 +26,8 @@ struct ListStats {
 	DUCKDB_API static BaseStatistics &GetChildStats(BaseStatistics &stats);
 	DUCKDB_API static void SetChildStats(BaseStatistics &stats, unique_ptr<BaseStatistics> new_stats);
 
-	DUCKDB_API static void Serialize(const BaseStatistics &stats, FieldWriter &writer);
-	DUCKDB_API static BaseStatistics Deserialize(FieldReader &reader, LogicalType type);
+	DUCKDB_API static void Serialize(const BaseStatistics &stats, Serializer &serializer);
+	DUCKDB_API static void Deserialize(Deserializer &deserializer, BaseStatistics &base);
 
 	DUCKDB_API static string ToString(const BaseStatistics &stats);
 

package/src/duckdb/src/include/duckdb/storage/statistics/node_statistics.hpp

@@ -9,7 +9,6 @@
 #pragma once
 
 #include "duckdb/common/common.hpp"
-#include "duckdb/common/serializer.hpp"
 
 namespace duckdb {
 
@@ -24,31 +23,6 @@ public:
 	    : has_estimated_cardinality(true), estimated_cardinality(estimated_cardinality), has_max_cardinality(true),
 	      max_cardinality(max_cardinality) {
 	}
-	void Serialize(Serializer &serializer) const {
-		serializer.Write(has_estimated_cardinality);
-		if (has_estimated_cardinality) {
-			serializer.Write(estimated_cardinality);
-			serializer.Write(has_max_cardinality);
-			if (has_max_cardinality) {
-				serializer.Write(max_cardinality);
-			}
-		} else {
-			D_ASSERT(!has_max_cardinality);
-		}
-	}
-	static unique_ptr<NodeStatistics> Deserialize(Deserializer &source) {
-		bool has_estimated_cardinality = source.Read<bool>();
-		if (!has_estimated_cardinality) {
-			return make_uniq<NodeStatistics>();
-		}
-		idx_t estimated_cardinality = source.Read<idx_t>();
-		bool has_max_cardinality = source.Read<bool>();
-		if (!has_max_cardinality) {
-			return make_uniq<NodeStatistics>(estimated_cardinality);
-		}
-		idx_t max_cardinality = source.Read<idx_t>();
-		return make_uniq<NodeStatistics>(estimated_cardinality, max_cardinality);
-	}
 
 	//! Whether or not the node has an estimated cardinality specified
 	bool has_estimated_cardinality;

package/src/duckdb/src/include/duckdb/storage/statistics/numeric_stats.hpp

@@ -16,8 +16,6 @@
 
 namespace duckdb {
 class BaseStatistics;
-class FieldWriter;
-class FieldReader;
 struct SelectionVector;
 class Vector;
 
@@ -61,8 +59,8 @@ struct NumericStats {
 
 	DUCKDB_API static void Merge(BaseStatistics &stats, const BaseStatistics &other_p);
 
-	DUCKDB_API static void Serialize(const BaseStatistics &stats, FieldWriter &writer);
-	DUCKDB_API static BaseStatistics Deserialize(FieldReader &reader, LogicalType type);
+	DUCKDB_API static void Serialize(const BaseStatistics &stats, Serializer &serializer);
+	DUCKDB_API static void Deserialize(Deserializer &deserializer, BaseStatistics &stats);
 
 	DUCKDB_API static string ToString(const BaseStatistics &stats);
 

package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp

@@ -17,8 +17,6 @@
 
 namespace duckdb {
 class BaseStatistics;
-class FieldWriter;
-class FieldReader;
 struct SelectionVector;
 class Vector;
 
@@ -48,14 +46,18 @@ struct StringStats {
 	DUCKDB_API static uint32_t MaxStringLength(const BaseStatistics &stats);
 	//! Whether or not the strings can contain unicode
 	DUCKDB_API static bool CanContainUnicode(const BaseStatistics &stats);
+	//! Returns the min value (up to a length of StringStatsData::MAX_STRING_MINMAX_SIZE)
+	DUCKDB_API static string Min(const BaseStatistics &stats);
+	//! Returns the max value (up to a length of StringStatsData::MAX_STRING_MINMAX_SIZE)
+	DUCKDB_API static string Max(const BaseStatistics &stats);
 
 	//! Resets the max string length so HasMaxStringLength() is false
 	DUCKDB_API static void ResetMaxStringLength(BaseStatistics &stats);
 	//! FIXME: make this part of Set on statistics
 	DUCKDB_API static void SetContainsUnicode(BaseStatistics &stats);
 
-	DUCKDB_API static void Serialize(const BaseStatistics &stats, FieldWriter &writer);
-	DUCKDB_API static BaseStatistics Deserialize(FieldReader &reader, LogicalType type);
+	DUCKDB_API static void Serialize(const BaseStatistics &stats, Serializer &serializer);
+	DUCKDB_API static void Deserialize(Deserializer &deserializer, BaseStatistics &base);
 
 	DUCKDB_API static string ToString(const BaseStatistics &stats);
 

package/src/duckdb/src/include/duckdb/storage/statistics/struct_stats.hpp

@@ -13,8 +13,6 @@
 
 namespace duckdb {
 class BaseStatistics;
-class FieldWriter;
-class FieldReader;
 struct SelectionVector;
 class Vector;
 
@@ -29,8 +27,8 @@ struct StructStats {
 	DUCKDB_API static void SetChildStats(BaseStatistics &stats, idx_t i, const BaseStatistics &new_stats);
 	DUCKDB_API static void SetChildStats(BaseStatistics &stats, idx_t i, unique_ptr<BaseStatistics> new_stats);
 
-	DUCKDB_API static void Serialize(const BaseStatistics &stats, FieldWriter &writer);
-	DUCKDB_API static BaseStatistics Deserialize(FieldReader &reader, LogicalType type);
+	DUCKDB_API static void Serialize(const BaseStatistics &stats, Serializer &serializer);
+	DUCKDB_API static void Deserialize(Deserializer &deserializer, BaseStatistics &base);
 
 	DUCKDB_API static string ToString(const BaseStatistics &stats);
 

package/src/duckdb/src/include/duckdb/storage/storage_info.hpp

@@ -12,8 +12,6 @@
 #include "duckdb/common/vector_size.hpp"
 
 namespace duckdb {
-class Serializer;
-class Deserializer;
 struct FileHandle;
 
 #define STANDARD_ROW_GROUPS_SIZE 122880
@@ -25,6 +23,25 @@ struct FileHandle;
 #error Row group size should be cleanly divisible by vector size
 #endif
 
+struct Storage {
+	//! The size of a hard disk sector, only really needed for Direct IO
+	constexpr static int SECTOR_SIZE = 4096;
+	//! Block header size for blocks written to the storage
+	constexpr static int BLOCK_HEADER_SIZE = sizeof(uint64_t);
+	// Size of a memory slot managed by the StorageManager. This is the quantum of allocation for Blocks on DuckDB. We
+	// default to 256KB. (1 << 18)
+	constexpr static int BLOCK_ALLOC_SIZE = 262144;
+	//! The actual memory space that is available within the blocks
+	constexpr static int BLOCK_SIZE = BLOCK_ALLOC_SIZE - BLOCK_HEADER_SIZE;
+	//! The size of the headers. This should be small and written more or less atomically by the hard disk. We default
+	//! to the page size, which is 4KB. (1 << 12)
+	constexpr static int FILE_HEADER_SIZE = 4096;
+	//! The number of rows per row group (must be a multiple of the vector size)
+	constexpr static const idx_t ROW_GROUP_SIZE = STANDARD_ROW_GROUPS_SIZE;
+	//! The number of vectors per row group
+	constexpr static const idx_t ROW_GROUP_VECTOR_COUNT = ROW_GROUP_SIZE / STANDARD_VECTOR_SIZE;
+};
+
 //! The version number of the database storage format
 extern const uint64_t VERSION_NUMBER;
 
@@ -53,8 +70,8 @@ struct MainHeader {
 
 	static void CheckMagicBytes(FileHandle &handle);
 
-	void Serialize(Serializer &ser);
-	static MainHeader Deserialize(Deserializer &source);
+	void Write(WriteStream &ser);
+	static MainHeader Read(ReadStream &source);
 };
 
 //! The DatabaseHeader contains information about the current state of the database. Every storage file has two
@@ -65,15 +82,15 @@ struct DatabaseHeader {
 	//! The iteration count, increases by 1 every time the storage is checkpointed.
 	uint64_t iteration;
 	//! A pointer to the initial meta block
-	block_id_t meta_block;
+	idx_t meta_block;
 	//! A pointer to the block containing the free list
-	block_id_t free_list;
+	idx_t free_list;
 	//! The number of blocks that is in the file as of this database header. If the file is larger than BLOCK_SIZE *
 	//! block_count any blocks appearing AFTER block_count are implicitly part of the free_list.
 	uint64_t block_count;
 
-	void Serialize(Serializer &ser);
-	static DatabaseHeader Deserialize(Deserializer &source);
+	void Write(WriteStream &ser);
+	static DatabaseHeader Read(ReadStream &source);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/storage_manager.hpp

@@ -65,9 +65,10 @@ public:
 
 	virtual bool AutomaticCheckpoint(idx_t estimated_wal_bytes) = 0;
 	virtual unique_ptr<StorageCommitState> GenStorageCommitState(Transaction &transaction, bool checkpoint) = 0;
-	virtual bool IsCheckpointClean(block_id_t checkpoint_id) = 0;
+	virtual bool IsCheckpointClean(MetaBlockPointer checkpoint_id) = 0;
 	virtual void CreateCheckpoint(bool delete_wal = false, bool force_checkpoint = false) = 0;
 	virtual DatabaseSize GetDatabaseSize() = 0;
+	virtual vector<MetadataBlockInfo> GetMetadataInfo() = 0;
 	virtual shared_ptr<TableIOManager> GetTableIOManager(BoundCreateTableInfo *info) = 0;
 
 protected:
@@ -109,9 +110,10 @@ public:
 public:
 	bool AutomaticCheckpoint(idx_t estimated_wal_bytes) override;
 	unique_ptr<StorageCommitState> GenStorageCommitState(Transaction &transaction, bool checkpoint) override;
-	bool IsCheckpointClean(block_id_t checkpoint_id) override;
+	bool IsCheckpointClean(MetaBlockPointer checkpoint_id) override;
 	void CreateCheckpoint(bool delete_wal, bool force_checkpoint) override;
 	DatabaseSize GetDatabaseSize() override;
+	vector<MetadataBlockInfo> GetMetadataInfo() override;
 	shared_ptr<TableIOManager> GetTableIOManager(BoundCreateTableInfo *info) override;
 
 protected:

package/src/duckdb/src/include/duckdb/storage/string_uncompressed.hpp

@@ -56,7 +56,8 @@ public:
 	static void StringScan(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result);
 	static void StringFetchRow(ColumnSegment &segment, ColumnFetchState &state, row_t row_id, Vector &result,
 	                           idx_t result_idx);
-	static unique_ptr<CompressedSegmentState> StringInitSegment(ColumnSegment &segment, block_id_t block_id);
+	static unique_ptr<CompressedSegmentState> StringInitSegment(ColumnSegment &segment, block_id_t block_id,
+	                                                            optional_ptr<ColumnSegmentState> segment_state);
 
 	static unique_ptr<CompressionAppendState> StringInitAppend(ColumnSegment &segment) {
 		auto &buffer_manager = BufferManager::GetBufferManager(segment.db);
@@ -194,5 +195,9 @@ public:
 	                                    data_ptr_t baseptr, int32_t dict_offset, uint32_t string_length);
 	static string_t FetchString(ColumnSegment &segment, StringDictionaryContainer dict, Vector &result,
 	                            data_ptr_t baseptr, string_location_t location, uint32_t string_length);
+
+	static unique_ptr<ColumnSegmentState> SerializeState(ColumnSegment &segment);
+	static unique_ptr<ColumnSegmentState> DeserializeState(Deserializer &deserializer);
+	static void CleanupState(ColumnSegment &segment);
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp

@@ -18,6 +18,9 @@ struct SelectionVector;
 class Transaction;
 struct TransactionData;
 
+class Serializer;
+class Deserializer;
+
 enum class ChunkInfoType : uint8_t { CONSTANT_INFO, VECTOR_INFO, EMPTY_INFO };
 
 class ChunkInfo {
@@ -41,9 +44,12 @@ public:
 	//! Returns whether or not a single row in the ChunkInfo should be used or not for the given transaction
 	virtual bool Fetch(TransactionData transaction, row_t row) = 0;
 	virtual void CommitAppend(transaction_t commit_id, idx_t start, idx_t end) = 0;
+	virtual idx_t GetCommittedDeletedCount(idx_t max_count) = 0;
+
+	virtual bool HasDeletes() const = 0;
 
-	virtual void Serialize(Serializer &serialize) = 0;
-	static unique_ptr<ChunkInfo> Deserialize(Deserializer &source);
+	virtual void Write(WriteStream &writer) const;
+	static unique_ptr<ChunkInfo> Read(ReadStream &reader);
 
 public:
 	template <class TARGET>
@@ -70,8 +76,8 @@ public:
 public:
 	explicit ChunkConstantInfo(idx_t start);
 
-	atomic<transaction_t> insert_id;
-	atomic<transaction_t> delete_id;
+	transaction_t insert_id;
+	transaction_t delete_id;
 
 public:
 	idx_t GetSelVector(TransactionData transaction, SelectionVector &sel_vector, idx_t max_count) override;
@@ -79,14 +85,17 @@ public:
 	                            SelectionVector &sel_vector, idx_t max_count) override;
 	bool Fetch(TransactionData transaction, row_t row) override;
 	void CommitAppend(transaction_t commit_id, idx_t start, idx_t end) override;
+	idx_t GetCommittedDeletedCount(idx_t max_count) override;
 
-	void Serialize(Serializer &serialize) override;
-	static unique_ptr<ChunkInfo> Deserialize(Deserializer &source);
+	bool HasDeletes() const override;
+
+	void Write(WriteStream &writer) const override;
+	static unique_ptr<ChunkInfo> Read(ReadStream &reader);
 
 private:
 	template <class OP>
 	idx_t TemplatedGetSelVector(transaction_t start_time, transaction_t transaction_id, SelectionVector &sel_vector,
-	                            idx_t max_count);
+	                            idx_t max_count) const;
 };
 
 class ChunkVectorInfo : public ChunkInfo {
@@ -97,22 +106,23 @@ public:
 	explicit ChunkVectorInfo(idx_t start);
 
 	//! The transaction ids of the transactions that inserted the tuples (if any)
-	atomic<transaction_t> inserted[STANDARD_VECTOR_SIZE];
-	atomic<transaction_t> insert_id;
-	atomic<bool> same_inserted_id;
+	transaction_t inserted[STANDARD_VECTOR_SIZE];
+	transaction_t insert_id;
+	bool same_inserted_id;
 
 	//! The transaction ids of the transactions that deleted the tuples (if any)
-	atomic<transaction_t> deleted[STANDARD_VECTOR_SIZE];
-	atomic<bool> any_deleted;
+	transaction_t deleted[STANDARD_VECTOR_SIZE];
+	bool any_deleted;
 
 public:
 	idx_t GetSelVector(transaction_t start_time, transaction_t transaction_id, SelectionVector &sel_vector,
-	                   idx_t max_count);
+	                   idx_t max_count) const;
 	idx_t GetSelVector(TransactionData transaction, SelectionVector &sel_vector, idx_t max_count) override;
 	idx_t GetCommittedSelVector(transaction_t min_start_id, transaction_t min_transaction_id,
 	                            SelectionVector &sel_vector, idx_t max_count) override;
 	bool Fetch(TransactionData transaction, row_t row) override;
 	void CommitAppend(transaction_t commit_id, idx_t start, idx_t end) override;
+	idx_t GetCommittedDeletedCount(idx_t max_count) override;
 
 	void Append(idx_t start, idx_t end, transaction_t commit_id);
 
@@ -124,13 +134,15 @@ public:
 	idx_t Delete(transaction_t transaction_id, row_t rows[], idx_t count);
 	void CommitDelete(transaction_t commit_id, row_t rows[], idx_t count);
 
-	void Serialize(Serializer &serialize) override;
-	static unique_ptr<ChunkInfo> Deserialize(Deserializer &source);
+	bool HasDeletes() const override;
+
+	void Write(WriteStream &writer) const override;
+	static unique_ptr<ChunkInfo> Read(ReadStream &reader);
 
 private:
 	template <class OP>
 	idx_t TemplatedGetSelVector(transaction_t start_time, transaction_t transaction_id, SelectionVector &sel_vector,
-	                            idx_t max_count);
+	                            idx_t max_count) const;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/column_checkpoint_state.hpp

@@ -41,7 +41,7 @@ public:
 	virtual unique_ptr<BaseStatistics> GetStatistics();
 
 	virtual void FlushSegment(unique_ptr<ColumnSegment> segment, idx_t segment_size);
-	virtual void WriteDataPointers(RowGroupWriter &writer);
+	virtual void WriteDataPointers(RowGroupWriter &writer, Serializer &serializer);
 
 public:
 	template <class TARGET>
@@ -68,37 +68,22 @@ struct PartialBlockForCheckpoint : public PartialBlock {
 	};
 
 public:
-	PartialBlockForCheckpoint(ColumnData &data, ColumnSegment &segment, BlockManager &block_manager,
-	                          PartialBlockState state);
+	PartialBlockForCheckpoint(ColumnData &data, ColumnSegment &segment, PartialBlockState state,
+	                          BlockManager &block_manager);
 	~PartialBlockForCheckpoint() override;
 
 	// We will copy all segment data into the memory of the shared block.
 	// Once the block is full (or checkpoint is complete) we'll invoke Flush().
 	// This will cause the block to get written to storage (via BlockManger::ConvertToPersistent),
 	// and all segments to have their references updated (via ColumnSegment::ConvertToPersistent)
-	BlockManager &block_manager;
-	shared_ptr<BlockHandle> block;
 	vector<PartialColumnSegment> segments;
 
-private:
-	struct UninitializedRegion {
-		idx_t start;
-		idx_t end;
-	};
-	vector<UninitializedRegion> uninitialized_regions;
-
 public:
 	bool IsFlushed();
-
-	void AddUninitializedRegion(idx_t start, idx_t end) override;
-
-	void Flush(idx_t free_space_left) override;
-
-	void Clear() override;
-
+	void Flush(const idx_t free_space_left) override;
 	void Merge(PartialBlock &other, idx_t offset, idx_t other_size) override;
-
 	void AddSegmentToTail(ColumnData &data, ColumnSegment &segment, uint32_t offset_in_block);
+	void Clear() override;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/column_data.hpp

@@ -125,9 +125,9 @@ public:
 	virtual void CheckpointScan(ColumnSegment &segment, ColumnScanState &state, idx_t row_group_start, idx_t count,
 	                            Vector &scan_vector);
 
-	virtual void DeserializeColumn(Deserializer &source);
+	virtual void DeserializeColumn(Deserializer &deserializer);
 	static shared_ptr<ColumnData> Deserialize(BlockManager &block_manager, DataTableInfo &info, idx_t column_index,
-	                                          idx_t start_row, Deserializer &source, const LogicalType &type,
+	                                          idx_t start_row, ReadStream &source, const LogicalType &type,
 	                                          optional_ptr<ColumnData> parent);
 
 	virtual void GetColumnSegmentInfo(idx_t row_group_index, vector<idx_t> col_path, vector<ColumnSegmentInfo> &result);
@@ -151,7 +151,7 @@ protected:
 	void AppendTransientSegment(SegmentLock &l, idx_t start_row);
 
 	//! Scans a base vector from the column
-	idx_t ScanVector(ColumnScanState &state, Vector &result, idx_t remaining);
+	idx_t ScanVector(ColumnScanState &state, Vector &result, idx_t remaining, bool has_updates);
 	//! Scans a vector from the column merged with any potential updates
 	//! If ALLOW_UPDATES is set to false, the function will instead throw an exception if any updates are found
 	template <bool SCAN_COMMITTED, bool ALLOW_UPDATES>

package/src/duckdb/src/include/duckdb/storage/table/column_segment.hpp

@@ -57,7 +57,8 @@ public:
 	static unique_ptr<ColumnSegment> CreatePersistentSegment(DatabaseInstance &db, BlockManager &block_manager,
 	                                                         block_id_t id, idx_t offset, const LogicalType &type_p,
 	                                                         idx_t start, idx_t count, CompressionType compression_type,
-	                                                         BaseStatistics statistics);
+	                                                         BaseStatistics statistics,
+	                                                         unique_ptr<ColumnSegmentState> segment_state);
 	static unique_ptr<ColumnSegment> CreateTransientSegment(DatabaseInstance &db, const LogicalType &type, idx_t start,
 	                                                        idx_t segment_size = Storage::BLOCK_SIZE);
 	static unique_ptr<ColumnSegment> CreateSegment(ColumnSegment &other, idx_t start);
@@ -118,14 +119,17 @@ public:
 		return row_index - this->start;
 	}
 
-	CompressedSegmentState *GetSegmentState() {
+	optional_ptr<CompressedSegmentState> GetSegmentState() {
 		return segment_state.get();
 	}
 
+	void CommitDropSegment();
+
 public:
 	ColumnSegment(DatabaseInstance &db, shared_ptr<BlockHandle> block, LogicalType type, ColumnSegmentType segment_type,
 	              idx_t start, idx_t count, CompressionFunction &function, BaseStatistics statistics,
-	              block_id_t block_id, idx_t offset, idx_t segment_size);
+	              block_id_t block_id, idx_t offset, idx_t segment_size,
+	              unique_ptr<ColumnSegmentState> segment_state = nullptr);
 	ColumnSegment(ColumnSegment &other, idx_t start);
 
 private:

package/src/duckdb/src/include/duckdb/storage/table/list_column_data.hpp

@@ -56,7 +56,7 @@ public:
 	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, PartialBlockManager &partial_block_manager,
 	                                             ColumnCheckpointInfo &checkpoint_info) override;
 
-	void DeserializeColumn(Deserializer &source) override;
+	void DeserializeColumn(Deserializer &deserializer) override;
 
 	void GetColumnSegmentInfo(duckdb::idx_t row_group_index, vector<duckdb::idx_t> col_path,
 	                          vector<duckdb::ColumnSegmentInfo> &result) override;

package/src/duckdb/src/include/duckdb/storage/table/persistent_table_data.hpp

@@ -12,6 +12,7 @@
 #include "duckdb/common/vector.hpp"
 #include "duckdb/storage/data_pointer.hpp"
 #include "duckdb/storage/table/table_statistics.hpp"
+#include "duckdb/storage/metadata/metadata_manager.hpp"
 
 namespace duckdb {
 class BaseStatistics;
@@ -24,8 +25,7 @@ public:
 	TableStatistics table_stats;
 	idx_t total_rows;
 	idx_t row_group_count;
-	block_id_t block_id;
-	idx_t offset;
+	MetaBlockPointer block_pointer;
 };
 
 } // namespace duckdb