duckdb 0.9.3-dev0.0 → 0.9.3-dev14.0

package/src/duckdb/src/include/duckdb/storage/object_cache.hpp

@@ -16,7 +16,6 @@
 #include "duckdb/main/database.hpp"
 
 namespace duckdb {
-class ClientContext;
 
 //! ObjectCache is the base class for objects caches in DuckDB
 class ObjectCacheEntry {

package/src/duckdb/src/include/duckdb/storage/partial_block_manager.hpp

@@ -107,14 +107,8 @@ public:
 	virtual ~PartialBlockManager();
 
 public:
-	//! Flush any remaining partial blocks to disk
-	void FlushPartialBlocks();
-
 	PartialBlockAllocation GetBlockAllocation(uint32_t segment_size);
 
-	virtual void AllocateBlock(PartialBlockState &state, uint32_t segment_size);
-
-	void Merge(PartialBlockManager &other);
 	//! Register a partially filled block that is filled with "segment_size" entries
 	void RegisterPartialBlock(PartialBlockAllocation &&allocation);
 
@@ -124,9 +118,20 @@ public:
 	//! Rollback all data written by this partial block manager
 	void Rollback();
 
+	//! Merge this block manager into another one
+	void Merge(PartialBlockManager &other);
+
+	//! Flush any remaining partial blocks to disk
+	void FlushPartialBlocks();
+
+	unique_lock<mutex> GetLock() {
+		return unique_lock<mutex>(partial_block_lock);
+	}
+
 protected:
 	BlockManager &block_manager;
 	CheckpointType checkpoint_type;
+	mutex partial_block_lock;
 	//! A map of (available space -> PartialBlock) for partially filled blocks
 	//! This is a multimap because there might be outstanding partial blocks with
 	//! the same amount of left-over space
@@ -139,6 +144,7 @@ protected:
 	uint32_t max_use_count;
 
 protected:
+	virtual void AllocateBlock(PartialBlockState &state, uint32_t segment_size);
 	//! Try to obtain a partially filled block that can fit "segment_size" bytes
 	//! If successful, returns true and returns the block_id and offset_in_block to write to
 	//! Otherwise, returns false

package/src/duckdb/src/include/duckdb/storage/segment/uncompressed.hpp

@@ -40,9 +40,14 @@ public:
 	static CompressionFunction GetFunction(PhysicalType data_type);
 
 public:
-	//! The max string size that is allowed within a block. Strings bigger than this will be labeled as a BIG STRING and
-	//! offloaded to the overflow blocks.
-	static constexpr uint16_t STRING_BLOCK_LIMIT = 4096;
+	//! The default maximum string size for sufficiently big block sizes
+	static constexpr uint16_t DEFAULT_STRING_BLOCK_LIMIT = 4096;
+	//! The maximum string size within a block. We offload bigger strings to overflow blocks
+	static constexpr uint16_t STRING_BLOCK_LIMIT =
+	    MinValue(AlignValueFloor(Storage::BLOCK_SIZE / 4), idx_t(DEFAULT_STRING_BLOCK_LIMIT));
 };
 
+//! Detect mismatching constant values
+static_assert(StringUncompressed::STRING_BLOCK_LIMIT != 0, "the string block limit cannot be 0");
+
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/standard_buffer_manager.hpp

@@ -13,13 +13,14 @@
 #include "duckdb/common/file_system.hpp"
 #include "duckdb/common/mutex.hpp"
 #include "duckdb/storage/block_manager.hpp"
-
 #include "duckdb/storage/buffer/block_handle.hpp"
-#include "duckdb/storage/buffer_manager.hpp"
 #include "duckdb/storage/buffer/buffer_pool.hpp"
+#include "duckdb/storage/buffer_manager.hpp"
 
 namespace duckdb {
+
 class BlockManager;
+class TemporaryMemoryManager;
 class DatabaseInstance;
 class TemporaryDirectoryHandle;
 struct EvictionQueue;
@@ -49,7 +50,7 @@ public:
 
 	//! Allocate an in-memory buffer with a single pin.
 	//! The allocated memory is released when the buffer handle is destroyed.
-	DUCKDB_API BufferHandle Allocate(idx_t block_size, bool can_destroy = true,
+	DUCKDB_API BufferHandle Allocate(MemoryTag tag, idx_t block_size, bool can_destroy = true,
 	                                 shared_ptr<BlockHandle> *block = nullptr) final override;
 
 	//! Reallocate an in-memory buffer that is pinned.
@@ -62,6 +63,9 @@ public:
 	//! blocks can be evicted
 	void SetLimit(idx_t limit = (idx_t)-1) final override;
 
+	//! Returns informaton about memory usage
+	vector<MemoryInformation> GetMemoryUsageInfo() const override;
+
 	//! Returns a list of all temporary files
 	vector<TemporaryFileInformation> GetTemporaryFiles() final override;
 
@@ -88,33 +92,27 @@ public:
 protected:
 	//! Helper
 	template <typename... ARGS>
-	TempBufferPoolReservation EvictBlocksOrThrow(idx_t memory_delta, unique_ptr<FileBuffer> *buffer, ARGS...);
+	TempBufferPoolReservation EvictBlocksOrThrow(MemoryTag tag, idx_t memory_delta, unique_ptr<FileBuffer> *buffer,
+	                                             ARGS...);
 
 	//! Register an in-memory buffer of arbitrary size, as long as it is >= BLOCK_SIZE. can_destroy signifies whether or
 	//! not the buffer can be destroyed when unpinned, or whether or not it needs to be written to a temporary file so
 	//! it can be reloaded. The resulting buffer will already be allocated, but needs to be pinned in order to be used.
 	//! This needs to be private to prevent creating blocks without ever pinning them:
 	//! blocks that are never pinned are never added to the eviction queue
-	shared_ptr<BlockHandle> RegisterMemory(idx_t block_size, bool can_destroy);
-
-	//! Evict blocks until the currently used memory + extra_memory fit, returns false if this was not possible
-	//! (i.e. not enough blocks could be evicted)
-	//! If the "buffer" argument is specified AND the system can find a buffer to re-use for the given allocation size
-	//! "buffer" will be made to point to the re-usable memory. Note that this is not guaranteed.
-	//! Returns a pair. result.first indicates if eviction was successful. result.second contains the
-	//! reservation handle, which can be moved to the BlockHandle that will own the reservation.
-	BufferPool::EvictionResult EvictBlocks(idx_t extra_memory, idx_t memory_limit,
-	                                       unique_ptr<FileBuffer> *buffer = nullptr);
+	shared_ptr<BlockHandle> RegisterMemory(MemoryTag tag, idx_t block_size, bool can_destroy);
 
 	//! Garbage collect eviction queue
 	void PurgeQueue() final override;
 
-	BufferPool &GetBufferPool() final override;
+	BufferPool &GetBufferPool() const final override;
+	TemporaryMemoryManager &GetTemporaryMemoryManager() final override;
 
 	//! Write a temporary buffer to disk
-	void WriteTemporaryBuffer(block_id_t block_id, FileBuffer &buffer) final override;
+	void WriteTemporaryBuffer(MemoryTag tag, block_id_t block_id, FileBuffer &buffer) final override;
 	//! Read a temporary buffer from disk
-	unique_ptr<FileBuffer> ReadTemporaryBuffer(block_id_t id, unique_ptr<FileBuffer> buffer = nullptr) final override;
+	unique_ptr<FileBuffer> ReadTemporaryBuffer(MemoryTag tag, block_id_t id,
+	                                           unique_ptr<FileBuffer> buffer = nullptr) final override;
 	//! Get the path of the temporary buffer
 	string GetTemporaryPath(block_id_t id);
 
@@ -152,6 +150,8 @@ protected:
 	Allocator buffer_allocator;
 	//! Block manager for temp data
 	unique_ptr<BlockManager> temp_block_manager;
+	//! Temporary evicted memory data per tag
+	atomic<idx_t> evicted_data_per_tag[MEMORY_TAG_COUNT];
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/statistics/array_stats.hpp

@@ -0,0 +1,40 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/statistics/array_stats.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+#include "duckdb/common/exception.hpp"
+
+namespace duckdb {
+class BaseStatistics;
+struct SelectionVector;
+class Vector;
+class Serializer;
+class Deserializer;
+
+struct ArrayStats {
+	DUCKDB_API static void Construct(BaseStatistics &stats);
+	DUCKDB_API static BaseStatistics CreateUnknown(LogicalType type);
+	DUCKDB_API static BaseStatistics CreateEmpty(LogicalType type);
+
+	DUCKDB_API static const BaseStatistics &GetChildStats(const BaseStatistics &stats);
+	DUCKDB_API static BaseStatistics &GetChildStats(BaseStatistics &stats);
+	DUCKDB_API static void SetChildStats(BaseStatistics &stats, unique_ptr<BaseStatistics> new_stats);
+
+	DUCKDB_API static void Serialize(const BaseStatistics &stats, Serializer &serializer);
+	DUCKDB_API static void Deserialize(Deserializer &deserializer, BaseStatistics &base);
+
+	DUCKDB_API static string ToString(const BaseStatistics &stats);
+
+	DUCKDB_API static void Merge(BaseStatistics &stats, const BaseStatistics &other);
+	DUCKDB_API static void Copy(BaseStatistics &stats, const BaseStatistics &other);
+	DUCKDB_API static void Verify(const BaseStatistics &stats, Vector &vector, const SelectionVector &sel, idx_t count);
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/statistics/base_statistics.hpp

@@ -33,13 +33,14 @@ enum class StatsInfo : uint8_t {
 	CAN_HAVE_NULL_AND_VALID_VALUES = 4
 };
 
-enum class StatisticsType : uint8_t { NUMERIC_STATS, STRING_STATS, LIST_STATS, STRUCT_STATS, BASE_STATS };
+enum class StatisticsType : uint8_t { NUMERIC_STATS, STRING_STATS, LIST_STATS, STRUCT_STATS, BASE_STATS, ARRAY_STATS };
 
 class BaseStatistics {
 	friend struct NumericStats;
 	friend struct StringStats;
 	friend struct StructStats;
 	friend struct ListStats;
+	friend struct ArrayStats;
 
 public:
 	DUCKDB_API ~BaseStatistics();
@@ -76,12 +77,20 @@ public:
 	void Set(StatsInfo info);
 	void CombineValidity(BaseStatistics &left, BaseStatistics &right);
 	void CopyValidity(BaseStatistics &stats);
-	inline void SetHasNull() {
+	//! Set that the CURRENT level can have null values
+	//! Note that this is not correct for nested types unless this information is propagated in a different manner
+	//! Use Set(StatsInfo::CAN_HAVE_NULL_VALUES) in the general case
+	inline void SetHasNullFast() {
 		has_null = true;
 	}
-	inline void SetHasNoNull() {
+	//! Set that the CURRENT level can have valiod values
+	//! Note that this is not correct for nested types unless this information is propagated in a different manner
+	//! Use Set(StatsInfo::CAN_HAVE_VALID_VALUES) in the general case
+	inline void SetHasNoNullFast() {
 		has_no_null = true;
 	}
+	void SetHasNull();
+	void SetHasNoNull();
 
 	void Merge(const BaseStatistics &other);
 

package/src/duckdb/src/include/duckdb/storage/statistics/numeric_stats_union.hpp

@@ -26,6 +26,7 @@ struct NumericValueUnion {
 		uint32_t uinteger;
 		uint64_t ubigint;
 		hugeint_t hugeint;
+		uhugeint_t uhugeint;
 		float float_;
 		double double_;
 	} value_;
@@ -47,6 +48,8 @@ DUCKDB_API int64_t &NumericValueUnion::GetReferenceUnsafe();
 template <>
 DUCKDB_API hugeint_t &NumericValueUnion::GetReferenceUnsafe();
 template <>
+DUCKDB_API uhugeint_t &NumericValueUnion::GetReferenceUnsafe();
+template <>
 DUCKDB_API uint8_t &NumericValueUnion::GetReferenceUnsafe();
 template <>
 DUCKDB_API uint16_t &NumericValueUnion::GetReferenceUnsafe();

package/src/duckdb/src/include/duckdb/storage/storage_extension.hpp

@@ -24,8 +24,9 @@ struct StorageExtensionInfo {
 	}
 };
 
-typedef unique_ptr<Catalog> (*attach_function_t)(StorageExtensionInfo *storage_info, AttachedDatabase &db,
-                                                 const string &name, AttachInfo &info, AccessMode access_mode);
+typedef unique_ptr<Catalog> (*attach_function_t)(StorageExtensionInfo *storage_info, ClientContext &context,
+                                                 AttachedDatabase &db, const string &name, AttachInfo &info,
+                                                 AccessMode access_mode);
 typedef unique_ptr<TransactionManager> (*create_transaction_manager_t)(StorageExtensionInfo *storage_info,
                                                                        AttachedDatabase &db, Catalog &catalog);
 

package/src/duckdb/src/include/duckdb/storage/storage_info.hpp

@@ -9,33 +9,37 @@
 #pragma once
 
 #include "duckdb/common/constants.hpp"
+#include "duckdb/common/string.hpp"
 #include "duckdb/common/vector_size.hpp"
+#include "duckdb/common/limits.hpp"
 
 namespace duckdb {
 struct FileHandle;
 
+//! The standard row group size
 #define STANDARD_ROW_GROUPS_SIZE 122880
-#if STANDARD_ROW_GROUPS_SIZE < STANDARD_VECTOR_SIZE
-#error Row groups should be able to hold at least one vector
-#endif
+//! The definition of an invalid block
+#define INVALID_BLOCK (-1)
+//! The maximum block id is 2^62
+#define MAXIMUM_BLOCK 4611686018427388000LL
+//! The default block size
+#define DEFAULT_BLOCK_ALLOC_SIZE 262144
 
-#if ((STANDARD_ROW_GROUPS_SIZE % STANDARD_VECTOR_SIZE) != 0)
-#error Row group size should be cleanly divisible by vector size
-#endif
+using block_id_t = int64_t;
 
 struct Storage {
 	//! The size of a hard disk sector, only really needed for Direct IO
-	constexpr static int SECTOR_SIZE = 4096;
+	constexpr static idx_t SECTOR_SIZE = 4096;
 	//! Block header size for blocks written to the storage
-	constexpr static int BLOCK_HEADER_SIZE = sizeof(uint64_t);
-	// Size of a memory slot managed by the StorageManager. This is the quantum of allocation for Blocks on DuckDB. We
-	// default to 256KB. (1 << 18)
-	constexpr static int BLOCK_ALLOC_SIZE = 262144;
+	constexpr static idx_t BLOCK_HEADER_SIZE = sizeof(uint64_t);
+	//! Size of a memory slot managed by the StorageManager. This is the quantum of allocation for Blocks on DuckDB. We
+	//! default to 256KB. (1 << 18)
+	constexpr static idx_t BLOCK_ALLOC_SIZE = DEFAULT_BLOCK_ALLOC_SIZE;
 	//! The actual memory space that is available within the blocks
-	constexpr static int BLOCK_SIZE = BLOCK_ALLOC_SIZE - BLOCK_HEADER_SIZE;
+	constexpr static idx_t BLOCK_SIZE = BLOCK_ALLOC_SIZE - BLOCK_HEADER_SIZE;
 	//! The size of the headers. This should be small and written more or less atomically by the hard disk. We default
 	//! to the page size, which is 4KB. (1 << 12)
-	constexpr static int FILE_HEADER_SIZE = 4096;
+	constexpr static idx_t FILE_HEADER_SIZE = 4096;
 	//! The number of rows per row group (must be a multiple of the vector size)
 	constexpr static const idx_t ROW_GROUP_SIZE = STANDARD_ROW_GROUPS_SIZE;
 	//! The number of vectors per row group
@@ -44,34 +48,36 @@ struct Storage {
 
 //! The version number of the database storage format
 extern const uint64_t VERSION_NUMBER;
-
 const char *GetDuckDBVersion(idx_t version_number);
 
-using block_id_t = int64_t;
-
-#define INVALID_BLOCK (-1)
-
-// maximum block id, 2^62
-#define MAXIMUM_BLOCK 4611686018427388000LL
-
 //! The MainHeader is the first header in the storage file. The MainHeader is typically written only once for a database
 //! file.
 struct MainHeader {
+	static constexpr idx_t MAX_VERSION_SIZE = 32;
 	static constexpr idx_t MAGIC_BYTE_SIZE = 4;
 	static constexpr idx_t MAGIC_BYTE_OFFSET = Storage::BLOCK_HEADER_SIZE;
 	static constexpr idx_t FLAG_COUNT = 4;
-	// the magic bytes in front of the file
-	// should be "DUCK"
+	//! The magic bytes in front of the file should be "DUCK"
 	static const char MAGIC_BYTES[];
 	//! The version of the database
 	uint64_t version_number;
 	//! The set of flags used by the database
 	uint64_t flags[FLAG_COUNT];
-
 	static void CheckMagicBytes(FileHandle &handle);
 
+	string LibraryGitDesc() {
+		return string((char *)library_git_desc, 0, MAX_VERSION_SIZE);
+	}
+	string LibraryGitHash() {
+		return string((char *)library_git_hash, 0, MAX_VERSION_SIZE);
+	}
+
 	void Write(WriteStream &ser);
 	static MainHeader Read(ReadStream &source);
+
+private:
+	data_t library_git_desc[MAX_VERSION_SIZE];
+	data_t library_git_hash[MAX_VERSION_SIZE];
 };
 
 //! The DatabaseHeader contains information about the current state of the database. Every storage file has two
@@ -88,9 +94,28 @@ struct DatabaseHeader {
 	//! The number of blocks that is in the file as of this database header. If the file is larger than BLOCK_SIZE *
 	//! block_count any blocks appearing AFTER block_count are implicitly part of the free_list.
 	uint64_t block_count;
+	//! The block size of the database file
+	idx_t block_size;
+	//! The vector size of the database file
+	idx_t vector_size;
 
 	void Write(WriteStream &ser);
 	static DatabaseHeader Read(ReadStream &source);
 };
 
+//! Detect mismatching constant values when compiling
+
+#if (STANDARD_ROW_GROUPS_SIZE % STANDARD_VECTOR_SIZE != 0)
+#error The row group size must be a multiple of the vector size
+#endif
+#if (STANDARD_ROW_GROUPS_SIZE < STANDARD_VECTOR_SIZE)
+#error Row groups must be able to hold at least one vector
+#endif
+
+static_assert(Storage::BLOCK_ALLOC_SIZE % Storage::SECTOR_SIZE == 0,
+              "the block allocation size has to be a multiple of the sector size");
+static_assert(Storage::BLOCK_SIZE < idx_t(NumericLimits<int32_t>::Maximum()),
+              "the block size cannot exceed the maximum signed integer value,"
+              "as some comparisons require casts");
+
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/storage_manager.hpp

@@ -54,13 +54,14 @@ public:
 	}
 
 	//! Get the WAL of the StorageManager, returns nullptr if in-memory
-	optional_ptr<WriteAheadLog> GetWriteAheadLog() {
-		return wal.get();
-	}
+	optional_ptr<WriteAheadLog> GetWriteAheadLog();
 
+	//! Returns the database file path
 	string GetDBPath() {
 		return path;
 	}
+	//! The path to the WAL, derived from the database file path
+	string GetWALPath();
 	bool InMemory();
 
 	virtual bool AutomaticCheckpoint(idx_t estimated_wal_bytes) = 0;
@@ -75,7 +76,7 @@ protected:
 	virtual void LoadDatabase() = 0;
 
 protected:
-	//! The database this storagemanager belongs to
+	//! The database this storage manager belongs to
 	AttachedDatabase &db;
 	//! The path of the database
 	string path;
@@ -83,6 +84,9 @@ protected:
 	unique_ptr<WriteAheadLog> wal;
 	//! Whether or not the database is opened in read-only mode
 	bool read_only;
+	//! When loading a database, we do not yet set the wal-field. Therefore, GetWriteAheadLog must
+	//! return nullptr when loading a database
+	bool load_complete = false;
 
 public:
 	template <class TARGET>

package/src/duckdb/src/include/duckdb/storage/table/array_column_data.hpp

@@ -0,0 +1,67 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/table/list_column_data.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/storage/table/column_data.hpp"
+#include "duckdb/storage/table/validity_column_data.hpp"
+
+namespace duckdb {
+
+//! List column data represents a list
+class ArrayColumnData : public ColumnData {
+public:
+	ArrayColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t column_index, idx_t start_row,
+	                LogicalType type, optional_ptr<ColumnData> parent = nullptr);
+
+	//! The child-column of the list
+	unique_ptr<ColumnData> child_column;
+	//! The validity column data of the struct
+	ValidityColumnData validity;
+
+public:
+	void SetStart(idx_t new_start) override;
+	bool CheckZonemap(ColumnScanState &state, TableFilter &filter) override;
+
+	void InitializeScan(ColumnScanState &state) override;
+	void InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx) override;
+
+	idx_t Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result) override;
+	idx_t ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates) override;
+	idx_t ScanCount(ColumnScanState &state, Vector &result, idx_t count) override;
+
+	void Skip(ColumnScanState &state, idx_t count = STANDARD_VECTOR_SIZE) override;
+
+	void InitializeAppend(ColumnAppendState &state) override;
+	void Append(BaseStatistics &stats, ColumnAppendState &state, Vector &vector, idx_t count) override;
+	void RevertAppend(row_t start_row) override;
+	idx_t Fetch(ColumnScanState &state, row_t row_id, Vector &result) override;
+	void FetchRow(TransactionData transaction, ColumnFetchState &state, row_t row_id, Vector &result,
+	              idx_t result_idx) override;
+	void Update(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids,
+	            idx_t update_count) override;
+	void UpdateColumn(TransactionData transaction, const vector<column_t> &column_path, Vector &update_vector,
+	                  row_t *row_ids, idx_t update_count, idx_t depth) override;
+	unique_ptr<BaseStatistics> GetUpdateStatistics() override;
+
+	void CommitDropColumn() override;
+
+	unique_ptr<ColumnCheckpointState> CreateCheckpointState(RowGroup &row_group,
+	                                                        PartialBlockManager &partial_block_manager) override;
+	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, PartialBlockManager &partial_block_manager,
+	                                             ColumnCheckpointInfo &checkpoint_info) override;
+
+	void DeserializeColumn(Deserializer &source, BaseStatistics &target_stats) override;
+
+	void GetColumnSegmentInfo(duckdb::idx_t row_group_index, vector<duckdb::idx_t> col_path,
+	                          vector<duckdb::ColumnSegmentInfo> &result) override;
+
+	void Verify(RowGroup &parent) override;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/column_data.hpp

@@ -26,6 +26,7 @@ class RowGroupWriter;
 class TableDataWriter;
 class TableStorageInfo;
 struct TransactionData;
+struct TableScanOptions;
 
 struct DataTableInfo;
 
@@ -125,10 +126,9 @@ public:
 	virtual void CheckpointScan(ColumnSegment &segment, ColumnScanState &state, idx_t row_group_start, idx_t count,
 	                            Vector &scan_vector);
 
-	virtual void DeserializeColumn(Deserializer &deserializer);
+	virtual void DeserializeColumn(Deserializer &deserializer, BaseStatistics &target_stats);
 	static shared_ptr<ColumnData> Deserialize(BlockManager &block_manager, DataTableInfo &info, idx_t column_index,
-	                                          idx_t start_row, ReadStream &source, const LogicalType &type,
-	                                          optional_ptr<ColumnData> parent);
+	                                          idx_t start_row, ReadStream &source, const LogicalType &type);
 
 	virtual void GetColumnSegmentInfo(idx_t row_group_index, vector<idx_t> col_path, vector<ColumnSegmentInfo> &result);
 	virtual void Verify(RowGroup &parent);

package/src/duckdb/src/include/duckdb/storage/table/column_data_checkpointer.hpp

@@ -13,6 +13,7 @@
 #include "duckdb/storage/table/column_checkpoint_state.hpp"
 
 namespace duckdb {
+struct TableScanOptions;
 
 class ColumnDataCheckpointer {
 public:

package/src/duckdb/src/include/duckdb/storage/table/column_segment.hpp

@@ -60,7 +60,6 @@ public:
 	                                                         unique_ptr<ColumnSegmentState> segment_state);
 	static unique_ptr<ColumnSegment> CreateTransientSegment(DatabaseInstance &db, const LogicalType &type, idx_t start,
 	                                                        idx_t segment_size = Storage::BLOCK_SIZE);
-	static unique_ptr<ColumnSegment> CreateSegment(ColumnSegment &other, idx_t start);
 
 public:
 	void InitializeScan(ColumnScanState &state);
@@ -140,7 +139,7 @@ private:
 	block_id_t block_id;
 	//! The offset into the block (persistent segment only)
 	idx_t offset;
-	//! The allocated segment size
+	//! The allocated segment size, which is bounded by Storage::BLOCK_SIZE
 	idx_t segment_size;
 	//! Storage associated with the compressed segment
 	unique_ptr<CompressedSegmentState> segment_state;

package/src/duckdb/src/include/duckdb/storage/table/data_table_info.hpp

@@ -19,6 +19,9 @@ class TableIOManager;
 struct DataTableInfo {
 	DataTableInfo(AttachedDatabase &db, shared_ptr<TableIOManager> table_io_manager_p, string schema, string table);
 
+	//! Initialize any unknown indexes whose types might now be present after an extension load
+	void InitializeIndexes(ClientContext &context);
+
 	//! The database instance of the table
 	AttachedDatabase &db;
 	//! The table IO manager
@@ -26,12 +29,14 @@ struct DataTableInfo {
 	//! The amount of elements in the table. Note that this number signifies the amount of COMMITTED entries in the
 	//! table. It can be inaccurate inside of transactions. More work is needed to properly support that.
 	atomic<idx_t> cardinality;
-	// schema of the table
+	//! The schema of the table
 	string schema;
-	// name of the table
+	//! The name of the table
 	string table;
-
+	//! The physical list of indexes of this table
 	TableIndexList indexes;
+	//! Index storage information of the indexes created by this table
+	vector<IndexStorageInfo> index_storage_infos;
 
 	bool IsTemporary() const;
 };

package/src/duckdb/src/include/duckdb/storage/table/list_column_data.hpp

@@ -56,7 +56,7 @@ public:
 	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, PartialBlockManager &partial_block_manager,
 	                                             ColumnCheckpointInfo &checkpoint_info) override;
 
-	void DeserializeColumn(Deserializer &deserializer) override;
+	void DeserializeColumn(Deserializer &deserializer, BaseStatistics &target_stats) override;
 
 	void GetColumnSegmentInfo(duckdb::idx_t row_group_index, vector<duckdb::idx_t> col_path,
 	                          vector<duckdb::ColumnSegmentInfo> &result) override;

package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp

@@ -118,8 +118,10 @@ public:
 	idx_t Delete(TransactionData transaction, DataTable &table, row_t *row_ids, idx_t count);
 
 	RowGroupWriteData WriteToDisk(PartialBlockManager &manager, const vector<CompressionType> &compression_types);
-	bool AllDeleted();
-	RowGroupPointer Checkpoint(RowGroupWriter &writer, TableStatistics &global_stats);
+	//! Returns the number of committed rows (count - committed deletes)
+	idx_t GetCommittedRowCount();
+	RowGroupWriteData WriteToDisk(RowGroupWriter &writer);
+	RowGroupPointer Checkpoint(RowGroupWriteData write_data, RowGroupWriter &writer, TableStatistics &global_stats);
 
 	void InitializeAppend(RowGroupAppendState &append_state);
 	void Append(RowGroupAppendState &append_state, DataChunk &chunk, idx_t append_count);

package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp

@@ -14,6 +14,7 @@
 #include "duckdb/storage/table/table_statistics.hpp"
 
 namespace duckdb {
+
 struct ParallelTableScanState;
 struct ParallelCollectionScanState;
 class CreateIndexScanState;
@@ -28,6 +29,8 @@ class BoundConstraint;
 class RowGroupSegmentTree;
 struct ColumnSegmentInfo;
 class MetadataManager;
+struct VacuumState;
+struct CollectionCheckpointState;
 
 class RowGroupCollection {
 public:
@@ -86,6 +89,10 @@ public:
 
 	void Checkpoint(TableDataWriter &writer, TableStatistics &global_stats);
 
+	void InitializeVacuumState(VacuumState &state, vector<SegmentNode<RowGroup>> &segments);
+	bool ScheduleVacuumTasks(CollectionCheckpointState &checkpoint_state, VacuumState &state, idx_t segment_idx);
+	void ScheduleCheckpointTask(CollectionCheckpointState &checkpoint_state, idx_t segment_idx);
+
 	void CommitDropColumn(idx_t index);
 	void CommitDropTable();