duckdb 0.10.3-dev3.0 → 0.10.3-dev8.0

package/src/duckdb/src/include/duckdb/storage/partial_block_manager.hpp

@@ -85,7 +85,7 @@ struct PartialBlockAllocation {
 	unique_ptr<PartialBlock> partial_block;
 };
 
-enum class CheckpointType { FULL_CHECKPOINT, APPEND_TO_TABLE };
+enum class PartialBlockType { FULL_CHECKPOINT, APPEND_TO_TABLE };
 
 //! Enables sharing blocks across some scope. Scope is whatever we want to share
 //! blocks across. It may be an entire checkpoint or just a single row group.
@@ -101,7 +101,7 @@ public:
 	static constexpr const idx_t MAX_BLOCK_MAP_SIZE = 1u << 31;
 
 public:
-	PartialBlockManager(BlockManager &block_manager, CheckpointType checkpoint_type,
+	PartialBlockManager(BlockManager &block_manager, PartialBlockType partial_block_type,
 	                    uint32_t max_partial_block_size = DEFAULT_MAX_PARTIAL_BLOCK_SIZE,
 	                    uint32_t max_use_count = DEFAULT_MAX_USE_COUNT);
 	virtual ~PartialBlockManager();
@@ -130,7 +130,7 @@ public:
 
 protected:
 	BlockManager &block_manager;
-	CheckpointType checkpoint_type;
+	PartialBlockType partial_block_type;
 	mutex partial_block_lock;
 	//! A map of (available space -> PartialBlock) for partially filled blocks
 	//! This is a multimap because there might be outstanding partial blocks with

package/src/duckdb/src/include/duckdb/storage/standard_buffer_manager.hpp

@@ -50,6 +50,8 @@ public:
 
 	idx_t GetUsedMemory() const final;
 	idx_t GetMaxMemory() const final;
+	idx_t GetUsedSwap() final;
+	optional_idx GetMaxSwap() const final;
 
 	//! Allocate an in-memory buffer with a single pin.
 	//! The allocated memory is released when the buffer handle is destroyed.
@@ -64,7 +66,8 @@ public:
 
 	//! Set a new memory limit to the buffer manager, throws an exception if the new limit is too low and not enough
 	//! blocks can be evicted
-	void SetLimit(idx_t limit = (idx_t)-1) final;
+	void SetMemoryLimit(idx_t limit = (idx_t)-1) final;
+	void SetSwapLimit(optional_idx limit = optional_idx()) final;
 
 	//! Returns informaton about memory usage
 	vector<MemoryInformation> GetMemoryUsageInfo() const override;
@@ -73,14 +76,14 @@ public:
 	vector<TemporaryFileInformation> GetTemporaryFiles() final;
 
 	const string &GetTemporaryDirectory() const final {
-		return temp_directory;
+		return temporary_directory.path;
 	}
 
 	void SetTemporaryDirectory(const string &new_dir) final;
 
 	DUCKDB_API Allocator &GetBufferAllocator() final;
 
-	DatabaseInstance &GetDatabase() {
+	DatabaseInstance &GetDatabase() override {
 		return db;
 	}
 
@@ -136,17 +139,27 @@ protected:
 	//! overwrites the data within with garbage. Any readers that do not hold the pin will notice
 	void VerifyZeroReaders(shared_ptr<BlockHandle> &handle);
 
+protected:
+	// These are stored here because temp_directory creation is lazy
+	// so we need to store information related to the temporary directory before it's created
+	struct TemporaryFileData {
+		//! The directory name where temporary files are stored
+		string path;
+		//! Lock for creating the temp handle (marked mutable so 'GetMaxSwap' can be const)
+		mutable mutex lock;
+		//! Handle for the temporary directory
+		unique_ptr<TemporaryDirectoryHandle> handle;
+		//! The maximum swap space that can be used
+		optional_idx maximum_swap_space = optional_idx();
+	};
+
 protected:
 	//! The database instance
 	DatabaseInstance &db;
 	//! The buffer pool
 	BufferPool &buffer_pool;
-	//! The directory name where temporary files are stored
-	string temp_directory;
-	//! Lock for creating the temp handle
-	mutex temp_handle_lock;
-	//! Handle for the temporary directory
-	unique_ptr<TemporaryDirectoryHandle> temp_directory_handle;
+	//! The variables related to temporary file management
+	TemporaryFileData temporary_directory;
 	//! The temporary id used for managed buffers
 	atomic<block_id_t> temporary_id;
 	//! Allocator associated with the buffer manager, that passes all allocations through this buffer manager

package/src/duckdb/src/include/duckdb/storage/storage_info.hpp

@@ -29,7 +29,7 @@ using block_id_t = int64_t;
 
 struct Storage {
 	//! The size of a hard disk sector, only really needed for Direct IO
-	constexpr static idx_t SECTOR_SIZE = 4096;
+	constexpr static idx_t SECTOR_SIZE = 4096U;
 	//! Block header size for blocks written to the storage
 	constexpr static idx_t BLOCK_HEADER_SIZE = sizeof(uint64_t);
 	//! Size of a memory slot managed by the StorageManager. This is the quantum of allocation for Blocks on DuckDB. We
@@ -39,7 +39,7 @@ struct Storage {
 	constexpr static idx_t BLOCK_SIZE = BLOCK_ALLOC_SIZE - BLOCK_HEADER_SIZE;
 	//! The size of the headers. This should be small and written more or less atomically by the hard disk. We default
 	//! to the page size, which is 4KB. (1 << 12)
-	constexpr static idx_t FILE_HEADER_SIZE = 4096;
+	constexpr static idx_t FILE_HEADER_SIZE = 4096U;
 	//! The number of rows per row group (must be a multiple of the vector size)
 	constexpr static const idx_t ROW_GROUP_SIZE = STANDARD_ROW_GROUPS_SIZE;
 	//! The number of vectors per row group
@@ -48,7 +48,10 @@ struct Storage {
 
 //! The version number of the database storage format
 extern const uint64_t VERSION_NUMBER;
-const char *GetDuckDBVersion(idx_t version_number);
+string GetDuckDBVersion(idx_t version_number);
+optional_idx GetStorageVersion(const char *version_string);
+optional_idx GetSerializationVersion(const char *version_string);
+vector<string> GetSerializationCandidates();
 
 //! The MainHeader is the first header in the storage file. The MainHeader is typically written only once for a database
 //! file.

package/src/duckdb/src/include/duckdb/storage/storage_lock.hpp

@@ -11,42 +11,46 @@
 #include "duckdb/common/constants.hpp"
 #include "duckdb/common/atomic.hpp"
 #include "duckdb/common/mutex.hpp"
+#include "duckdb/common/shared_ptr.hpp"
 
 namespace duckdb {
-class StorageLock;
+struct StorageLockInternals;
 
 enum class StorageLockType { SHARED = 0, EXCLUSIVE = 1 };
 
 class StorageLockKey {
 public:
-	StorageLockKey(StorageLock &lock, StorageLockType type);
+	StorageLockKey(shared_ptr<StorageLockInternals> internals, StorageLockType type);
 	~StorageLockKey();
 
+	StorageLockType GetType() const {
+		return type;
+	}
+
 private:
-	StorageLock &lock;
+	shared_ptr<StorageLockInternals> internals;
 	StorageLockType type;
 };
 
 class StorageLock {
-	friend class StorageLockKey;
-
 public:
 	StorageLock();
+	~StorageLock();
 
 	//! Get an exclusive lock
 	unique_ptr<StorageLockKey> GetExclusiveLock();
 	//! Get a shared lock
 	unique_ptr<StorageLockKey> GetSharedLock();
+	//! Try to get an exclusive lock - if we cannot get it immediately we return `nullptr`
+	unique_ptr<StorageLockKey> TryGetExclusiveLock();
+	//! This is a special method that only exists for checkpointing
+	//! This method takes a shared lock, and returns an exclusive lock if the parameter is the only active shared lock
+	//! If this method succeeds, we have **both** a shared and exclusive lock active (which normally is not allowed)
+	//! But this behavior is required for checkpointing
+	unique_ptr<StorageLockKey> TryUpgradeCheckpointLock(StorageLockKey &lock);
 
 private:
-	mutex exclusive_lock;
-	atomic<idx_t> read_count;
-
-private:
-	//! Release an exclusive lock
-	void ReleaseExclusiveLock();
-	//! Release a shared lock
-	void ReleaseSharedLock();
+	shared_ptr<StorageLockInternals> internals;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/storage_manager.hpp

@@ -14,6 +14,7 @@
 #include "duckdb/storage/table_io_manager.hpp"
 #include "duckdb/storage/write_ahead_log.hpp"
 #include "duckdb/storage/database_size.hpp"
+#include "duckdb/common/enums/checkpoint_type.hpp"
 
 namespace duckdb {
 class BlockManager;
@@ -34,6 +35,17 @@ public:
 	virtual void FlushCommit() = 0;
 };
 
+struct CheckpointOptions {
+	CheckpointOptions()
+	    : wal_action(CheckpointWALAction::DONT_DELETE_WAL), action(CheckpointAction::CHECKPOINT_IF_REQUIRED),
+	      type(CheckpointType::FULL_CHECKPOINT) {
+	}
+
+	CheckpointWALAction wal_action;
+	CheckpointAction action;
+	CheckpointType type;
+};
+
 //! StorageManager is responsible for managing the physical storage of the
 //! database on disk
 class StorageManager {
@@ -46,20 +58,27 @@ public:
 	static StorageManager &Get(Catalog &catalog);
 
 	//! Initialize a database or load an existing database from the given path
-	void Initialize(optional_ptr<ClientContext> context);
+	void Initialize();
 
 	DatabaseInstance &GetDatabase();
 	AttachedDatabase &GetAttached() {
 		return db;
 	}
 
-	//! Get the WAL of the StorageManager, returns nullptr if in-memory
-	optional_ptr<WriteAheadLog> GetWriteAheadLog();
+	//! Gets the size of the WAL, or zero, if there is no WAL.
+	int64_t GetWALSize();
+	//! Gets the WAL of the StorageManager, or nullptr, if there is no WAL.
+	optional_ptr<WriteAheadLog> GetWAL();
+	//! Deletes the WAL file, and resets the unique pointer.
+	void ResetWAL();
 
 	//! Returns the database file path
-	string GetDBPath() {
+	string GetDBPath() const {
 		return path;
 	}
+	bool IsLoaded() const {
+		return load_complete;
+	}
 	//! The path to the WAL, derived from the database file path
 	string GetWALPath();
 	bool InMemory();
@@ -67,13 +86,13 @@ public:
 	virtual bool AutomaticCheckpoint(idx_t estimated_wal_bytes) = 0;
 	virtual unique_ptr<StorageCommitState> GenStorageCommitState(Transaction &transaction, bool checkpoint) = 0;
 	virtual bool IsCheckpointClean(MetaBlockPointer checkpoint_id) = 0;
-	virtual void CreateCheckpoint(bool delete_wal = false, bool force_checkpoint = false) = 0;
+	virtual void CreateCheckpoint(CheckpointOptions options = CheckpointOptions()) = 0;
 	virtual DatabaseSize GetDatabaseSize() = 0;
 	virtual vector<MetadataBlockInfo> GetMetadataInfo() = 0;
 	virtual shared_ptr<TableIOManager> GetTableIOManager(BoundCreateTableInfo *info) = 0;
 
 protected:
-	virtual void LoadDatabase(optional_ptr<ClientContext> context = nullptr) = 0;
+	virtual void LoadDatabase() = 0;
 
 protected:
 	//! The database this storage manager belongs to
@@ -115,12 +134,12 @@ public:
 	bool AutomaticCheckpoint(idx_t estimated_wal_bytes) override;
 	unique_ptr<StorageCommitState> GenStorageCommitState(Transaction &transaction, bool checkpoint) override;
 	bool IsCheckpointClean(MetaBlockPointer checkpoint_id) override;
-	void CreateCheckpoint(bool delete_wal, bool force_checkpoint) override;
+	void CreateCheckpoint(CheckpointOptions options) override;
 	DatabaseSize GetDatabaseSize() override;
 	vector<MetadataBlockInfo> GetMetadataInfo() override;
 	shared_ptr<TableIOManager> GetTableIOManager(BoundCreateTableInfo *info) override;
 
 protected:
-	void LoadDatabase(optional_ptr<ClientContext> context = nullptr) override;
+	void LoadDatabase() override;
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/string_uncompressed.hpp

@@ -149,8 +149,9 @@ public:
 
 				// place the dictionary offset into the set of vectors
 				// note: for overflow strings we write negative value
+
 				D_ASSERT(*dictionary_size <= int32_t(Storage::BLOCK_SIZE));
-				result_data[target_idx] = -(*dictionary_size);
+				result_data[target_idx] = -NumericCast<int32_t>((*dictionary_size));
 			} else {
 				// string fits in block, append to dictionary and increment dictionary position
 				D_ASSERT(string_length < NumericLimits<uint16_t>::Maximum());
@@ -162,7 +163,7 @@ public:
 
 				// place the dictionary offset into the set of vectors
 				D_ASSERT(*dictionary_size <= int32_t(Storage::BLOCK_SIZE));
-				result_data[target_idx] = *dictionary_size;
+				result_data[target_idx] = NumericCast<int32_t>(*dictionary_size);
 			}
 			D_ASSERT(RemainingSpace(segment, handle) <= Storage::BLOCK_SIZE);
 #ifdef DEBUG

package/src/duckdb/src/include/duckdb/storage/table/append_state.hpp

@@ -14,6 +14,7 @@
 #include "duckdb/common/vector.hpp"
 #include "duckdb/function/compression_function.hpp"
 #include "duckdb/transaction/transaction_data.hpp"
+#include "duckdb/planner/bound_constraint.hpp"
 
 namespace duckdb {
 class ColumnSegment;
@@ -21,6 +22,7 @@ class DataTable;
 class LocalTableStorage;
 class RowGroup;
 class UpdateSegment;
+class TableCatalogEntry;
 
 struct TableAppendState;
 
@@ -69,9 +71,19 @@ struct TableAppendState {
 	TransactionData transaction;
 };
 
+struct ConstraintState {
+	explicit ConstraintState(TableCatalogEntry &table_p, const vector<unique_ptr<BoundConstraint>> &bound_constraints)
+	    : table(table_p), bound_constraints(bound_constraints) {
+	}
+
+	TableCatalogEntry &table;
+	const vector<unique_ptr<BoundConstraint>> &bound_constraints;
+};
+
 struct LocalAppendState {
 	TableAppendState append_state;
 	LocalTableStorage *storage;
+	unique_ptr<ConstraintState> constraint_state;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/array_column_data.hpp

@@ -31,8 +31,10 @@ public:
 	void InitializeScan(ColumnScanState &state) override;
 	void InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx) override;
 
-	idx_t Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result) override;
-	idx_t ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates) override;
+	idx_t Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
+	           idx_t scan_count) override;
+	idx_t ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates,
+	                    idx_t scan_count) override;
 	idx_t ScanCount(ColumnScanState &state, Vector &result, idx_t count) override;
 
 	void Skip(ColumnScanState &state, idx_t count = STANDARD_VECTOR_SIZE) override;
@@ -53,8 +55,7 @@ public:
 
 	unique_ptr<ColumnCheckpointState> CreateCheckpointState(RowGroup &row_group,
 	                                                        PartialBlockManager &partial_block_manager) override;
-	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, PartialBlockManager &partial_block_manager,
-	                                             ColumnCheckpointInfo &checkpoint_info) override;
+	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, ColumnCheckpointInfo &info) override;
 
 	void DeserializeColumn(Deserializer &source, BaseStatistics &target_stats) override;
 

package/src/duckdb/src/include/duckdb/storage/table/column_data.hpp

@@ -16,6 +16,7 @@
 #include "duckdb/storage/table/segment_tree.hpp"
 #include "duckdb/storage/table/column_segment_tree.hpp"
 #include "duckdb/common/mutex.hpp"
+#include "duckdb/common/enums/scan_vector_type.hpp"
 
 namespace duckdb {
 class ColumnData;
@@ -29,10 +30,17 @@ struct TransactionData;
 struct TableScanOptions;
 
 struct DataTableInfo;
+struct RowGroupWriteInfo;
 
 struct ColumnCheckpointInfo {
-	explicit ColumnCheckpointInfo(CompressionType compression_type_p) : compression_type(compression_type_p) {};
-	CompressionType compression_type;
+	ColumnCheckpointInfo(RowGroupWriteInfo &info, idx_t column_idx) : info(info), column_idx(column_idx) {
+	}
+
+	RowGroupWriteInfo &info;
+	idx_t column_idx;
+
+public:
+	CompressionType GetCompressionType();
 };
 
 class ColumnData {
@@ -46,7 +54,7 @@ public:
 	//! The start row
 	idx_t start;
 	//! The count of the column data
-	idx_t count;
+	atomic<idx_t> count;
 	//! The block manager
 	BlockManager &block_manager;
 	//! Table info for the column
@@ -76,15 +84,22 @@ public:
 	//! The root type of the column
 	const LogicalType &RootType() const;
 	//! Whether or not the column has any updates
-	virtual bool HasUpdates() const;
+	bool HasUpdates() const;
+	//! Whether or not we can scan an entire vector
+	virtual ScanVectorType GetVectorScanType(ColumnScanState &state, idx_t scan_count);
 
 	//! Initialize a scan of the column
 	virtual void InitializeScan(ColumnScanState &state);
 	//! Initialize a scan starting at the specified offset
 	virtual void InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx);
 	//! Scan the next vector from the column
-	virtual idx_t Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result);
-	virtual idx_t ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates);
+	idx_t Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result);
+	idx_t ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates);
+	virtual idx_t Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
+	                   idx_t scan_count);
+	virtual idx_t ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates,
+	                            idx_t scan_count);
+
 	virtual void ScanCommittedRange(idx_t row_group_start, idx_t offset_in_row_group, idx_t count, Vector &result);
 	virtual idx_t ScanCount(ColumnScanState &state, Vector &result, idx_t count);
 	//! Select
@@ -124,8 +139,7 @@ public:
 
 	virtual unique_ptr<ColumnCheckpointState> CreateCheckpointState(RowGroup &row_group,
 	                                                                PartialBlockManager &partial_block_manager);
-	virtual unique_ptr<ColumnCheckpointState>
-	Checkpoint(RowGroup &row_group, PartialBlockManager &partial_block_manager, ColumnCheckpointInfo &checkpoint_info);
+	virtual unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, ColumnCheckpointInfo &info);
 
 	virtual void CheckpointScan(ColumnSegment &segment, ColumnScanState &state, idx_t row_group_start, idx_t count,
 	                            Vector &scan_vector);
@@ -155,11 +169,12 @@ protected:
 	void AppendTransientSegment(SegmentLock &l, idx_t start_row);
 
 	//! Scans a base vector from the column
-	idx_t ScanVector(ColumnScanState &state, Vector &result, idx_t remaining, bool has_updates);
+	idx_t ScanVector(ColumnScanState &state, Vector &result, idx_t remaining, ScanVectorType scan_type);
 	//! Scans a vector from the column merged with any potential updates
 	//! If ALLOW_UPDATES is set to false, the function will instead throw an exception if any updates are found
 	template <bool SCAN_COMMITTED, bool ALLOW_UPDATES>
-	idx_t ScanVector(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result);
+	idx_t ScanVector(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
+	                 idx_t target_scan);
 
 	void ClearUpdates();
 	void FetchUpdates(TransactionData transaction, idx_t vector_index, Vector &result, idx_t scan_count,
@@ -168,6 +183,8 @@ protected:
 	void UpdateInternal(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids,
 	                    idx_t update_count, Vector &base_vector);
 
+	idx_t GetVectorCount(idx_t vector_index) const;
+
 protected:
 	//! The segments holding the data of this column segment
 	ColumnSegmentTree data;
@@ -175,6 +192,8 @@ protected:
 	mutable mutex update_lock;
 	//! The updates for this column segment
 	unique_ptr<UpdateSegment> updates;
+	//! The lock for the stats
+	mutable mutex stats_lock;
 	//! The stats of the root segment
 	unique_ptr<SegmentStatistics> stats;
 	//! Total transient allocation size

package/src/duckdb/src/include/duckdb/storage/table/column_segment.hpp

@@ -17,6 +17,7 @@
 #include "duckdb/function/compression_function.hpp"
 #include "duckdb/storage/table/segment_base.hpp"
 #include "duckdb/storage/buffer/block_handle.hpp"
+#include "duckdb/common/enums/scan_vector_type.hpp"
 
 namespace duckdb {
 class ColumnSegment;
@@ -64,7 +65,7 @@ public:
 public:
 	void InitializeScan(ColumnScanState &state);
 	//! Scan one vector from this segment
-	void Scan(ColumnScanState &state, idx_t scan_count, Vector &result, idx_t result_offset, bool entire_vector);
+	void Scan(ColumnScanState &state, idx_t scan_count, Vector &result, idx_t result_offset, ScanVectorType scan_type);
 	//! Fetch a value of the specific row id and append it to the result
 	void FetchRow(ColumnFetchState &state, row_t row_id, Vector &result, idx_t result_idx);
 

package/src/duckdb/src/include/duckdb/storage/table/data_table_info.hpp

@@ -11,25 +11,51 @@
 #include "duckdb/common/atomic.hpp"
 #include "duckdb/common/common.hpp"
 #include "duckdb/storage/table/table_index_list.hpp"
+#include "duckdb/storage/storage_lock.hpp"
 
 namespace duckdb {
 class DatabaseInstance;
 class TableIOManager;
 
 struct DataTableInfo {
+	friend class DataTable;
+
+public:
 	DataTableInfo(AttachedDatabase &db, shared_ptr<TableIOManager> table_io_manager_p, string schema, string table);
 
 	//! Initialize any unknown indexes whose types might now be present after an extension load, optionally throwing an
 	//! exception if an index can't be initialized
-	void InitializeIndexes(ClientContext &context, bool throw_on_failure = false);
+	void InitializeIndexes(ClientContext &context, const char *index_type = nullptr);
+
+	//! Whether or not the table is temporary
+	bool IsTemporary() const;
+
+	AttachedDatabase &GetDB() {
+		return db;
+	}
+
+	TableIOManager &GetIOManager() {
+		return *table_io_manager;
+	}
 
+	TableIndexList &GetIndexes() {
+		return indexes;
+	}
+	const vector<IndexStorageInfo> &GetIndexStorageInfo() const {
+		return index_storage_infos;
+	}
+
+	string GetSchemaName();
+	string GetTableName();
+	void SetTableName(string name);
+
+private:
 	//! The database instance of the table
 	AttachedDatabase &db;
 	//! The table IO manager
 	shared_ptr<TableIOManager> table_io_manager;
-	//! The amount of elements in the table. Note that this number signifies the amount of COMMITTED entries in the
-	//! table. It can be inaccurate inside of transactions. More work is needed to properly support that.
-	atomic<idx_t> cardinality;
+	//! Lock for modifying the name
+	mutex name_lock;
 	//! The schema of the table
 	string schema;
 	//! The name of the table
@@ -38,8 +64,8 @@ struct DataTableInfo {
 	TableIndexList indexes;
 	//! Index storage information of the indexes created by this table
 	vector<IndexStorageInfo> index_storage_infos;
-
-	bool IsTemporary() const;
+	//! Lock held while checkpointing
+	StorageLock checkpoint_lock;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/delete_state.hpp

@@ -0,0 +1,23 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/table/delete_state.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/storage/table/append_state.hpp"
+
+namespace duckdb {
+class TableCatalogEntry;
+
+struct TableDeleteState {
+	unique_ptr<ConstraintState> constraint_state;
+	bool has_delete_constraints = false;
+	DataChunk verify_chunk;
+	vector<column_t> col_ids;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/list_column_data.hpp

@@ -31,8 +31,10 @@ public:
 	void InitializeScan(ColumnScanState &state) override;
 	void InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx) override;
 
-	idx_t Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result) override;
-	idx_t ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates) override;
+	idx_t Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
+	           idx_t scan_count) override;
+	idx_t ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates,
+	                    idx_t scan_count) override;
 	idx_t ScanCount(ColumnScanState &state, Vector &result, idx_t count) override;
 
 	void Skip(ColumnScanState &state, idx_t count = STANDARD_VECTOR_SIZE) override;
@@ -53,8 +55,7 @@ public:
 
 	unique_ptr<ColumnCheckpointState> CreateCheckpointState(RowGroup &row_group,
 	                                                        PartialBlockManager &partial_block_manager) override;
-	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, PartialBlockManager &partial_block_manager,
-	                                             ColumnCheckpointInfo &checkpoint_info) override;
+	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, ColumnCheckpointInfo &info) override;
 
 	void DeserializeColumn(Deserializer &deserializer, BaseStatistics &target_stats) override;
 

package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp

@@ -17,6 +17,7 @@
 #include "duckdb/parser/column_list.hpp"
 #include "duckdb/storage/table/segment_base.hpp"
 #include "duckdb/storage/block.hpp"
+#include "duckdb/common/enums/checkpoint_type.hpp"
 
 namespace duckdb {
 class AttachedDatabase;
@@ -43,6 +44,17 @@ struct RowGroupAppendState;
 class MetadataManager;
 class RowVersionManager;
 
+struct RowGroupWriteInfo {
+	RowGroupWriteInfo(PartialBlockManager &manager, const vector<CompressionType> &compression_types,
+	                  CheckpointType checkpoint_type = CheckpointType::FULL_CHECKPOINT)
+	    : manager(manager), compression_types(compression_types), checkpoint_type(checkpoint_type) {
+	}
+
+	PartialBlockManager &manager;
+	const vector<CompressionType> &compression_types;
+	CheckpointType checkpoint_type;
+};
+
 struct RowGroupWriteData {
 	vector<unique_ptr<ColumnCheckpointState>> states;
 	vector<BaseStatistics> statistics;
@@ -77,7 +89,7 @@ public:
 	                               ExpressionExecutor &executor, CollectionScanState &scan_state,
 	                               DataChunk &scan_chunk);
 	unique_ptr<RowGroup> AddColumn(RowGroupCollection &collection, ColumnDefinition &new_column,
-	                               ExpressionExecutor &executor, Expression &default_value, Vector &intermediate);
+	                               ExpressionExecutor &executor, Vector &intermediate);
 	unique_ptr<RowGroup> RemoveColumn(RowGroupCollection &collection, idx_t removed_column);
 
 	void CommitDrop();
@@ -117,7 +129,7 @@ public:
 	//! Delete the given set of rows in the version manager
 	idx_t Delete(TransactionData transaction, DataTable &table, row_t *row_ids, idx_t count);
 
-	RowGroupWriteData WriteToDisk(PartialBlockManager &manager, const vector<CompressionType> &compression_types);
+	RowGroupWriteData WriteToDisk(RowGroupWriteInfo &info);
 	//! Returns the number of committed rows (count - committed deletes)
 	idx_t GetCommittedRowCount();
 	RowGroupWriteData WriteToDisk(RowGroupWriter &writer);
@@ -171,7 +183,6 @@ private:
 
 private:
 	mutex row_group_lock;
-	mutex stats_lock;
 	vector<MetaBlockPointer> column_pointers;
 	unique_ptr<atomic<bool>[]> is_loaded;
 	vector<MetaBlockPointer> deletes_pointers;

package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp

@@ -90,7 +90,8 @@ public:
 
 	void Checkpoint(TableDataWriter &writer, TableStatistics &global_stats);
 
-	void InitializeVacuumState(VacuumState &state, vector<SegmentNode<RowGroup>> &segments);
+	void InitializeVacuumState(CollectionCheckpointState &checkpoint_state, VacuumState &state,
+	                           vector<SegmentNode<RowGroup>> &segments);
 	bool ScheduleVacuumTasks(CollectionCheckpointState &checkpoint_state, VacuumState &state, idx_t segment_idx);
 	void ScheduleCheckpointTask(CollectionCheckpointState &checkpoint_state, idx_t segment_idx);
 
@@ -101,7 +102,7 @@ public:
 	const vector<LogicalType> &GetTypes() const;
 
 	shared_ptr<RowGroupCollection> AddColumn(ClientContext &context, ColumnDefinition &new_column,
-	                                         Expression &default_value);
+	                                         ExpressionExecutor &default_executor);
 	shared_ptr<RowGroupCollection> RemoveColumn(idx_t col_idx);
 	shared_ptr<RowGroupCollection> AlterType(ClientContext &context, idx_t changed_idx, const LogicalType &target_type,
 	                                         vector<column_t> bound_columns, Expression &cast_expr);