duckdb 1.3.4 → 1.4.1-dev2.0

package/src/duckdb/src/include/duckdb/storage/storage_info.hpp

@@ -12,9 +12,12 @@
 #include "duckdb/common/limits.hpp"
 #include "duckdb/common/string.hpp"
 #include "duckdb/common/vector_size.hpp"
+#include "duckdb/common/encryption_state.hpp"
 
 namespace duckdb {
+
 struct FileHandle;
+class QueryContext;
 
 //! The standard row group size
 #define DEFAULT_ROW_GROUP_SIZE 122880ULL
@@ -25,17 +28,14 @@ struct FileHandle;
 
 //! The default block allocation size.
 #define DEFAULT_BLOCK_ALLOC_SIZE 262144ULL
-//! The configurable block allocation size.
-#ifndef DUCKDB_BLOCK_ALLOC_SIZE
-#define DUCKDB_BLOCK_ALLOC_SIZE DEFAULT_BLOCK_ALLOC_SIZE
-#endif
 //! The default block header size.
 #define DEFAULT_BLOCK_HEADER_STORAGE_SIZE 8ULL
 //! The default block header size.
 #define DEFAULT_ENCRYPTION_BLOCK_HEADER_SIZE 40ULL
 //! The configurable block allocation size.
 #ifndef DUCKDB_BLOCK_HEADER_STORAGE_SIZE
-#define DUCKDB_BLOCK_HEADER_STORAGE_SIZE DEFAULT_BLOCK_HEADER_STORAGE_SIZE
+#define DUCKDB_BLOCK_HEADER_STORAGE_SIZE     DEFAULT_BLOCK_HEADER_STORAGE_SIZE
+#define DEFAULT_ENCRYPTED_BUFFER_HEADER_SIZE 32ULL
 #endif
 
 using block_id_t = int64_t;
@@ -56,8 +56,8 @@ struct Storage {
 	constexpr static idx_t DEFAULT_BLOCK_HEADER_SIZE = sizeof(idx_t);
 	//! The default block header size for blocks written to storage.
 	constexpr static idx_t MAX_BLOCK_HEADER_SIZE = 128ULL;
-	//! Block header size for encrypted blocks (40 bytes)
-	constexpr static idx_t ENCRYPTED_BLOCK_HEADER_SIZE = 40ULL;
+	//! Block header size for encrypted blocks (64 bytes)
+	constexpr static idx_t ENCRYPTED_BLOCK_HEADER_SIZE = 64ULL;
 	//! The default block size.
 	constexpr static idx_t DEFAULT_BLOCK_SIZE = DEFAULT_BLOCK_ALLOC_SIZE - DEFAULT_BLOCK_HEADER_SIZE;
 
@@ -70,27 +70,47 @@ struct Storage {
 extern const uint64_t VERSION_NUMBER;
 extern const uint64_t VERSION_NUMBER_LOWER;
 extern const uint64_t VERSION_NUMBER_UPPER;
-string GetDuckDBVersion(idx_t version_number);
+string GetDuckDBVersions(const idx_t version_number);
 optional_idx GetStorageVersion(const char *version_string);
-string GetStorageVersionName(idx_t serialization_version);
+string GetStorageVersionName(const idx_t serialization_version, const bool add_suffix);
 optional_idx GetSerializationVersion(const char *version_string);
 vector<string> GetSerializationCandidates();
 
-//! The MainHeader is the first header in the storage file. The MainHeader is typically written only once for a database
-//! file.
-struct MainHeader {
+//! The MainHeader is the first header in the storage file.
+//! It is written only once for a database file.
+class MainHeader {
+public:
 	static constexpr idx_t MAX_VERSION_SIZE = 32;
 	static constexpr idx_t MAGIC_BYTE_SIZE = 4;
 	static constexpr idx_t MAGIC_BYTE_OFFSET = Storage::DEFAULT_BLOCK_HEADER_SIZE;
 	static constexpr idx_t FLAG_COUNT = 4;
+
+	//! Indicates whether database is encrypted or not.
 	static constexpr uint64_t ENCRYPTED_DATABASE_FLAG = 1;
-	//! The magic bytes in front of the file should be "DUCK"
+	//! The encryption key length.
+	static constexpr uint64_t DEFAULT_ENCRYPTION_KEY_LENGTH = 32;
+	//! The magic bytes in front of the file should be "DUCK".
 	static const char MAGIC_BYTES[];
-	//! The version of the database
+	//! The canary should be "DUCKKEY".
+	static const char CANARY[];
+
+	//! The (storage) version of the database.
 	uint64_t version_number;
-	//! The set of flags used by the database
+	//! The set of flags used by the database.
 	uint64_t flags[FLAG_COUNT];
-	static void CheckMagicBytes(FileHandle &handle);
+
+	//! The length of the unique database identifier.
+	static constexpr idx_t DB_IDENTIFIER_LEN = 16;
+	//! Optional metadata for encryption, if encryption flag is set.
+	static constexpr idx_t ENCRYPTION_METADATA_LEN = 8;
+	//! The canary is a known plaintext for detecting wrong keys early.
+	static constexpr idx_t CANARY_BYTE_SIZE = 8;
+	//! Nonce, IV (nonce + counter) and tag length
+	static constexpr uint64_t AES_NONCE_LEN = 16;
+	static constexpr uint64_t AES_IV_LEN = 16;
+	static constexpr uint64_t AES_TAG_LEN = 16;
+
+	static void CheckMagicBytes(QueryContext context, FileHandle &handle);
 
 	string LibraryGitDesc() {
 		return string(char_ptr_cast(library_git_desc), 0, MAX_VERSION_SIZE);
@@ -100,7 +120,46 @@ struct MainHeader {
 	}
 
 	bool IsEncrypted() const {
-		return flags[0] == MainHeader::ENCRYPTED_DATABASE_FLAG;
+		return flags[0] & MainHeader::ENCRYPTED_DATABASE_FLAG;
+	}
+	void SetEncrypted() {
+		flags[0] |= MainHeader::ENCRYPTED_DATABASE_FLAG;
+	}
+
+	void SetEncryptionMetadata(data_ptr_t source) {
+		memset(encryption_metadata, 0, ENCRYPTION_METADATA_LEN);
+		memcpy(encryption_metadata, source, ENCRYPTION_METADATA_LEN);
+	}
+
+	EncryptionTypes::CipherType GetEncryptionCipher() {
+		return static_cast<EncryptionTypes::CipherType>(encryption_metadata[2]);
+	}
+
+	void SetDBIdentifier(data_ptr_t source) {
+		memset(db_identifier, 0, DB_IDENTIFIER_LEN);
+		memcpy(db_identifier, source, DB_IDENTIFIER_LEN);
+	}
+
+	void SetEncryptedCanary(data_ptr_t source) {
+		memset(encrypted_canary, 0, CANARY_BYTE_SIZE);
+		memcpy(encrypted_canary, source, CANARY_BYTE_SIZE);
+	}
+
+	data_ptr_t GetDBIdentifier() {
+		return db_identifier;
+	}
+
+	static bool CompareDBIdentifiers(const data_ptr_t db_identifier_1, const data_ptr_t db_identifier_2) {
+		for (idx_t i = 0; i < DB_IDENTIFIER_LEN; i++) {
+			if (db_identifier_1[i] != db_identifier_2[i]) {
+				return false;
+			}
+		}
+		return true;
+	}
+
+	data_ptr_t GetEncryptedCanary() {
+		return encrypted_canary;
 	}
 
 	void Write(WriteStream &ser);
@@ -109,6 +168,10 @@ struct MainHeader {
 private:
 	data_t library_git_desc[MAX_VERSION_SIZE];
 	data_t library_git_hash[MAX_VERSION_SIZE];
+	data_t encryption_metadata[ENCRYPTION_METADATA_LEN];
+	//! The unique database identifier and optional encryption salt.
+	data_t db_identifier[DB_IDENTIFIER_LEN];
+	data_t encrypted_canary[CANARY_BYTE_SIZE];
 };
 
 //! The DatabaseHeader contains information about the current state of the database. Every storage file has two
@@ -147,8 +210,5 @@ struct DatabaseHeader {
 #if (DEFAULT_BLOCK_ALLOC_SIZE & (DEFAULT_BLOCK_ALLOC_SIZE - 1) != 0)
 #error The default block allocation size must be a power of two
 #endif
-#if (DUCKDB_BLOCK_ALLOC_SIZE & (DUCKDB_BLOCK_ALLOC_SIZE - 1) != 0)
-#error The duckdb block allocation size must be a power of two
-#endif
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/storage_manager.hpp

@@ -58,11 +58,10 @@ struct CheckpointOptions {
 	CheckpointType type;
 };
 
-//! StorageManager is responsible for managing the physical storage of the
-//! database on disk
+//! StorageManager is responsible for managing the physical storage of a persistent database.
 class StorageManager {
 public:
-	StorageManager(AttachedDatabase &db, string path, bool read_only);
+	StorageManager(AttachedDatabase &db, string path, const AttachOptions &options);
 	virtual ~StorageManager();
 
 public:
@@ -72,10 +71,10 @@ public:
 	//! Initialize a database or load an existing database from the database file path. The block_alloc_size is
 	//! either set, or invalid. If invalid, then DuckDB defaults to the default_block_alloc_size (DBConfig),
 	//! or the file's block allocation size, if it is an existing database.
-	void Initialize(StorageOptions options);
+	void Initialize(QueryContext context);
 
 	DatabaseInstance &GetDatabase();
-	AttachedDatabase &GetAttached() {
+	AttachedDatabase &GetAttached() const {
 		return db;
 	}
 
@@ -94,14 +93,13 @@ public:
 		return load_complete;
 	}
 	//! The path to the WAL, derived from the database file path
-	string GetWALPath();
-	bool InMemory();
+	string GetWALPath() const;
+	bool InMemory() const;
 
 	virtual bool AutomaticCheckpoint(idx_t estimated_wal_bytes) = 0;
 	virtual unique_ptr<StorageCommitState> GenStorageCommitState(WriteAheadLog &wal) = 0;
 	virtual bool IsCheckpointClean(MetaBlockPointer checkpoint_id) = 0;
-	virtual void CreateCheckpoint(optional_ptr<ClientContext> client_context,
-	                              CheckpointOptions options = CheckpointOptions()) = 0;
+	virtual void CreateCheckpoint(QueryContext context, CheckpointOptions options = CheckpointOptions()) = 0;
 	virtual DatabaseSize GetDatabaseSize() = 0;
 	virtual vector<MetadataBlockInfo> GetMetadataInfo() = 0;
 	virtual shared_ptr<TableIOManager> GetTableIOManager(BoundCreateTableInfo *info) = 0;
@@ -117,12 +115,34 @@ public:
 		D_ASSERT(HasStorageVersion());
 		return storage_version.GetIndex();
 	}
+	void AddInMemoryChange(idx_t size) {
+		in_memory_change_size += size;
+	}
+	void ResetInMemoryChange() {
+		in_memory_change_size = 0;
+	}
+	bool CompressionIsEnabled() const {
+		return storage_options.compress_in_memory == CompressInMemory::COMPRESS;
+	}
+	EncryptionTypes::CipherType GetCipher() const {
+		return storage_options.encryption_cipher;
+	}
+	void SetCipher(EncryptionTypes::CipherType cipher_p) {
+		D_ASSERT(cipher_p != EncryptionTypes::INVALID);
+		if (cipher_p == EncryptionTypes::CBC) {
+			throw InvalidInputException("CBC cipher is disabled");
+		}
+		storage_options.encryption_cipher = cipher_p;
+	}
+	bool IsEncrypted() const {
+		return storage_options.encryption;
+	}
 
 protected:
-	virtual void LoadDatabase(StorageOptions options) = 0;
+	virtual void LoadDatabase(QueryContext context) = 0;
 
 protected:
-	//! The database this storage manager belongs to
+	//! The attached database managed by this storage manager.
 	AttachedDatabase &db;
 	//! The path of the database
 	string path;
@@ -135,6 +155,10 @@ protected:
 	bool load_complete = false;
 	//! The serialization compatibility version when reading and writing from this database
 	optional_idx storage_version;
+	//! Estimated size of changes for determining automatic checkpointing on in-memory databases
+	atomic<idx_t> in_memory_change_size;
+	//! Storage options passed in through configuration
+	StorageOptions storage_options;
 
 public:
 	template <class TARGET>
@@ -149,28 +173,30 @@ public:
 	}
 };
 
-//! Stores database in a single file.
+//! Stores the database in a single file.
 class SingleFileStorageManager : public StorageManager {
 public:
 	SingleFileStorageManager() = delete;
-	SingleFileStorageManager(AttachedDatabase &db, string path, bool read_only);
+	SingleFileStorageManager(AttachedDatabase &db, string path, const AttachOptions &options);
 
-	//! The BlockManager to read/store meta information and data in blocks
+	//! The BlockManager to read from and write to blocks (meta data and data).
 	unique_ptr<BlockManager> block_manager;
-	//! TableIoManager
+	//! The table I/O manager.
 	unique_ptr<TableIOManager> table_io_manager;
 
 public:
 	bool AutomaticCheckpoint(idx_t estimated_wal_bytes) override;
 	unique_ptr<StorageCommitState> GenStorageCommitState(WriteAheadLog &wal) override;
 	bool IsCheckpointClean(MetaBlockPointer checkpoint_id) override;
-	void CreateCheckpoint(optional_ptr<ClientContext> client_context, CheckpointOptions options) override;
+	void CreateCheckpoint(QueryContext context, CheckpointOptions options) override;
 	DatabaseSize GetDatabaseSize() override;
 	vector<MetadataBlockInfo> GetMetadataInfo() override;
 	shared_ptr<TableIOManager> GetTableIOManager(BoundCreateTableInfo *info) override;
 	BlockManager &GetBlockManager() override;
 
 protected:
-	void LoadDatabase(StorageOptions options) override;
+	void LoadDatabase(QueryContext context) override;
+
+	unique_ptr<CheckpointWriter> CreateCheckpointWriter(QueryContext context, CheckpointOptions options);
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/storage_options.hpp

@@ -10,9 +10,13 @@
 
 #include "duckdb/common/common.hpp"
 #include "duckdb/common/optional_idx.hpp"
+#include "duckdb/common/types/value.hpp"
+#include "duckdb/common/encryption_state.hpp"
 
 namespace duckdb {
 
+enum class CompressInMemory { AUTOMATIC, COMPRESS, DO_NOT_COMPRESS };
+
 struct StorageOptions {
 	//! The allocation size of blocks for this attached database file (if any)
 	optional_idx block_alloc_size;
@@ -22,8 +26,25 @@ struct StorageOptions {
 	optional_idx storage_version;
 	//! Block header size (only used for encryption)
 	optional_idx block_header_size;
+
+	CompressInMemory compress_in_memory = CompressInMemory::AUTOMATIC;
+
 	//! Whether the database is encrypted
 	bool encryption = false;
+	//! Encryption algorithm
+	EncryptionTypes::CipherType encryption_cipher = EncryptionTypes::INVALID;
+	//! encryption key
+	//! FIXME: change to a unique_ptr in the future
+	shared_ptr<string> user_key;
+
+	void Initialize(const unordered_map<string, Value> &options);
 };
 
+inline void ClearUserKey(shared_ptr<string> const &encryption_key) {
+	if (encryption_key && !encryption_key->empty()) {
+		memset(&(*encryption_key)[0], 0, encryption_key->size());
+		encryption_key->clear();
+	}
+}
+
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/array_column_data.hpp

@@ -48,10 +48,10 @@ public:
 	idx_t Fetch(ColumnScanState &state, row_t row_id, Vector &result) override;
 	void FetchRow(TransactionData transaction, ColumnFetchState &state, row_t row_id, Vector &result,
 	              idx_t result_idx) override;
-	void Update(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids,
-	            idx_t update_count) override;
-	void UpdateColumn(TransactionData transaction, const vector<column_t> &column_path, Vector &update_vector,
-	                  row_t *row_ids, idx_t update_count, idx_t depth) override;
+	void Update(TransactionData transaction, DataTable &data_table, idx_t column_index, Vector &update_vector,
+	            row_t *row_ids, idx_t update_count) override;
+	void UpdateColumn(TransactionData transaction, DataTable &data_table, const vector<column_t> &column_path,
+	                  Vector &update_vector, row_t *row_ids, idx_t update_count, idx_t depth) override;
 	unique_ptr<BaseStatistics> GetUpdateStatistics() override;
 
 	void CommitDropColumn() override;
@@ -61,6 +61,7 @@ public:
 	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, ColumnCheckpointInfo &info) override;
 
 	bool IsPersistent() override;
+	bool HasAnyChanges() const override;
 	PersistentColumnData Serialize() override;
 	void InitializeColumn(PersistentColumnData &column_data, BaseStatistics &target_stats) override;
 

package/src/duckdb/src/include/duckdb/storage/table/column_checkpoint_state.hpp

@@ -85,9 +85,9 @@ public:
 
 public:
 	bool IsFlushed();
-	void Flush(const idx_t free_space_left) override;
+	void Flush(QueryContext context, const idx_t free_space_left) override;
 	void Merge(PartialBlock &other, idx_t offset, idx_t other_size) override;
-	void AddSegmentToTail(ColumnData &data, ColumnSegment &segment, uint32_t offset_in_block);
+	void AddSegmentToTail(ColumnData &data, ColumnSegment &segment, uint32_t offset_in_block) override;
 	void Clear() override;
 };
 

package/src/duckdb/src/include/duckdb/storage/table/column_data.hpp

@@ -102,6 +102,11 @@ public:
 	//! Whether or not the column has any updates
 	bool HasUpdates() const;
 	bool HasChanges(idx_t start_row, idx_t end_row) const;
+	//! Whether or not the column has changes at this level
+	bool HasChanges() const;
+
+	//! Whether or not the column has ANY changes, including in child columns
+	virtual bool HasAnyChanges() const;
 	//! Whether or not we can scan an entire vector
 	virtual ScanVectorType GetVectorScanType(ColumnScanState &state, idx_t scan_count, Vector &result);
 
@@ -149,10 +154,10 @@ public:
 	virtual void FetchRow(TransactionData transaction, ColumnFetchState &state, row_t row_id, Vector &result,
 	                      idx_t result_idx);
 
-	virtual void Update(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids,
-	                    idx_t update_count);
-	virtual void UpdateColumn(TransactionData transaction, const vector<column_t> &column_path, Vector &update_vector,
-	                          row_t *row_ids, idx_t update_count, idx_t depth);
+	virtual void Update(TransactionData transaction, DataTable &data_table, idx_t column_index, Vector &update_vector,
+	                    row_t *row_ids, idx_t update_count);
+	virtual void UpdateColumn(TransactionData transaction, DataTable &data_table, const vector<column_t> &column_path,
+	                          Vector &update_vector, row_t *row_ids, idx_t update_count, idx_t depth);
 	virtual unique_ptr<BaseStatistics> GetUpdateStatistics();
 
 	virtual void CommitDropColumn();
@@ -212,8 +217,9 @@ protected:
 	void FetchUpdates(TransactionData transaction, idx_t vector_index, Vector &result, idx_t scan_count,
 	                  bool allow_updates, bool scan_committed);
 	void FetchUpdateRow(TransactionData transaction, row_t row_id, Vector &result, idx_t result_idx);
-	void UpdateInternal(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids,
-	                    idx_t update_count, Vector &base_vector);
+	void UpdateInternal(TransactionData transaction, DataTable &data_table, idx_t column_index, Vector &update_vector,
+	                    row_t *row_ids, idx_t update_count, Vector &base_vector);
+	idx_t FetchUpdateData(ColumnScanState &state, row_t *row_ids, Vector &base_vector);
 
 	idx_t GetVectorCount(idx_t vector_index) const;
 
@@ -232,7 +238,7 @@ protected:
 	//! The stats of the root segment
 	unique_ptr<SegmentStatistics> stats;
 	//! Total transient allocation size
-	idx_t allocation_size;
+	atomic<idx_t> allocation_size;
 
 private:
 	//! The parent column (if any)

package/src/duckdb/src/include/duckdb/storage/table/column_data_checkpointer.hpp

@@ -22,9 +22,10 @@ public:
 	ColumnDataCheckpointData() {
 	}
 	ColumnDataCheckpointData(ColumnCheckpointState &checkpoint_state, ColumnData &col_data, DatabaseInstance &db,
-	                         RowGroup &row_group, ColumnCheckpointInfo &checkpoint_info)
+	                         RowGroup &row_group, ColumnCheckpointInfo &checkpoint_info,
+	                         StorageManager &storage_manager)
 	    : checkpoint_state(checkpoint_state), col_data(col_data), db(db), row_group(row_group),
-	      checkpoint_info(checkpoint_info) {
+	      checkpoint_info(checkpoint_info), storage_manager(storage_manager) {
 	}
 
 public:
@@ -34,6 +35,7 @@ public:
 	RowGroup &GetRowGroup();
 	ColumnCheckpointState &GetCheckpointState();
 	DatabaseInstance &GetDatabase();
+	StorageManager &GetStorageManager();
 
 private:
 	optional_ptr<ColumnCheckpointState> checkpoint_state;
@@ -41,6 +43,7 @@ private:
 	optional_ptr<DatabaseInstance> db;
 	optional_ptr<RowGroup> row_group;
 	optional_ptr<ColumnCheckpointInfo> checkpoint_info;
+	optional_ptr<StorageManager> storage_manager;
 };
 
 struct CheckpointAnalyzeResult {

package/src/duckdb/src/include/duckdb/storage/table/column_segment.hpp

@@ -20,19 +20,21 @@
 #include "duckdb/storage/table/segment_base.hpp"
 
 namespace duckdb {
-class ColumnSegment;
+
+class BaseStatistics;
 class BlockManager;
 class ColumnData;
+class ColumnSegment;
 class DatabaseInstance;
+class TableFilter;
 class Transaction;
-class BaseStatistics;
 class UpdateSegment;
-class TableFilter;
-struct TableFilterState;
+
+struct ColumnAppendState;
 struct ColumnFetchState;
 struct ColumnScanState;
-struct ColumnAppendState;
 struct PrefetchState;
+struct TableFilterState;
 
 enum class ColumnSegmentType : uint8_t { TRANSIENT, PERSISTENT };
 //! TableFilter represents a filter pushed down into the table scan.
@@ -97,12 +99,13 @@ public:
 	//! Revert an append made to this segment
 	void RevertAppend(idx_t start_row);
 
-	//! Convert a transient in-memory segment into a persistent segment blocked by an on-disk block.
+	//! Convert a transient in-memory segment to a persistent segment backed by an on-disk block.
 	//! Only used during checkpointing.
-	void ConvertToPersistent(optional_ptr<BlockManager> block_manager, block_id_t block_id);
+	void ConvertToPersistent(QueryContext context, optional_ptr<BlockManager> block_manager, const block_id_t block_id);
 	//! Updates pointers to refer to the given block and offset. This is only used
 	//! when sharing a block among segments. This is invoked only AFTER the block is written.
 	void MarkAsPersistent(shared_ptr<BlockHandle> block, uint32_t offset_in_block);
+	void SetBlock(shared_ptr<BlockHandle> block, uint32_t offset);
 	//! Gets a data pointer from a persistent column segment
 	DataPointer GetDataPointer();
 

package/src/duckdb/src/include/duckdb/storage/table/data_table_info.hpp

@@ -23,9 +23,9 @@ struct DataTableInfo {
 public:
 	DataTableInfo(AttachedDatabase &db, shared_ptr<TableIOManager> table_io_manager_p, string schema, string table);
 
-	//! Initialize any unknown indexes whose types might now be present after an extension load, optionally throwing an
-	//! exception if an index can't be initialized
-	void InitializeIndexes(ClientContext &context, const char *index_type = nullptr);
+	//! Bind unknown indexes throwing an exception if binding fails.
+	//! Only binds the specified index type, or all, if nullptr.
+	void BindIndexes(ClientContext &context, const char *index_type = nullptr);
 
 	//! Whether or not the table is temporary
 	bool IsTemporary() const;

package/src/duckdb/src/include/duckdb/storage/table/in_memory_checkpoint.hpp

@@ -0,0 +1,93 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/table/in_memory_checkpoint.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/storage/checkpoint/row_group_writer.hpp"
+#include "duckdb/storage/checkpoint/table_data_writer.hpp"
+#include "duckdb/storage/checkpoint_manager.hpp"
+
+namespace duckdb {
+
+class InMemoryCheckpointer final : public CheckpointWriter {
+public:
+	InMemoryCheckpointer(QueryContext context, AttachedDatabase &db, BlockManager &block_manager,
+	                     StorageManager &storage_manager, CheckpointType checkpoint_type);
+
+	void CreateCheckpoint() override;
+
+	MetadataWriter &GetMetadataWriter() override;
+	MetadataManager &GetMetadataManager() override;
+	unique_ptr<TableDataWriter> GetTableDataWriter(TableCatalogEntry &table) override;
+	optional_ptr<ClientContext> GetClientContext() const {
+		return context;
+	}
+	CheckpointType GetCheckpointType() const {
+		return checkpoint_type;
+	}
+	PartialBlockManager &GetPartialBlockManager() {
+		return partial_block_manager;
+	}
+
+public:
+	void WriteTable(TableCatalogEntry &table, Serializer &serializer) override;
+
+private:
+	optional_ptr<ClientContext> context;
+	PartialBlockManager partial_block_manager;
+	StorageManager &storage_manager;
+	CheckpointType checkpoint_type;
+};
+
+class InMemoryTableDataWriter : public TableDataWriter {
+public:
+	InMemoryTableDataWriter(InMemoryCheckpointer &checkpoint_manager, TableCatalogEntry &table);
+
+public:
+	void WriteUnchangedTable(MetaBlockPointer pointer, idx_t total_rows) override;
+	void FinalizeTable(const TableStatistics &global_stats, DataTableInfo &info, RowGroupCollection &collection,
+	                   Serializer &serializer) override;
+	unique_ptr<RowGroupWriter> GetRowGroupWriter(RowGroup &row_group) override;
+	CheckpointType GetCheckpointType() const override;
+	MetadataManager &GetMetadataManager() override;
+
+private:
+	InMemoryCheckpointer &checkpoint_manager;
+};
+
+class InMemoryRowGroupWriter : public RowGroupWriter {
+public:
+	InMemoryRowGroupWriter(TableCatalogEntry &table, PartialBlockManager &partial_block_manager,
+	                       InMemoryCheckpointer &checkpoint_manager);
+
+public:
+	CheckpointType GetCheckpointType() const override;
+	WriteStream &GetPayloadWriter() override;
+	MetaBlockPointer GetMetaBlockPointer() override;
+	optional_ptr<MetadataManager> GetMetadataManager() override;
+
+private:
+	//! Underlying writer object
+	InMemoryCheckpointer &checkpoint_manager;
+	// Nop metadata writer
+	MemoryStream metadata_writer;
+};
+
+struct InMemoryPartialBlock : public PartialBlock {
+public:
+	InMemoryPartialBlock(ColumnData &data, ColumnSegment &segment, PartialBlockState state,
+	                     BlockManager &block_manager);
+	~InMemoryPartialBlock() override;
+
+public:
+	void Flush(QueryContext context, const idx_t free_space_left) override;
+	void Merge(PartialBlock &other, idx_t offset, idx_t other_size) override;
+	void AddSegmentToTail(ColumnData &data, ColumnSegment &segment, uint32_t offset_in_block) override;
+	void Clear() override;
+};
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/list_column_data.hpp

@@ -46,10 +46,10 @@ public:
 	idx_t Fetch(ColumnScanState &state, row_t row_id, Vector &result) override;
 	void FetchRow(TransactionData transaction, ColumnFetchState &state, row_t row_id, Vector &result,
 	              idx_t result_idx) override;
-	void Update(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids,
-	            idx_t update_count) override;
-	void UpdateColumn(TransactionData transaction, const vector<column_t> &column_path, Vector &update_vector,
-	                  row_t *row_ids, idx_t update_count, idx_t depth) override;
+	void Update(TransactionData transaction, DataTable &data_table, idx_t column_index, Vector &update_vector,
+	            row_t *row_ids, idx_t update_count) override;
+	void UpdateColumn(TransactionData transaction, DataTable &data_table, const vector<column_t> &column_path,
+	                  Vector &update_vector, row_t *row_ids, idx_t update_count, idx_t depth) override;
 	unique_ptr<BaseStatistics> GetUpdateStatistics() override;
 
 	void CommitDropColumn() override;
@@ -59,6 +59,7 @@ public:
 	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, ColumnCheckpointInfo &info) override;
 
 	bool IsPersistent() override;
+	bool HasAnyChanges() const override;
 	PersistentColumnData Serialize() override;
 	void InitializeColumn(PersistentColumnData &column_data, BaseStatistics &target_stats) override;
 

package/src/duckdb/src/include/duckdb/storage/table/persistent_table_data.hpp

@@ -22,6 +22,7 @@ public:
 	explicit PersistentTableData(idx_t column_count);
 	~PersistentTableData();
 
+	MetaBlockPointer base_table_pointer;
 	TableStatistics table_stats;
 	idx_t total_rows;
 	idx_t row_group_count;