duckdb 1.3.4 → 1.4.0

package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp

@@ -16,12 +16,30 @@
 #include "duckdb/common/set.hpp"
 #include "duckdb/common/vector.hpp"
 #include "duckdb/main/config.hpp"
+#include "duckdb/common/encryption_functions.hpp"
 
 namespace duckdb {
 
 class DatabaseInstance;
 struct MetadataHandle;
 
+struct EncryptionOptions {
+	//! indicates whether the db is encrypted
+	bool encryption_enabled = false;
+	//! Whether Additional Authenticated Data is used
+	bool additional_authenticated_data = false;
+	//! derived encryption key id
+	string derived_key_id;
+	// //! Cipher used for encryption
+	// EncryptionTypes::CipherType cipher = EncryptionTypes::CipherType::INVALID;
+	//! key derivation function (kdf) used
+	EncryptionTypes::KeyDerivationFunction kdf = EncryptionTypes::KeyDerivationFunction::SHA256;
+	//! Key Length
+	uint32_t key_length = MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH;
+	//! User key pointer (to StorageOptions)
+	shared_ptr<string> user_key;
+};
+
 struct StorageManagerOptions {
 	bool read_only = false;
 	bool use_direct_io = false;
@@ -30,13 +48,9 @@ struct StorageManagerOptions {
 	optional_idx storage_version;
 	optional_idx version_number;
 	optional_idx block_header_size;
-
-	//! indicates whether db is encrypted
-	bool encryption = false;
-
-	bool NeedsEncryption() const {
-		return encryption;
-	}
+	//! Unique database identifier and optional encryption salt.
+	data_t db_identifier[MainHeader::DB_IDENTIFIER_LEN];
+	EncryptionOptions encryption_options;
 };
 
 //! SingleFileBlockManager is an implementation for a BlockManager which manages blocks in a single file
@@ -45,14 +59,14 @@ class SingleFileBlockManager : public BlockManager {
 	static constexpr uint64_t BLOCK_START = Storage::FILE_HEADER_SIZE * 3;
 
 public:
-	SingleFileBlockManager(AttachedDatabase &db, const string &path, const StorageManagerOptions &options_p);
+	SingleFileBlockManager(AttachedDatabase &db_p, const string &path_p, const StorageManagerOptions &options_p);
 
 	FileOpenFlags GetFileFlags(bool create_new) const;
 	//! Creates a new database.
-	void CreateNewDatabase();
+	void CreateNewDatabase(QueryContext context);
 	//! Loads an existing database. We pass the provided block allocation size as a parameter
 	//! to detect inconsistencies with the file header.
-	void LoadExistingDatabase();
+	void LoadExistingDatabase(QueryContext context);
 
 	//! Creates a new Block using the specified block_id and returns a pointer
 	unique_ptr<Block> ConvertBlock(block_id_t block_id, FileBuffer &source_buffer) override;
@@ -74,13 +88,19 @@ public:
 	//! Return the meta block id
 	idx_t GetMetaBlock() override;
 	//! Read the content of the block from disk
-	void Read(Block &block) override;
+	void Read(QueryContext context, Block &block) override;
+
+	//! Read individual blocks
+	void ReadBlock(Block &block, bool skip_block_header = false) const;
+	void ReadBlock(data_ptr_t internal_buffer, uint64_t block_size, bool skip_block_header = false) const;
 	//! Read the content of a range of blocks into a buffer
 	void ReadBlocks(FileBuffer &buffer, block_id_t start_block, idx_t block_count) override;
-	//! Write the given block to disk
-	void Write(FileBuffer &block, block_id_t block_id) override;
+	//! Write the block to disk. Use Write with client context instead.
+	void Write(FileBuffer &buffer, block_id_t block_id) override;
+	//! Write the block to disk.
+	void Write(QueryContext context, FileBuffer &buffer, block_id_t block_id) override;
 	//! Write the header to disk, this is the final step of the checkpointing process
-	void WriteHeader(DatabaseHeader header) override;
+	void WriteHeader(QueryContext context, DatabaseHeader header) override;
 	//! Sync changes to the underlying file
 	void FileSync() override;
 	//! Truncate the underlying database file after a checkpoint
@@ -96,17 +116,43 @@ public:
 	//! Whether or not the attached database is a remote file
 	bool IsRemote() override;
 
+	//! Return the checkpoint iteration of the file.
+	uint64_t GetCheckpointIteration() const {
+		return iteration_count;
+	}
+	//! Return the version number of the file.
+	uint64_t GetVersionNumber() const;
+	//! Return the database identifier.
+	data_ptr_t GetDBIdentifier() {
+		return options.db_identifier;
+	}
+
 private:
 	//! Loads the free list of the file.
-	void LoadFreeList();
+	void LoadFreeList(QueryContext context);
+
 	//! Initializes the database header. We pass the provided block allocation size as a parameter
 	//!	to detect inconsistencies with the file header.
 	void Initialize(const DatabaseHeader &header, const optional_idx block_alloc_size);
 
-	void ReadAndChecksum(FileBuffer &handle, uint64_t location, bool skip_block_header = false) const;
-	void ChecksumAndWrite(FileBuffer &handle, uint64_t location, bool skip_block_header = false) const;
+	void CheckChecksum(FileBuffer &block, uint64_t location, uint64_t delta, bool skip_block_header = false) const;
+	void CheckChecksum(data_ptr_t start_ptr, uint64_t delta, bool skip_block_header = false) const;
+
+	void ReadAndChecksum(QueryContext context, FileBuffer &handle, uint64_t location,
+	                     bool skip_block_header = false) const;
+	void ChecksumAndWrite(QueryContext context, FileBuffer &handle, uint64_t location,
+	                      bool skip_block_header = false) const;
+
+	idx_t GetBlockLocation(block_id_t block_id) const;
+
+	// Encrypt, Store, Decrypt the canary
+	static void StoreEncryptedCanary(AttachedDatabase &db, MainHeader &main_header, const string &key_id);
+	static void StoreDBIdentifier(MainHeader &main_header, const data_ptr_t db_identifier);
+	void StoreEncryptionMetadata(MainHeader &main_header) const;
 
-	idx_t GetBlockLocation(block_id_t block_id);
+	//! Check and adding Encryption Keys
+	void CheckAndAddEncryptionKey(MainHeader &main_header, string &user_key);
+	void CheckAndAddEncryptionKey(MainHeader &main_header);
 
 	//! Return the blocks to which we will write the free list and modified blocks
 	vector<MetadataHandle> GetFreeListBlocks();
@@ -118,7 +164,6 @@ private:
 	void VerifyBlocks(const unordered_map<block_id_t, idx_t> &block_usage_count) override;
 
 	void AddStorageVersionTag();
-	uint64_t GetVersionNumber();
 
 private:
 	AttachedDatabase &db;
@@ -146,7 +191,7 @@ private:
 	block_id_t max_block;
 	//! The block id where the free list can be found
 	idx_t free_list_id;
-	//! The current header iteration count
+	//! The current header iteration count.
 	uint64_t iteration_count;
 	//! The storage manager options
 	StorageManagerOptions options;

package/src/duckdb/src/include/duckdb/storage/standard_buffer_manager.hpp

@@ -40,9 +40,6 @@ public:
 
 public:
 	static unique_ptr<StandardBufferManager> CreateBufferManager(DatabaseInstance &db, string temp_directory);
-	static unique_ptr<FileBuffer> ReadTemporaryBufferInternal(BufferManager &buffer_manager, FileHandle &handle,
-	                                                          idx_t position, idx_t size,
-	                                                          unique_ptr<FileBuffer> reusable_buffer);
 
 	//! Registers a transient memory buffer.
 	shared_ptr<BlockHandle> RegisterTransientMemory(const idx_t size, BlockManager &block_manager) final;
@@ -59,7 +56,7 @@ public:
 	idx_t GetBlockAllocSize() const final;
 	//! Returns the block size for buffer-managed blocks.
 	idx_t GetBlockSize() const final;
-	idx_t GetTemporaryBlockHeaderSize() const final;
+	idx_t GetQueryMaxMemory() const final;
 
 	//! Allocate an in-memory buffer with a single pin.
 	//! The allocated memory is released when the buffer handle is destroyed.
@@ -72,8 +69,10 @@ public:
 
 	//! Reallocate an in-memory buffer that is pinned.
 	void ReAllocate(shared_ptr<BlockHandle> &handle, idx_t block_size) final;
-
 	BufferHandle Pin(shared_ptr<BlockHandle> &handle) final;
+
+	BufferHandle Pin(QueryContext context, shared_ptr<BlockHandle> &handle);
+
 	void Prefetch(vector<shared_ptr<BlockHandle>> &handles) final;
 	void Unpin(shared_ptr<BlockHandle> &handle) final;
 
@@ -107,6 +106,7 @@ public:
 	DUCKDB_API void ReserveMemory(idx_t size) final;
 	DUCKDB_API void FreeReservedMemory(idx_t size) final;
 	bool HasTemporaryDirectory() const final;
+	bool HasFilesInTemporaryDirectory() const final;
 
 protected:
 	//! Helper
@@ -137,7 +137,7 @@ protected:
 	//! Write a temporary buffer to disk
 	void WriteTemporaryBuffer(MemoryTag tag, block_id_t block_id, FileBuffer &buffer) final;
 	//! Read a temporary buffer from disk
-	unique_ptr<FileBuffer> ReadTemporaryBuffer(MemoryTag tag, BlockHandle &block,
+	unique_ptr<FileBuffer> ReadTemporaryBuffer(QueryContext context, MemoryTag tag, BlockHandle &block,
 	                                           unique_ptr<FileBuffer> buffer = nullptr) final;
 	//! Get the path of the temporary buffer
 	string GetTemporaryPath(block_id_t id);

package/src/duckdb/src/include/duckdb/storage/statistics/base_statistics.hpp

@@ -83,7 +83,7 @@ public:
 	inline void SetHasNullFast() {
 		has_null = true;
 	}
-	//! Set that the CURRENT level can have valiod values
+	//! Set that the CURRENT level can have valid values
 	//! Note that this is not correct for nested types unless this information is propagated in a different manner
 	//! Use Set(StatsInfo::CAN_HAVE_VALID_VALUES) in the general case
 	inline void SetHasNoNullFast() {
@@ -104,7 +104,7 @@ public:
 	static BaseStatistics Deserialize(Deserializer &deserializer);
 
 	//! Verify that a vector does not violate the statistics
-	void Verify(Vector &vector, const SelectionVector &sel, idx_t count) const;
+	void Verify(Vector &vector, const SelectionVector &sel, idx_t count, bool ignore_has_null = false) const;
 	void Verify(Vector &vector, idx_t count) const;
 
 	string ToString() const;

package/src/duckdb/src/include/duckdb/storage/storage_extension.hpp

@@ -25,10 +25,10 @@ struct StorageExtensionInfo {
 	}
 };
 
-typedef unique_ptr<Catalog> (*attach_function_t)(StorageExtensionInfo *storage_info, ClientContext &context,
-                                                 AttachedDatabase &db, const string &name, AttachInfo &info,
-                                                 AccessMode access_mode);
-typedef unique_ptr<TransactionManager> (*create_transaction_manager_t)(StorageExtensionInfo *storage_info,
+typedef unique_ptr<Catalog> (*attach_function_t)(optional_ptr<StorageExtensionInfo> storage_info,
+                                                 ClientContext &context, AttachedDatabase &db, const string &name,
+                                                 AttachInfo &info, AttachOptions &options);
+typedef unique_ptr<TransactionManager> (*create_transaction_manager_t)(optional_ptr<StorageExtensionInfo> storage_info,
                                                                        AttachedDatabase &db, Catalog &catalog);
 
 class StorageExtension {

package/src/duckdb/src/include/duckdb/storage/storage_info.hpp

@@ -12,9 +12,12 @@
 #include "duckdb/common/limits.hpp"
 #include "duckdb/common/string.hpp"
 #include "duckdb/common/vector_size.hpp"
+#include "duckdb/common/encryption_state.hpp"
 
 namespace duckdb {
+
 struct FileHandle;
+class QueryContext;
 
 //! The standard row group size
 #define DEFAULT_ROW_GROUP_SIZE 122880ULL
@@ -25,17 +28,14 @@ struct FileHandle;
 
 //! The default block allocation size.
 #define DEFAULT_BLOCK_ALLOC_SIZE 262144ULL
-//! The configurable block allocation size.
-#ifndef DUCKDB_BLOCK_ALLOC_SIZE
-#define DUCKDB_BLOCK_ALLOC_SIZE DEFAULT_BLOCK_ALLOC_SIZE
-#endif
 //! The default block header size.
 #define DEFAULT_BLOCK_HEADER_STORAGE_SIZE 8ULL
 //! The default block header size.
 #define DEFAULT_ENCRYPTION_BLOCK_HEADER_SIZE 40ULL
 //! The configurable block allocation size.
 #ifndef DUCKDB_BLOCK_HEADER_STORAGE_SIZE
-#define DUCKDB_BLOCK_HEADER_STORAGE_SIZE DEFAULT_BLOCK_HEADER_STORAGE_SIZE
+#define DUCKDB_BLOCK_HEADER_STORAGE_SIZE     DEFAULT_BLOCK_HEADER_STORAGE_SIZE
+#define DEFAULT_ENCRYPTED_BUFFER_HEADER_SIZE 32ULL
 #endif
 
 using block_id_t = int64_t;
@@ -56,8 +56,8 @@ struct Storage {
 	constexpr static idx_t DEFAULT_BLOCK_HEADER_SIZE = sizeof(idx_t);
 	//! The default block header size for blocks written to storage.
 	constexpr static idx_t MAX_BLOCK_HEADER_SIZE = 128ULL;
-	//! Block header size for encrypted blocks (40 bytes)
-	constexpr static idx_t ENCRYPTED_BLOCK_HEADER_SIZE = 40ULL;
+	//! Block header size for encrypted blocks (64 bytes)
+	constexpr static idx_t ENCRYPTED_BLOCK_HEADER_SIZE = 64ULL;
 	//! The default block size.
 	constexpr static idx_t DEFAULT_BLOCK_SIZE = DEFAULT_BLOCK_ALLOC_SIZE - DEFAULT_BLOCK_HEADER_SIZE;
 
@@ -70,27 +70,47 @@ struct Storage {
 extern const uint64_t VERSION_NUMBER;
 extern const uint64_t VERSION_NUMBER_LOWER;
 extern const uint64_t VERSION_NUMBER_UPPER;
-string GetDuckDBVersion(idx_t version_number);
+string GetDuckDBVersions(const idx_t version_number);
 optional_idx GetStorageVersion(const char *version_string);
-string GetStorageVersionName(idx_t serialization_version);
+string GetStorageVersionName(const idx_t serialization_version, const bool add_suffix);
 optional_idx GetSerializationVersion(const char *version_string);
 vector<string> GetSerializationCandidates();
 
-//! The MainHeader is the first header in the storage file. The MainHeader is typically written only once for a database
-//! file.
-struct MainHeader {
+//! The MainHeader is the first header in the storage file.
+//! It is written only once for a database file.
+class MainHeader {
+public:
 	static constexpr idx_t MAX_VERSION_SIZE = 32;
 	static constexpr idx_t MAGIC_BYTE_SIZE = 4;
 	static constexpr idx_t MAGIC_BYTE_OFFSET = Storage::DEFAULT_BLOCK_HEADER_SIZE;
 	static constexpr idx_t FLAG_COUNT = 4;
+
+	//! Indicates whether database is encrypted or not.
 	static constexpr uint64_t ENCRYPTED_DATABASE_FLAG = 1;
-	//! The magic bytes in front of the file should be "DUCK"
+	//! The encryption key length.
+	static constexpr uint64_t DEFAULT_ENCRYPTION_KEY_LENGTH = 32;
+	//! The magic bytes in front of the file should be "DUCK".
 	static const char MAGIC_BYTES[];
-	//! The version of the database
+	//! The canary should be "DUCKKEY".
+	static const char CANARY[];
+
+	//! The (storage) version of the database.
 	uint64_t version_number;
-	//! The set of flags used by the database
+	//! The set of flags used by the database.
 	uint64_t flags[FLAG_COUNT];
-	static void CheckMagicBytes(FileHandle &handle);
+
+	//! The length of the unique database identifier.
+	static constexpr idx_t DB_IDENTIFIER_LEN = 16;
+	//! Optional metadata for encryption, if encryption flag is set.
+	static constexpr idx_t ENCRYPTION_METADATA_LEN = 8;
+	//! The canary is a known plaintext for detecting wrong keys early.
+	static constexpr idx_t CANARY_BYTE_SIZE = 8;
+	//! Nonce, IV (nonce + counter) and tag length
+	static constexpr uint64_t AES_NONCE_LEN = 16;
+	static constexpr uint64_t AES_IV_LEN = 16;
+	static constexpr uint64_t AES_TAG_LEN = 16;
+
+	static void CheckMagicBytes(QueryContext context, FileHandle &handle);
 
 	string LibraryGitDesc() {
 		return string(char_ptr_cast(library_git_desc), 0, MAX_VERSION_SIZE);
@@ -100,7 +120,46 @@ struct MainHeader {
 	}
 
 	bool IsEncrypted() const {
-		return flags[0] == MainHeader::ENCRYPTED_DATABASE_FLAG;
+		return flags[0] & MainHeader::ENCRYPTED_DATABASE_FLAG;
+	}
+	void SetEncrypted() {
+		flags[0] |= MainHeader::ENCRYPTED_DATABASE_FLAG;
+	}
+
+	void SetEncryptionMetadata(data_ptr_t source) {
+		memset(encryption_metadata, 0, ENCRYPTION_METADATA_LEN);
+		memcpy(encryption_metadata, source, ENCRYPTION_METADATA_LEN);
+	}
+
+	EncryptionTypes::CipherType GetEncryptionCipher() {
+		return static_cast<EncryptionTypes::CipherType>(encryption_metadata[2]);
+	}
+
+	void SetDBIdentifier(data_ptr_t source) {
+		memset(db_identifier, 0, DB_IDENTIFIER_LEN);
+		memcpy(db_identifier, source, DB_IDENTIFIER_LEN);
+	}
+
+	void SetEncryptedCanary(data_ptr_t source) {
+		memset(encrypted_canary, 0, CANARY_BYTE_SIZE);
+		memcpy(encrypted_canary, source, CANARY_BYTE_SIZE);
+	}
+
+	data_ptr_t GetDBIdentifier() {
+		return db_identifier;
+	}
+
+	static bool CompareDBIdentifiers(const data_ptr_t db_identifier_1, const data_ptr_t db_identifier_2) {
+		for (idx_t i = 0; i < DB_IDENTIFIER_LEN; i++) {
+			if (db_identifier_1[i] != db_identifier_2[i]) {
+				return false;
+			}
+		}
+		return true;
+	}
+
+	data_ptr_t GetEncryptedCanary() {
+		return encrypted_canary;
 	}
 
 	void Write(WriteStream &ser);
@@ -109,6 +168,10 @@ struct MainHeader {
 private:
 	data_t library_git_desc[MAX_VERSION_SIZE];
 	data_t library_git_hash[MAX_VERSION_SIZE];
+	data_t encryption_metadata[ENCRYPTION_METADATA_LEN];
+	//! The unique database identifier and optional encryption salt.
+	data_t db_identifier[DB_IDENTIFIER_LEN];
+	data_t encrypted_canary[CANARY_BYTE_SIZE];
 };
 
 //! The DatabaseHeader contains information about the current state of the database. Every storage file has two
@@ -147,8 +210,5 @@ struct DatabaseHeader {
 #if (DEFAULT_BLOCK_ALLOC_SIZE & (DEFAULT_BLOCK_ALLOC_SIZE - 1) != 0)
 #error The default block allocation size must be a power of two
 #endif
-#if (DUCKDB_BLOCK_ALLOC_SIZE & (DUCKDB_BLOCK_ALLOC_SIZE - 1) != 0)
-#error The duckdb block allocation size must be a power of two
-#endif
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/storage_manager.hpp

@@ -58,11 +58,10 @@ struct CheckpointOptions {
 	CheckpointType type;
 };
 
-//! StorageManager is responsible for managing the physical storage of the
-//! database on disk
+//! StorageManager is responsible for managing the physical storage of a persistent database.
 class StorageManager {
 public:
-	StorageManager(AttachedDatabase &db, string path, bool read_only);
+	StorageManager(AttachedDatabase &db, string path, const AttachOptions &options);
 	virtual ~StorageManager();
 
 public:
@@ -72,10 +71,10 @@ public:
 	//! Initialize a database or load an existing database from the database file path. The block_alloc_size is
 	//! either set, or invalid. If invalid, then DuckDB defaults to the default_block_alloc_size (DBConfig),
 	//! or the file's block allocation size, if it is an existing database.
-	void Initialize(StorageOptions options);
+	void Initialize(QueryContext context);
 
 	DatabaseInstance &GetDatabase();
-	AttachedDatabase &GetAttached() {
+	AttachedDatabase &GetAttached() const {
 		return db;
 	}
 
@@ -94,14 +93,13 @@ public:
 		return load_complete;
 	}
 	//! The path to the WAL, derived from the database file path
-	string GetWALPath();
-	bool InMemory();
+	string GetWALPath() const;
+	bool InMemory() const;
 
 	virtual bool AutomaticCheckpoint(idx_t estimated_wal_bytes) = 0;
 	virtual unique_ptr<StorageCommitState> GenStorageCommitState(WriteAheadLog &wal) = 0;
 	virtual bool IsCheckpointClean(MetaBlockPointer checkpoint_id) = 0;
-	virtual void CreateCheckpoint(optional_ptr<ClientContext> client_context,
-	                              CheckpointOptions options = CheckpointOptions()) = 0;
+	virtual void CreateCheckpoint(QueryContext context, CheckpointOptions options = CheckpointOptions()) = 0;
 	virtual DatabaseSize GetDatabaseSize() = 0;
 	virtual vector<MetadataBlockInfo> GetMetadataInfo() = 0;
 	virtual shared_ptr<TableIOManager> GetTableIOManager(BoundCreateTableInfo *info) = 0;
@@ -117,12 +115,34 @@ public:
 		D_ASSERT(HasStorageVersion());
 		return storage_version.GetIndex();
 	}
+	void AddInMemoryChange(idx_t size) {
+		in_memory_change_size += size;
+	}
+	void ResetInMemoryChange() {
+		in_memory_change_size = 0;
+	}
+	bool CompressionIsEnabled() const {
+		return storage_options.compress_in_memory == CompressInMemory::COMPRESS;
+	}
+	EncryptionTypes::CipherType GetCipher() const {
+		return storage_options.encryption_cipher;
+	}
+	void SetCipher(EncryptionTypes::CipherType cipher_p) {
+		D_ASSERT(cipher_p != EncryptionTypes::INVALID);
+		if (cipher_p == EncryptionTypes::CBC) {
+			throw InvalidInputException("CBC cipher is disabled");
+		}
+		storage_options.encryption_cipher = cipher_p;
+	}
+	bool IsEncrypted() const {
+		return storage_options.encryption;
+	}
 
 protected:
-	virtual void LoadDatabase(StorageOptions options) = 0;
+	virtual void LoadDatabase(QueryContext context) = 0;
 
 protected:
-	//! The database this storage manager belongs to
+	//! The attached database managed by this storage manager.
 	AttachedDatabase &db;
 	//! The path of the database
 	string path;
@@ -135,6 +155,10 @@ protected:
 	bool load_complete = false;
 	//! The serialization compatibility version when reading and writing from this database
 	optional_idx storage_version;
+	//! Estimated size of changes for determining automatic checkpointing on in-memory databases
+	atomic<idx_t> in_memory_change_size;
+	//! Storage options passed in through configuration
+	StorageOptions storage_options;
 
 public:
 	template <class TARGET>
@@ -149,28 +173,30 @@ public:
 	}
 };
 
-//! Stores database in a single file.
+//! Stores the database in a single file.
 class SingleFileStorageManager : public StorageManager {
 public:
 	SingleFileStorageManager() = delete;
-	SingleFileStorageManager(AttachedDatabase &db, string path, bool read_only);
+	SingleFileStorageManager(AttachedDatabase &db, string path, const AttachOptions &options);
 
-	//! The BlockManager to read/store meta information and data in blocks
+	//! The BlockManager to read from and write to blocks (meta data and data).
 	unique_ptr<BlockManager> block_manager;
-	//! TableIoManager
+	//! The table I/O manager.
 	unique_ptr<TableIOManager> table_io_manager;
 
 public:
 	bool AutomaticCheckpoint(idx_t estimated_wal_bytes) override;
 	unique_ptr<StorageCommitState> GenStorageCommitState(WriteAheadLog &wal) override;
 	bool IsCheckpointClean(MetaBlockPointer checkpoint_id) override;
-	void CreateCheckpoint(optional_ptr<ClientContext> client_context, CheckpointOptions options) override;
+	void CreateCheckpoint(QueryContext context, CheckpointOptions options) override;
 	DatabaseSize GetDatabaseSize() override;
 	vector<MetadataBlockInfo> GetMetadataInfo() override;
 	shared_ptr<TableIOManager> GetTableIOManager(BoundCreateTableInfo *info) override;
 	BlockManager &GetBlockManager() override;
 
 protected:
-	void LoadDatabase(StorageOptions options) override;
+	void LoadDatabase(QueryContext context) override;
+
+	unique_ptr<CheckpointWriter> CreateCheckpointWriter(QueryContext context, CheckpointOptions options);
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/storage_options.hpp

@@ -10,9 +10,13 @@
 
 #include "duckdb/common/common.hpp"
 #include "duckdb/common/optional_idx.hpp"
+#include "duckdb/common/types/value.hpp"
+#include "duckdb/common/encryption_state.hpp"
 
 namespace duckdb {
 
+enum class CompressInMemory { AUTOMATIC, COMPRESS, DO_NOT_COMPRESS };
+
 struct StorageOptions {
 	//! The allocation size of blocks for this attached database file (if any)
 	optional_idx block_alloc_size;
@@ -22,8 +26,25 @@ struct StorageOptions {
 	optional_idx storage_version;
 	//! Block header size (only used for encryption)
 	optional_idx block_header_size;
+
+	CompressInMemory compress_in_memory = CompressInMemory::AUTOMATIC;
+
 	//! Whether the database is encrypted
 	bool encryption = false;
+	//! Encryption algorithm
+	EncryptionTypes::CipherType encryption_cipher = EncryptionTypes::INVALID;
+	//! encryption key
+	//! FIXME: change to a unique_ptr in the future
+	shared_ptr<string> user_key;
+
+	void Initialize(const unordered_map<string, Value> &options);
 };
 
+inline void ClearUserKey(shared_ptr<string> const &encryption_key) {
+	if (encryption_key && !encryption_key->empty()) {
+		memset(&(*encryption_key)[0], 0, encryption_key->size());
+		encryption_key->clear();
+	}
+}
+
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/array_column_data.hpp

@@ -61,6 +61,7 @@ public:
 	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, ColumnCheckpointInfo &info) override;
 
 	bool IsPersistent() override;
+	bool HasAnyChanges() const override;
 	PersistentColumnData Serialize() override;
 	void InitializeColumn(PersistentColumnData &column_data, BaseStatistics &target_stats) override;
 

package/src/duckdb/src/include/duckdb/storage/table/column_checkpoint_state.hpp

@@ -85,9 +85,9 @@ public:
 
 public:
 	bool IsFlushed();
-	void Flush(const idx_t free_space_left) override;
+	void Flush(QueryContext context, const idx_t free_space_left) override;
 	void Merge(PartialBlock &other, idx_t offset, idx_t other_size) override;
-	void AddSegmentToTail(ColumnData &data, ColumnSegment &segment, uint32_t offset_in_block);
+	void AddSegmentToTail(ColumnData &data, ColumnSegment &segment, uint32_t offset_in_block) override;
 	void Clear() override;
 };
 

package/src/duckdb/src/include/duckdb/storage/table/column_data.hpp

@@ -102,6 +102,11 @@ public:
 	//! Whether or not the column has any updates
 	bool HasUpdates() const;
 	bool HasChanges(idx_t start_row, idx_t end_row) const;
+	//! Whether or not the column has changes at this level
+	bool HasChanges() const;
+
+	//! Whether or not the column has ANY changes, including in child columns
+	virtual bool HasAnyChanges() const;
 	//! Whether or not we can scan an entire vector
 	virtual ScanVectorType GetVectorScanType(ColumnScanState &state, idx_t scan_count, Vector &result);
 
@@ -214,6 +219,7 @@ protected:
 	void FetchUpdateRow(TransactionData transaction, row_t row_id, Vector &result, idx_t result_idx);
 	void UpdateInternal(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids,
 	                    idx_t update_count, Vector &base_vector);
+	idx_t FetchUpdateData(ColumnScanState &state, row_t *row_ids, Vector &base_vector);
 
 	idx_t GetVectorCount(idx_t vector_index) const;
 
@@ -232,7 +238,7 @@ protected:
 	//! The stats of the root segment
 	unique_ptr<SegmentStatistics> stats;
 	//! Total transient allocation size
-	idx_t allocation_size;
+	atomic<idx_t> allocation_size;
 
 private:
 	//! The parent column (if any)

package/src/duckdb/src/include/duckdb/storage/table/column_data_checkpointer.hpp

@@ -22,9 +22,10 @@ public:
 	ColumnDataCheckpointData() {
 	}
 	ColumnDataCheckpointData(ColumnCheckpointState &checkpoint_state, ColumnData &col_data, DatabaseInstance &db,
-	                         RowGroup &row_group, ColumnCheckpointInfo &checkpoint_info)
+	                         RowGroup &row_group, ColumnCheckpointInfo &checkpoint_info,
+	                         StorageManager &storage_manager)
 	    : checkpoint_state(checkpoint_state), col_data(col_data), db(db), row_group(row_group),
-	      checkpoint_info(checkpoint_info) {
+	      checkpoint_info(checkpoint_info), storage_manager(storage_manager) {
 	}
 
 public:
@@ -34,6 +35,7 @@ public:
 	RowGroup &GetRowGroup();
 	ColumnCheckpointState &GetCheckpointState();
 	DatabaseInstance &GetDatabase();
+	StorageManager &GetStorageManager();
 
 private:
 	optional_ptr<ColumnCheckpointState> checkpoint_state;
@@ -41,6 +43,7 @@ private:
 	optional_ptr<DatabaseInstance> db;
 	optional_ptr<RowGroup> row_group;
 	optional_ptr<ColumnCheckpointInfo> checkpoint_info;
+	optional_ptr<StorageManager> storage_manager;
 };
 
 struct CheckpointAnalyzeResult {