duckdb 1.0.1-dev21.0 → 1.0.1-dev27.0

package/src/duckdb/src/include/duckdb/storage/table/row_version_manager.hpp

@@ -21,7 +21,7 @@ struct MetaBlockPointer;
 
 class RowVersionManager {
 public:
-	explicit RowVersionManager(idx_t start);
+	explicit RowVersionManager(idx_t start) noexcept;
 
 	idx_t GetStart() {
 		return start;
@@ -37,6 +37,7 @@ public:
 	void AppendVersionInfo(TransactionData transaction, idx_t count, idx_t row_group_start, idx_t row_group_end);
 	void CommitAppend(transaction_t commit_id, idx_t row_group_start, idx_t count);
 	void RevertAppend(idx_t start_row);
+	void CleanupAppend(transaction_t lowest_active_transaction, idx_t row_group_start, idx_t count);
 
 	idx_t DeleteRows(idx_t vector_idx, transaction_t transaction_id, row_t rows[], idx_t count);
 	void CommitDelete(idx_t vector_idx, transaction_t commit_id, const DeleteInfo &info);

package/src/duckdb/src/include/duckdb/storage/table/scan_state.hpp

@@ -9,13 +9,15 @@
 #pragma once
 
 #include "duckdb/common/common.hpp"
+#include "duckdb/common/map.hpp"
 #include "duckdb/storage/buffer/buffer_handle.hpp"
 #include "duckdb/storage/storage_lock.hpp"
 #include "duckdb/common/enums/scan_options.hpp"
-#include "duckdb/execution/adaptive_filter.hpp"
 #include "duckdb/storage/table/segment_lock.hpp"
+#include "duckdb/common/types/data_chunk.hpp"
 
 namespace duckdb {
+class AdaptiveFilter;
 class ColumnSegment;
 class LocalTableStorage;
 class CollectionScanState;
@@ -31,6 +33,8 @@ class TableFilterSet;
 class ColumnData;
 class DuckTransaction;
 class RowGroupSegmentTree;
+class TableFilter;
+struct AdaptiveFilterState;
 struct TableScanOptions;
 
 struct SegmentScanState {
@@ -109,6 +113,60 @@ struct ColumnFetchState {
 	BufferHandle &GetOrInsertHandle(ColumnSegment &segment);
 };
 
+struct ScanFilter {
+	ScanFilter(idx_t index, const vector<column_t> &column_ids, TableFilter &filter);
+
+	idx_t scan_column_index;
+	idx_t table_column_index;
+	TableFilter &filter;
+	bool always_true;
+
+	bool IsAlwaysTrue() const {
+		return always_true;
+	}
+};
+
+class ScanFilterInfo {
+public:
+	~ScanFilterInfo();
+
+	void Initialize(TableFilterSet &filters, const vector<column_t> &column_ids);
+
+	const vector<ScanFilter> &GetFilterList() const {
+		return filter_list;
+	}
+
+	optional_ptr<AdaptiveFilter> GetAdaptiveFilter();
+	AdaptiveFilterState BeginFilter() const;
+	void EndFilter(AdaptiveFilterState state);
+
+	//! Whether or not there is any filter we need to execute
+	bool HasFilters() const;
+
+	//! Whether or not there is a filter we need to execute for this column currently
+	bool ColumnHasFilters(idx_t col_idx);
+
+	//! Resets any SetFilterAlwaysTrue flags
+	void CheckAllFilters();
+	//! Labels the filters for this specific column as always true
+	//! We do not need to execute them anymore until CheckAllFilters is called
+	void SetFilterAlwaysTrue(idx_t filter_idx);
+
+private:
+	//! The table filters (if any)
+	optional_ptr<TableFilterSet> table_filters;
+	//! Adaptive filter info (if any)
+	unique_ptr<AdaptiveFilter> adaptive_filter;
+	//! The set of filters
+	vector<ScanFilter> filter_list;
+	//! Whether or not the column has a filter active right now
+	unsafe_vector<bool> column_has_filter;
+	//! Whether or not the column has a filter active at all
+	unsafe_vector<bool> base_column_has_filter;
+	//! The amount of filters that are always true currently
+	idx_t always_true_filters = 0;
+};
+
 class CollectionScanState {
 public:
 	explicit CollectionScanState(TableScanState &parent_p);
@@ -127,12 +185,13 @@ public:
 	idx_t max_row;
 	//! The current batch index
 	idx_t batch_index;
+	//! The valid selection
+	SelectionVector valid_sel;
 
 public:
 	void Initialize(const vector<LogicalType> &types);
 	const vector<storage_t> &GetColumnIds();
-	TableFilterSet *GetFilters();
-	AdaptiveFilter *GetAdaptiveFilter();
+	ScanFilterInfo &GetFilterInfo();
 	TableScanOptions &GetOptions();
 	bool Scan(DuckTransaction &transaction, DataChunk &result);
 	bool ScanCommitted(DataChunk &result, TableScanType type);
@@ -143,13 +202,23 @@ private:
 };
 
 struct TableScanOptions {
-	//! Test config that forces fetching rows one by one instead of regular scans
+	//! Fetch rows one-at-a-time instead of using the regular scans.
 	bool force_fetch_row = false;
 };
 
+class CheckpointLock {
+public:
+	explicit CheckpointLock(unique_ptr<StorageLockKey> lock_p) : lock(std::move(lock_p)) {
+	}
+
+private:
+	unique_ptr<StorageLockKey> lock;
+};
+
 class TableScanState {
 public:
-	TableScanState() : table_state(*this), local_state(*this), table_filters(nullptr) {};
+	TableScanState();
+	~TableScanState();
 
 	//! The underlying table scan state
 	CollectionScanState table_state;
@@ -158,22 +227,20 @@ public:
 	//! Options for scanning
 	TableScanOptions options;
 	//! Shared lock over the checkpoint to prevent checkpoints while reading
-	unique_ptr<StorageLockKey> checkpoint_lock;
+	shared_ptr<CheckpointLock> checkpoint_lock;
+	//! Filter info
+	ScanFilterInfo filters;
 
 public:
-	void Initialize(vector<storage_t> column_ids, TableFilterSet *table_filters = nullptr);
+	void Initialize(vector<storage_t> column_ids, optional_ptr<TableFilterSet> table_filters = nullptr);
 
 	const vector<storage_t> &GetColumnIds();
-	TableFilterSet *GetFilters();
-	AdaptiveFilter *GetAdaptiveFilter();
+
+	ScanFilterInfo &GetFilterInfo();
 
 private:
 	//! The column identifiers of the scan
 	vector<storage_t> column_ids;
-	//! The table filters (if any)
-	TableFilterSet *table_filters;
-	//! Adaptive filter info (if any)
-	unique_ptr<AdaptiveFilter> adaptive_filter;
 };
 
 struct ParallelCollectionScanState {
@@ -195,7 +262,15 @@ struct ParallelTableScanState {
 	//! Parallel scan state for the transaction-local state
 	ParallelCollectionScanState local_state;
 	//! Shared lock over the checkpoint to prevent checkpoints while reading
-	unique_ptr<StorageLockKey> checkpoint_lock;
+	shared_ptr<CheckpointLock> checkpoint_lock;
+};
+
+struct PrefetchState {
+	~PrefetchState();
+
+	void AddBlock(shared_ptr<BlockHandle> block);
+
+	vector<shared_ptr<BlockHandle>> blocks;
 };
 
 class CreateIndexScanState : public TableScanState {

package/src/duckdb/src/include/duckdb/storage/table/standard_column_data.hpp

@@ -24,9 +24,9 @@ public:
 
 public:
 	void SetStart(idx_t new_start) override;
-	bool CheckZonemap(ColumnScanState &state, TableFilter &filter) override;
 
 	ScanVectorType GetVectorScanType(ColumnScanState &state, idx_t scan_count) override;
+	void InitializePrefetch(PrefetchState &prefetch_state, ColumnScanState &scan_state, idx_t rows) override;
 	void InitializeScan(ColumnScanState &state) override;
 	void InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx) override;
 
@@ -59,7 +59,9 @@ public:
 	void GetColumnSegmentInfo(duckdb::idx_t row_group_index, vector<duckdb::idx_t> col_path,
 	                          vector<duckdb::ColumnSegmentInfo> &result) override;
 
-	void DeserializeColumn(Deserializer &deserializer, BaseStatistics &target_stats) override;
+	bool IsPersistent() override;
+	PersistentColumnData Serialize() override;
+	void InitializeColumn(PersistentColumnData &column_data, BaseStatistics &target_stats) override;
 
 	void Verify(RowGroup &parent) override;
 };

package/src/duckdb/src/include/duckdb/storage/table/struct_column_data.hpp

@@ -26,9 +26,9 @@ public:
 
 public:
 	void SetStart(idx_t new_start) override;
-	bool CheckZonemap(ColumnScanState &state, TableFilter &filter) override;
 	idx_t GetMaxEntry() override;
 
+	void InitializePrefetch(PrefetchState &prefetch_state, ColumnScanState &scan_state, idx_t rows) override;
 	void InitializeScan(ColumnScanState &state) override;
 	void InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx) override;
 
@@ -58,7 +58,9 @@ public:
 	                                                        PartialBlockManager &partial_block_manager) override;
 	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, ColumnCheckpointInfo &info) override;
 
-	void DeserializeColumn(Deserializer &source, BaseStatistics &target_stats) override;
+	bool IsPersistent() override;
+	PersistentColumnData Serialize() override;
+	void InitializeColumn(PersistentColumnData &column_data, BaseStatistics &target_stats) override;
 
 	void GetColumnSegmentInfo(duckdb::idx_t row_group_index, vector<duckdb::idx_t> col_path,
 	                          vector<duckdb::ColumnSegmentInfo> &result) override;

package/src/duckdb/src/include/duckdb/storage/table/table_index_list.hpp

@@ -75,8 +75,8 @@ public:
 	Index *FindForeignKeyIndex(const vector<PhysicalIndex> &fk_keys, ForeignKeyType fk_type);
 	void VerifyForeignKey(const vector<PhysicalIndex> &fk_keys, DataChunk &chunk, ConflictManager &conflict_manager);
 
-	//! Serialize all indexes of this table
-	vector<IndexStorageInfo> GetStorageInfos();
+	//! Serialize all indexes of this table.
+	vector<IndexStorageInfo> GetStorageInfos(const case_insensitive_map_t<Value> &options);
 
 	vector<column_t> GetRequiredColumns();
 

package/src/duckdb/src/include/duckdb/storage/table/validity_column_data.hpp

@@ -19,7 +19,7 @@ public:
 	                   ColumnData &parent);
 
 public:
-	bool CheckZonemap(ColumnScanState &state, TableFilter &filter) override;
+	FilterPropagateResult CheckZonemap(ColumnScanState &state, TableFilter &filter) override;
 	void AppendData(BaseStatistics &stats, ColumnAppendState &state, UnifiedVectorFormat &vdata, idx_t count) override;
 };
 

package/src/duckdb/src/include/duckdb/storage/temporary_memory_manager.hpp

@@ -30,14 +30,26 @@ public:
 	~TemporaryMemoryState();
 
 public:
-	//! Set the remaining size needed for this state, and updates the reservation
-	void SetRemainingSize(ClientContext &context, idx_t new_remaining_size);
+	//! Set the remaining size needed for this state (NOTE: does not update the reservation!)
+	void SetRemainingSize(idx_t new_remaining_size);
+	//! Set the remaining size needed for this state and update the reservation
+	void SetRemainingSizeAndUpdateReservation(ClientContext &context, idx_t new_remaining_size);
+	//! Set the remaining size to 0 (NOTE: updates the reservation to 0 as well)
+	void SetZero();
 	//! Get the remaining size that was set for this state
 	idx_t GetRemainingSize() const;
 	//! Set the minimum reservation for this state
 	void SetMinimumReservation(idx_t new_minimum_reservation);
+	//! Get the minimum reservation for this state
+	idx_t GetMinimumReservation() const;
+	//! Updates the reservation based on current remaining size
+	void UpdateReservation(ClientContext &context);
 	//! Get the reservation of this state
 	idx_t GetReservation() const;
+	//! Set the materialization penalty for this state
+	void SetMaterializationPenalty(idx_t new_materialization_penalty);
+	//! Get the materialization penalty for this state
+	idx_t GetMaterializationPenalty() const;
 
 private:
 	//! The TemporaryMemoryManager that owns this state
@@ -49,6 +61,8 @@ private:
 	atomic<idx_t> minimum_reservation;
 	//! How much memory this operator has reserved
 	atomic<idx_t> reservation;
+	//! The weight used for determining the reservation for this state
+	atomic<idx_t> materialization_penalty;
 };
 
 //! TemporaryMemoryManager is a one-of class owned by the buffer pool that tries to dynamically assign memory
@@ -62,16 +76,18 @@ public:
 	TemporaryMemoryManager();
 
 private:
-	//! TemporaryMemoryState is initialized with a minimum reservation guarantee, which is either
-	//! 512 blocks per state per thread, which is 0.125GB per thread for Storage::BLOCK_ALLOC_SIZE = 262144
-	static constexpr const idx_t MINIMUM_RESERVATION_PER_STATE_PER_THREAD = idx_t(512) * Storage::BLOCK_ALLOC_SIZE;
-	//! Or 1/16th of main memory, if that is lower
-	static constexpr const idx_t MINIMUM_RESERVATION_MEMORY_LIMIT_DIVISOR = 16;
+	//! TemporaryMemoryState is initialized with a minimum reservation guarantee, which is the minimum of
+	//! MINIMUM_RESERVATION_PER_STATE_PER_THREAD and MINIMUM_RESERVATION_MEMORY_LIMIT_DIVISOR.
+
+	//! 512 blocks per state per thread, which is 0.125GB per thread for DEFAULT_BLOCK_ALLOC_SIZE.
+	static constexpr idx_t MINIMUM_RESERVATION_PER_STATE_PER_THREAD = 512ULL * DEFAULT_BLOCK_ALLOC_SIZE;
+	//! 1/16th of the available main memory.
+	static constexpr idx_t MINIMUM_RESERVATION_MEMORY_LIMIT_DIVISOR = 16ULL;
 
 	//! The maximum ratio of the memory limit that we reserve using the TemporaryMemoryManager
-	static constexpr const double MAXIMUM_MEMORY_LIMIT_RATIO = 0.8;
+	static constexpr double MAXIMUM_MEMORY_LIMIT_RATIO = 0.8;
 	//! The maximum ratio of the remaining memory that we reserve per TemporaryMemoryState
-	static constexpr const double MAXIMUM_FREE_MEMORY_RATIO = double(2) / double(3);
+	static constexpr double MAXIMUM_FREE_MEMORY_RATIO = 2.0 / 3.0;
 
 public:
 	//! Get the TemporaryMemoryManager
@@ -82,6 +98,8 @@ public:
 private:
 	//! Locks the TemporaryMemoryManager
 	unique_lock<mutex> Lock();
+	//! Unregister a TemporaryMemoryState (called by the destructor of TemporaryMemoryState)
+	void Unregister(TemporaryMemoryState &temporary_memory_state);
 	//! Update memory_limit, has_temporary_directory, and num_threads (must hold the lock)
 	void UpdateConfiguration(ClientContext &context);
 	//! Update the TemporaryMemoryState to the new remaining size, and updates the reservation (must hold the lock)
@@ -90,8 +108,8 @@ private:
 	void SetRemainingSize(TemporaryMemoryState &temporary_memory_state, idx_t new_remaining_size);
 	//! Set the reservation of a TemporaryMemoryState (must hold the lock)
 	void SetReservation(TemporaryMemoryState &temporary_memory_state, idx_t new_reservation);
-	//! Unregister a TemporaryMemoryState (called by the destructor of TemporaryMemoryState)
-	void Unregister(TemporaryMemoryState &temporary_memory_state);
+	//! Computes optimal reservation of a TemporaryMemoryState based on a cost function
+	idx_t ComputeReservation(const TemporaryMemoryState &temporary_memory_state) const;
 	//! Verify internal counts (must hold the lock)
 	void Verify() const;
 
@@ -100,13 +118,13 @@ private:
 	mutex lock;
 
 	//! Memory limit of the buffer pool
-	idx_t memory_limit;
+	idx_t memory_limit = DConstants::INVALID_INDEX;
 	//! Whether there is a temporary directory that we can offload blocks to
-	bool has_temporary_directory;
+	bool has_temporary_directory = false;
 	//! Number of threads
-	idx_t num_threads;
+	idx_t num_threads = DConstants::INVALID_INDEX;
 	//! Max memory per query
-	idx_t query_max_memory;
+	idx_t query_max_memory = DConstants::INVALID_INDEX;
 
 	//! Currently active states
 	reference_set_t<TemporaryMemoryState> active_states;

package/src/duckdb/src/include/duckdb/storage/write_ahead_log.hpp

@@ -36,6 +36,7 @@ class TableCatalogEntry;
 class Transaction;
 class TransactionManager;
 class WriteAheadLogDeserializer;
+struct PersistentCollectionData;
 
 //! The WriteAheadLog (WAL) is a log that is used to provide durability. Prior
 //! to committing a transaction it writes the changes the transaction made to
@@ -47,26 +48,21 @@ public:
 	explicit WriteAheadLog(AttachedDatabase &database, const string &wal_path);
 	virtual ~WriteAheadLog();
 
-	//! Skip writing to the WAL
-	bool skip_writing;
-
 public:
 	//! Replay the WAL
 	static bool Replay(AttachedDatabase &database, unique_ptr<FileHandle> handle);
 
+	//! Gets the total bytes written to the WAL since startup
+	idx_t GetWALSize();
 	//! Gets the total bytes written to the WAL since startup
 	idx_t GetTotalWritten();
 
 	//! A WAL is initialized, if a writer to a file exists.
 	bool Initialized() {
-		return writer != nullptr;
+		return initialized;
 	}
 	//! Initializes the file of the WAL by creating the file writer.
 	BufferedFileWriter &Initialize();
-	//! Returns the WAL file writer.
-	BufferedFileWriter &GetWriter() {
-		return *writer;
-	}
 
 	void WriteVersion();
 
@@ -100,6 +96,7 @@ public:
 	void WriteAlter(const AlterInfo &info);
 
 	void WriteInsert(DataChunk &chunk);
+	void WriteRowGroupData(const PersistentCollectionData &data);
 	void WriteDelete(DataChunk &chunk);
 	//! Write a single (sub-) column update to the WAL. Chunk must be a pair of (COL, ROW_ID).
 	//! The column_path vector is a *path* towards a column within the table
@@ -112,7 +109,7 @@ public:
 	void WriteUpdate(DataChunk &chunk, const vector<column_t> &column_path);
 
 	//! Truncate the WAL to a previous size, and clear anything currently set in the writer
-	void Truncate(int64_t size);
+	void Truncate(idx_t size);
 	//! Delete the WAL file on disk. The WAL should not be used after this point.
 	void Delete();
 	void Flush();
@@ -121,8 +118,11 @@ public:
 
 protected:
 	AttachedDatabase &database;
+	mutex wal_lock;
 	unique_ptr<BufferedFileWriter> writer;
 	string wal_path;
+	atomic<idx_t> wal_size;
+	atomic<bool> initialized;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/transaction/cleanup_state.hpp

@@ -21,7 +21,7 @@ struct UpdateInfo;
 
 class CleanupState {
 public:
-	CleanupState();
+	explicit CleanupState(transaction_t lowest_active_transaction);
 	~CleanupState();
 
 	// all tables with indexes that possibly need a vacuum (after e.g. a delete)
@@ -31,6 +31,8 @@ public:
 	void CleanupEntry(UndoFlags type, data_ptr_t data);
 
 private:
+	//! Lowest active transaction
+	transaction_t lowest_active_transaction;
 	// data for index cleanup
 	optional_ptr<DataTable> current_table;
 	DataChunk chunk;

package/src/duckdb/src/include/duckdb/transaction/commit_state.hpp

@@ -23,29 +23,17 @@ struct UpdateInfo;
 
 class CommitState {
 public:
-	explicit CommitState(transaction_t commit_id, optional_ptr<WriteAheadLog> log = nullptr);
-
-	optional_ptr<WriteAheadLog> log;
-	transaction_t commit_id;
-	UndoFlags current_op;
-
-	optional_ptr<DataTableInfo> current_table_info;
-	idx_t row_identifiers[STANDARD_VECTOR_SIZE];
-
-	unique_ptr<DataChunk> delete_chunk;
-	unique_ptr<DataChunk> update_chunk;
+	explicit CommitState(transaction_t commit_id);
 
 public:
-	template <bool HAS_LOG>
 	void CommitEntry(UndoFlags type, data_ptr_t data);
 	void RevertCommit(UndoFlags type, data_ptr_t data);
 
 private:
-	void SwitchTable(DataTableInfo *table, UndoFlags new_op);
+	void CommitEntryDrop(CatalogEntry &entry, data_ptr_t extra_data);
 
-	void WriteCatalogEntry(CatalogEntry &entry, data_ptr_t extra_data);
-	void WriteDelete(DeleteInfo &info);
-	void WriteUpdate(UpdateInfo &info);
+private:
+	transaction_t commit_id;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/transaction/duck_transaction.hpp

@@ -12,15 +12,19 @@
 #include "duckdb/common/reference_map.hpp"
 
 namespace duckdb {
+class CheckpointLock;
+class RowGroupCollection;
 class RowVersionManager;
 class DuckTransactionManager;
 class StorageLockKey;
+class StorageCommitState;
+struct DataTableInfo;
 struct UndoBufferProperties;
 
 class DuckTransaction : public Transaction {
 public:
 	DuckTransaction(DuckTransactionManager &manager, ClientContext &context, transaction_t start_time,
-	                transaction_t transaction_id);
+	                transaction_t transaction_id, idx_t catalog_version);
 	~DuckTransaction() override;
 
 	//! The start timestamp of this transaction
@@ -32,25 +36,30 @@ public:
 	//! Highest active query when the transaction finished, used for cleaning up
 	transaction_t highest_active_query;
 
+	atomic<idx_t> catalog_version;
+
 public:
 	static DuckTransaction &Get(ClientContext &context, AttachedDatabase &db);
 	static DuckTransaction &Get(ClientContext &context, Catalog &catalog);
 	LocalStorage &GetLocalStorage();
 
-	void PushCatalogEntry(CatalogEntry &entry, data_ptr_t extra_data = nullptr, idx_t extra_data_size = 0);
+	void PushCatalogEntry(CatalogEntry &entry, data_ptr_t extra_data, idx_t extra_data_size);
 
 	void SetReadWrite() override;
 
+	bool ShouldWriteToWAL(AttachedDatabase &db);
+	ErrorData WriteToWAL(AttachedDatabase &db, unique_ptr<StorageCommitState> &commit_state) noexcept;
 	//! Commit the current transaction with the given commit identifier. Returns an error message if the transaction
 	//! commit failed, or an empty string if the commit was sucessful
-	ErrorData Commit(AttachedDatabase &db, transaction_t commit_id, bool checkpoint) noexcept;
+	ErrorData Commit(AttachedDatabase &db, transaction_t commit_id,
+	                 unique_ptr<StorageCommitState> commit_state) noexcept;
 	//! Returns whether or not a commit of this transaction should trigger an automatic checkpoint
 	bool AutomaticCheckpoint(AttachedDatabase &db, const UndoBufferProperties &properties);
 
 	//! Rollback
 	void Rollback() noexcept;
 	//! Cleanup the undo buffer
-	void Cleanup();
+	void Cleanup(transaction_t lowest_active_transaction);
 
 	bool ChangesMade();
 	UndoBufferProperties GetUndoProperties();
@@ -66,6 +75,14 @@ public:
 	}
 
 	unique_ptr<StorageLockKey> TryGetCheckpointLock();
+	bool HasWriteLock() const {
+		return write_lock.get();
+	}
+
+	void UpdateCollection(shared_ptr<RowGroupCollection> &collection);
+
+	//! Get a shared lock on a table
+	shared_ptr<CheckpointLock> SharedLockTable(DataTableInfo &info);
 
 private:
 	DuckTransactionManager &transaction_manager;
@@ -80,6 +97,12 @@ private:
 	mutex sequence_lock;
 	//! Map of all sequences that were used during the transaction and the value they had in this transaction
 	reference_map_t<SequenceCatalogEntry, reference<SequenceValue>> sequence_usage;
+	//! Collections that are updated by this transaction
+	reference_map_t<RowGroupCollection, shared_ptr<RowGroupCollection>> updated_collections;
+	//! Lock for the active_locks map
+	mutex active_locks_lock;
+	//! Active locks on tables
+	reference_map_t<DataTableInfo, weak_ptr<CheckpointLock>> active_locks;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/transaction/duck_transaction_manager.hpp

@@ -52,6 +52,12 @@ public:
 	unique_ptr<StorageLockKey> SharedCheckpointLock();
 	unique_ptr<StorageLockKey> TryUpgradeCheckpointLock(StorageLockKey &lock);
 
+	//! Returns the current version of the catalog (incremented whenever anything changes, not stored between restarts)
+	DUCKDB_API idx_t GetCatalogVersion(Transaction &transaction);
+
+	void PushCatalogEntry(Transaction &transaction_p, CatalogEntry &entry, data_ptr_t extra_data = nullptr,
+	                      idx_t extra_data_size = 0);
+
 protected:
 	struct CheckpointDecision {
 		explicit CheckpointDecision(string reason_p);
@@ -98,6 +104,11 @@ private:
 	StorageLock checkpoint_lock;
 	//! Lock necessary to start transactions only - used by FORCE CHECKPOINT to prevent new transactions from starting
 	mutex start_transaction_lock;
+	//! Mutex used to control writes to the WAL - separate from the transaction lock
+	mutex wal_lock;
+
+	atomic<idx_t> last_uncommitted_catalog_version = {TRANSACTION_ID_START};
+	idx_t last_committed_version = 0;
 
 protected:
 	virtual void OnCommitCheckpointDecision(const CheckpointDecision &decision, DuckTransaction &transaction) {

package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp

@@ -18,6 +18,7 @@ namespace duckdb {
 class AttachedDatabase;
 class Catalog;
 class DataTable;
+class StorageCommitState;
 class Transaction;
 class WriteAheadLog;
 struct LocalAppendState;
@@ -52,6 +53,8 @@ public:
 	vector<unique_ptr<OptimisticDataWriter>> optimistic_writers;
 	//! Whether or not storage was merged
 	bool merged_storage = false;
+	//! Whether or not the storage was dropped
+	bool is_dropped = false;
 
 public:
 	void InitializeScan(CollectionScanState &state, optional_ptr<TableFilterSet> table_filters = nullptr);
@@ -134,13 +137,14 @@ public:
 	void Update(DataTable &table, Vector &row_ids, const vector<PhysicalIndex> &column_ids, DataChunk &data);
 
 	//! Commits the local storage, writing it to the WAL and completing the commit
-	void Commit(LocalStorage::CommitState &commit_state, DuckTransaction &transaction);
+	void Commit(optional_ptr<StorageCommitState> commit_state);
 	//! Rollback the local storage
 	void Rollback();
 
 	bool ChangesMade() noexcept;
 	idx_t EstimatedSize();
 
+	void DropTable(DataTable &table);
 	bool Find(DataTable &table);
 
 	idx_t AddedRows(DataTable &table);
@@ -163,7 +167,7 @@ private:
 	DuckTransaction &transaction;
 	LocalTableManager table_manager;
 
-	void Flush(DataTable &table, LocalTableStorage &storage);
+	void Flush(DataTable &table, LocalTableStorage &storage, optional_ptr<StorageCommitState> commit_state);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/transaction/meta_transaction.hpp

@@ -25,17 +25,13 @@ class Transaction;
 //! The MetaTransaction manages multiple transactions for different attached databases
 class MetaTransaction {
 public:
-	DUCKDB_API MetaTransaction(ClientContext &context, timestamp_t start_timestamp, idx_t catalog_version);
+	DUCKDB_API MetaTransaction(ClientContext &context, timestamp_t start_timestamp);
 
 	ClientContext &context;
 	//! The timestamp when the transaction started
 	timestamp_t start_timestamp;
-	//! The catalog version when the transaction was started
-	idx_t catalog_version;
 	//! The validity checker of the transaction
 	ValidChecker transaction_validity;
-	//! Whether or not any transaction have made modifications
-	bool read_only;
 	//! The active query number
 	transaction_t active_query;
 
@@ -55,6 +51,8 @@ public:
 	idx_t GetActiveQuery();
 	void SetActiveQuery(transaction_t query_number);
 
+	void SetReadOnly();
+	bool IsReadOnly() const;
 	void ModifyDatabase(AttachedDatabase &db);
 	optional_ptr<AttachedDatabase> ModifiedDatabase() {
 		return modified_database;
@@ -72,6 +70,8 @@ private:
 	vector<reference<AttachedDatabase>> all_transactions;
 	//! The database we are modifying - we can only modify one database per transaction
 	optional_ptr<AttachedDatabase> modified_database;
+	//! Whether or not the meta transaction is marked as read only
+	bool is_read_only;
 };
 
 } // namespace duckdb