duckdb 0.10.3-dev3.0 → 0.10.3-dev6.0

package/src/duckdb/src/include/duckdb/storage/table/scan_state.hpp

@@ -157,6 +157,8 @@ public:
 	CollectionScanState local_state;
 	//! Options for scanning
 	TableScanOptions options;
+	//! Shared lock over the checkpoint to prevent checkpoints while reading
+	unique_ptr<StorageLockKey> checkpoint_lock;
 
 public:
 	void Initialize(vector<storage_t> column_ids, TableFilterSet *table_filters = nullptr);
@@ -192,6 +194,8 @@ struct ParallelTableScanState {
 	ParallelCollectionScanState scan_state;
 	//! Parallel scan state for the transaction-local state
 	ParallelCollectionScanState local_state;
+	//! Shared lock over the checkpoint to prevent checkpoints while reading
+	unique_ptr<StorageLockKey> checkpoint_lock;
 };
 
 class CreateIndexScanState : public TableScanState {

package/src/duckdb/src/include/duckdb/storage/table/segment_tree.hpp

@@ -83,11 +83,11 @@ public:
 		if (index < 0) {
 			// load all segments
 			LoadAllSegments(l);
-			index = nodes.size() + index;
+			index += nodes.size();
 			if (index < 0) {
 				return nullptr;
 			}
-			return nodes[index].node.get();
+			return nodes[UnsafeNumericCast<idx_t>(index)].node.get();
 		} else {
 			// lazily load segments until we reach the specific segment
 			while (idx_t(index) >= nodes.size() && LoadNextSegment(l)) {
@@ -95,7 +95,7 @@ public:
 			if (idx_t(index) >= nodes.size()) {
 				return nullptr;
 			}
-			return nodes[index].node.get();
+			return nodes[UnsafeNumericCast<idx_t>(index)].node.get();
 		}
 	}
 	//! Gets the next segment
@@ -116,7 +116,7 @@ public:
 #ifdef DEBUG
 		D_ASSERT(nodes[segment->index].node.get() == segment);
 #endif
-		return GetSegmentByIndex(l, segment->index + 1);
+		return GetSegmentByIndex(l, UnsafeNumericCast<int64_t>(segment->index + 1));
 	}
 
 	//! Gets a pointer to the last segment. Useful for appends.
@@ -182,7 +182,7 @@ public:
 		if (segment_start >= nodes.size() - 1) {
 			return;
 		}
-		nodes.erase(nodes.begin() + segment_start + 1, nodes.end());
+		nodes.erase(nodes.begin() + UnsafeNumericCast<int64_t>(segment_start) + 1, nodes.end());
 	}
 
 	//! Get the segment index of the column segment for the given row

package/src/duckdb/src/include/duckdb/storage/table/standard_column_data.hpp

@@ -23,16 +23,17 @@ public:
 	ValidityColumnData validity;
 
 public:
-	bool HasUpdates() const override;
-
 	void SetStart(idx_t new_start) override;
 	bool CheckZonemap(ColumnScanState &state, TableFilter &filter) override;
 
+	ScanVectorType GetVectorScanType(ColumnScanState &state, idx_t scan_count) override;
 	void InitializeScan(ColumnScanState &state) override;
 	void InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx) override;
 
-	idx_t Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result) override;
-	idx_t ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates) override;
+	idx_t Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
+	           idx_t target_count) override;
+	idx_t ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates,
+	                    idx_t target_count) override;
 	idx_t ScanCount(ColumnScanState &state, Vector &result, idx_t count) override;
 
 	void InitializeAppend(ColumnAppendState &state) override;
@@ -51,8 +52,7 @@ public:
 
 	unique_ptr<ColumnCheckpointState> CreateCheckpointState(RowGroup &row_group,
 	                                                        PartialBlockManager &partial_block_manager) override;
-	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, PartialBlockManager &partial_block_manager,
-	                                             ColumnCheckpointInfo &checkpoint_info) override;
+	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, ColumnCheckpointInfo &info) override;
 	void CheckpointScan(ColumnSegment &segment, ColumnScanState &state, idx_t row_group_start, idx_t count,
 	                    Vector &scan_vector) override;
 

package/src/duckdb/src/include/duckdb/storage/table/struct_column_data.hpp

@@ -32,8 +32,10 @@ public:
 	void InitializeScan(ColumnScanState &state) override;
 	void InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx) override;
 
-	idx_t Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result) override;
-	idx_t ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates) override;
+	idx_t Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
+	           idx_t scan_count) override;
+	idx_t ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates,
+	                    idx_t scan_count) override;
 	idx_t ScanCount(ColumnScanState &state, Vector &result, idx_t count) override;
 
 	void Skip(ColumnScanState &state, idx_t count = STANDARD_VECTOR_SIZE) override;
@@ -54,8 +56,7 @@ public:
 
 	unique_ptr<ColumnCheckpointState> CreateCheckpointState(RowGroup &row_group,
 	                                                        PartialBlockManager &partial_block_manager) override;
-	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, PartialBlockManager &partial_block_manager,
-	                                             ColumnCheckpointInfo &checkpoint_info) override;
+	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, ColumnCheckpointInfo &info) override;
 
 	void DeserializeColumn(Deserializer &source, BaseStatistics &target_stats) override;
 

package/src/duckdb/src/include/duckdb/storage/table/table_index_list.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/common/mutex.hpp"
 #include "duckdb/storage/index.hpp"
+#include "duckdb/parser/constraint.hpp"
 
 namespace duckdb {
 
@@ -29,6 +30,29 @@ public:
 			}
 		}
 	}
+
+	//! Scan the indexes, invoking the callback method for every bound entry of a specific type
+	template <class T, class FUNC>
+	void ScanBound(FUNC &&callback) {
+		lock_guard<mutex> lock(indexes_lock);
+		for (auto &index : indexes) {
+			if (index->IsBound() && T::TYPE_NAME == index->GetIndexType()) {
+				if (callback(index->Cast<T>())) {
+					break;
+				}
+			}
+		}
+	}
+
+	// Bind any unbound indexes of the specified type and invoke the callback method for every bound entry of the
+	// specified type, regardless if it was bound before or not
+	template <class T, class FUNC>
+	void BindAndScan(ClientContext &context, DataTableInfo &table_info, FUNC &&callback) {
+		// FIXME: optimize this by only looping through the indexes once without re-acquiring the lock
+		InitializeIndexes(context, table_info, T::TYPE_NAME);
+		ScanBound<T>(callback);
+	}
+
 	//! Returns a reference to the indexes of this table
 	const vector<unique_ptr<Index>> &Indexes() const {
 		return indexes;
@@ -43,7 +67,7 @@ public:
 	bool NameIsUnique(const string &name);
 	//! Initializes unknown indexes that might now be present after an extension load, optionally throwing an exception
 	//! if a index cant be initialized
-	void InitializeIndexes(ClientContext &context, DataTableInfo &table_info, bool throw_on_failure = false);
+	void InitializeIndexes(ClientContext &context, DataTableInfo &table_info, const char *index_type = nullptr);
 	bool Empty();
 	idx_t Count();
 	void Move(TableIndexList &other);

package/src/duckdb/src/include/duckdb/storage/table/table_statistics.hpp

@@ -43,8 +43,11 @@ public:
 	void MergeStats(TableStatisticsLock &lock, idx_t i, BaseStatistics &stats);
 
 	void CopyStats(TableStatistics &other);
+	void CopyStats(TableStatisticsLock &lock, TableStatistics &other);
 	unique_ptr<BaseStatistics> CopyStats(idx_t i);
-	ColumnStatistics &GetStats(idx_t i);
+	//! Get a reference to the stats - this requires us to hold the lock.
+	//! The reference can only be safely accessed while the lock is held
+	ColumnStatistics &GetStats(TableStatisticsLock &lock, idx_t i);
 
 	bool Empty();
 
@@ -55,7 +58,7 @@ public:
 
 private:
 	//! The statistics lock
-	mutex stats_lock;
+	shared_ptr<mutex> stats_lock;
 	//! Column statistics
 	vector<shared_ptr<ColumnStatistics>> column_stats;
 	//! The table sample

package/src/duckdb/src/include/duckdb/storage/table/update_state.hpp

@@ -0,0 +1,20 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/table/update_state.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/storage/table/append_state.hpp"
+
+namespace duckdb {
+class TableCatalogEntry;
+
+struct TableUpdateState {
+	unique_ptr<ConstraintState> constraint_state;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/validity_column_data.hpp

@@ -20,6 +20,7 @@ public:
 
 public:
 	bool CheckZonemap(ColumnScanState &state, TableFilter &filter) override;
+	void AppendData(BaseStatistics &stats, ColumnAppendState &state, UnifiedVectorFormat &vdata, idx_t count) override;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/temporary_file_manager.hpp

@@ -23,8 +23,11 @@ namespace duckdb {
 // BlockIndexManager
 //===--------------------------------------------------------------------===//
 
+class TemporaryFileManager;
+
 struct BlockIndexManager {
 public:
+	explicit BlockIndexManager(TemporaryFileManager &manager);
 	BlockIndexManager();
 
 public:
@@ -37,12 +40,14 @@ public:
 	bool HasFreeBlocks();
 
 private:
+	void SetMaxIndex(idx_t blocks);
 	idx_t GetNewBlockIndexInternal();
 
 private:
 	idx_t max_index;
 	set<idx_t> free_indexes;
 	set<idx_t> indexes_in_use;
+	optional_ptr<TemporaryFileManager> manager;
 };
 
 //===--------------------------------------------------------------------===//
@@ -69,7 +74,8 @@ class TemporaryFileHandle {
 	constexpr static idx_t MAX_ALLOWED_INDEX_BASE = 4000;
 
 public:
-	TemporaryFileHandle(idx_t temp_file_count, DatabaseInstance &db, const string &temp_directory, idx_t index);
+	TemporaryFileHandle(idx_t temp_file_count, DatabaseInstance &db, const string &temp_directory, idx_t index,
+	                    TemporaryFileManager &manager);
 
 public:
 	struct TemporaryFileLock {
@@ -103,15 +109,13 @@ private:
 	BlockIndexManager index_manager;
 };
 
-class TemporaryFileManager;
-
 //===--------------------------------------------------------------------===//
 // TemporaryDirectoryHandle
 //===--------------------------------------------------------------------===//
 
 class TemporaryDirectoryHandle {
 public:
-	TemporaryDirectoryHandle(DatabaseInstance &db, string path_p);
+	TemporaryDirectoryHandle(DatabaseInstance &db, string path_p, optional_idx max_swap_space);
 	~TemporaryDirectoryHandle();
 
 	TemporaryFileManager &GetTempFile();
@@ -130,6 +134,7 @@ private:
 class TemporaryFileManager {
 public:
 	TemporaryFileManager(DatabaseInstance &db, const string &temp_directory_p);
+	~TemporaryFileManager();
 
 public:
 	struct TemporaryManagerLock {
@@ -145,6 +150,13 @@ public:
 	unique_ptr<FileBuffer> ReadTemporaryBuffer(block_id_t id, unique_ptr<FileBuffer> reusable_buffer);
 	void DeleteTemporaryBuffer(block_id_t id);
 	vector<TemporaryFileInformation> GetTemporaryFiles();
+	idx_t GetTotalUsedSpaceInBytes();
+	optional_idx GetMaxSwapSpace() const;
+	void SetMaxSwapSpace(optional_idx limit);
+	//! Register temporary file size growth
+	void IncreaseSizeOnDisk(idx_t amount);
+	//! Register temporary file size decrease
+	void DecreaseSizeOnDisk(idx_t amount);
 
 private:
 	void EraseUsedBlock(TemporaryManagerLock &lock, block_id_t id, TemporaryFileHandle *handle,
@@ -164,6 +176,10 @@ private:
 	unordered_map<block_id_t, TemporaryFileIndex> used_blocks;
 	//! Manager of in-use temporary file indexes
 	BlockIndexManager index_manager;
+	//! The size in bytes of the temporary files that are currently alive
+	atomic<idx_t> size_on_disk;
+	//! The max amount of disk space that can be used
+	idx_t max_swap_space;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/write_ahead_log.hpp

@@ -44,7 +44,7 @@ class WriteAheadLogDeserializer;
 class WriteAheadLog {
 public:
 	//! Initialize the WAL in the specified directory
-	explicit WriteAheadLog(AttachedDatabase &database, const string &path);
+	explicit WriteAheadLog(AttachedDatabase &database, const string &wal_path);
 	virtual ~WriteAheadLog();
 
 	//! Skip writing to the WAL
@@ -54,11 +54,16 @@ public:
 	//! Replay the WAL
 	static bool Replay(AttachedDatabase &database, unique_ptr<FileHandle> handle);
 
-	//! Returns the current size of the WAL in bytes
-	int64_t GetWALSize();
 	//! Gets the total bytes written to the WAL since startup
 	idx_t GetTotalWritten();
 
+	//! A WAL is initialized, if a writer to a file exists.
+	bool Initialized() {
+		return writer != nullptr;
+	}
+	//! Initializes the file of the WAL by creating the file writer.
+	BufferedFileWriter &Initialize();
+	//! Returns the WAL file writer.
 	BufferedFileWriter &GetWriter() {
 		return *writer;
 	}
@@ -76,7 +81,7 @@ public:
 
 	void WriteCreateSequence(const SequenceCatalogEntry &entry);
 	void WriteDropSequence(const SequenceCatalogEntry &entry);
-	void WriteSequenceValue(const SequenceCatalogEntry &entry, SequenceValue val);
+	void WriteSequenceValue(SequenceValue val);
 
 	void WriteCreateMacro(const ScalarMacroCatalogEntry &entry);
 	void WriteDropMacro(const ScalarMacroCatalogEntry &entry);
@@ -90,7 +95,7 @@ public:
 	void WriteCreateType(const TypeCatalogEntry &entry);
 	void WriteDropType(const TypeCatalogEntry &entry);
 	//! Sets the table used for subsequent insert/delete/update commands
-	void WriteSetTable(string &schema, string &table);
+	void WriteSetTable(const string &schema, const string &table);
 
 	void WriteAlter(const AlterInfo &info);
 

package/src/duckdb/src/include/duckdb/transaction/duck_transaction.hpp

@@ -9,13 +9,17 @@
 #pragma once
 
 #include "duckdb/transaction/transaction.hpp"
+#include "duckdb/common/reference_map.hpp"
 
 namespace duckdb {
 class RowVersionManager;
+class DuckTransactionManager;
+class StorageLockKey;
+struct UndoBufferProperties;
 
 class DuckTransaction : public Transaction {
 public:
-	DuckTransaction(TransactionManager &manager, ClientContext &context, transaction_t start_time,
+	DuckTransaction(DuckTransactionManager &manager, ClientContext &context, transaction_t start_time,
 	                transaction_t transaction_id);
 	~DuckTransaction() override;
 
@@ -25,8 +29,6 @@ public:
 	transaction_t transaction_id;
 	//! The commit id of this transaction, if it has successfully been committed
 	transaction_t commit_id;
-	//! Map of all sequences that were used during the transaction and the value they had in this transaction
-	unordered_map<SequenceCatalogEntry *, SequenceValue> sequence_usage;
 	//! Highest active query when the transaction finished, used for cleaning up
 	transaction_t highest_active_query;
 
@@ -37,11 +39,13 @@ public:
 
 	void PushCatalogEntry(CatalogEntry &entry, data_ptr_t extra_data = nullptr, idx_t extra_data_size = 0);
 
+	void SetReadWrite() override;
+
 	//! Commit the current transaction with the given commit identifier. Returns an error message if the transaction
 	//! commit failed, or an empty string if the commit was sucessful
 	ErrorData Commit(AttachedDatabase &db, transaction_t commit_id, bool checkpoint) noexcept;
 	//! Returns whether or not a commit of this transaction should trigger an automatic checkpoint
-	bool AutomaticCheckpoint(AttachedDatabase &db);
+	bool AutomaticCheckpoint(AttachedDatabase &db, const UndoBufferProperties &properties);
 
 	//! Rollback
 	void Rollback() noexcept;
@@ -49,9 +53,11 @@ public:
 	void Cleanup();
 
 	bool ChangesMade();
+	UndoBufferProperties GetUndoProperties();
 
 	void PushDelete(DataTable &table, RowVersionManager &info, idx_t vector_idx, row_t rows[], idx_t count,
 	                idx_t base_row);
+	void PushSequenceUsage(SequenceCatalogEntry &entry, const SequenceData &data);
 	void PushAppend(DataTable &table, idx_t row_start, idx_t row_count);
 	UpdateInfo *CreateUpdateInfo(idx_t type_size, idx_t entries);
 
@@ -59,12 +65,21 @@ public:
 		return true;
 	}
 
+	unique_ptr<StorageLockKey> TryGetCheckpointLock();
+
 private:
+	DuckTransactionManager &transaction_manager;
 	//! The undo buffer is used to store old versions of rows that are updated
 	//! or deleted
 	UndoBuffer undo_buffer;
 	//! The set of uncommitted appends for the transaction
 	unique_ptr<LocalStorage> storage;
+	//! Write lock
+	unique_ptr<StorageLockKey> write_lock;
+	//! Lock for accessing sequence_usage
+	mutex sequence_lock;
+	//! Map of all sequences that were used during the transaction and the value they had in this transaction
+	reference_map_t<SequenceCatalogEntry, reference<SequenceValue>> sequence_usage;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/transaction/duck_transaction_manager.hpp

@@ -9,6 +9,8 @@
 #pragma once
 
 #include "duckdb/transaction/transaction_manager.hpp"
+#include "duckdb/storage/storage_lock.hpp"
+#include "duckdb/common/enums/checkpoint_type.hpp"
 
 namespace duckdb {
 class DuckTransaction;
@@ -16,8 +18,6 @@ class DuckTransaction;
 //! The Transaction Manager is responsible for creating and managing
 //! transactions
 class DuckTransactionManager : public TransactionManager {
-	friend struct CheckpointLock;
-
 public:
 	explicit DuckTransactionManager(AttachedDatabase &db);
 	~DuckTransactionManager() override;
@@ -34,27 +34,46 @@ public:
 
 	void Checkpoint(ClientContext &context, bool force = false) override;
 
-	transaction_t LowestActiveId() {
+	transaction_t LowestActiveId() const {
 		return lowest_active_id;
 	}
-	transaction_t LowestActiveStart() {
+	transaction_t LowestActiveStart() const {
 		return lowest_active_start;
 	}
+	transaction_t GetLastCommit() const {
+		return last_commit;
+	}
 
 	bool IsDuckTransactionManager() override {
 		return true;
 	}
 
+	//! Obtains a shared lock to the checkpoint lock
+	unique_ptr<StorageLockKey> SharedCheckpointLock();
+	unique_ptr<StorageLockKey> TryUpgradeCheckpointLock(StorageLockKey &lock);
+
 protected:
 	struct CheckpointDecision {
+		explicit CheckpointDecision(string reason_p);
+		explicit CheckpointDecision(CheckpointType type);
+		~CheckpointDecision();
+
 		bool can_checkpoint;
 		string reason;
+		CheckpointType type;
 	};
 
 private:
-	CheckpointDecision CanCheckpoint(optional_ptr<DuckTransaction> current = nullptr);
+	//! Generates a new commit timestamp
+	transaction_t GetCommitTimestamp();
 	//! Remove the given transaction from the list of active transactions
 	void RemoveTransaction(DuckTransaction &transaction) noexcept;
+	//! Remove the given transaction from the list of active transactions
+	void RemoveTransaction(DuckTransaction &transaction, bool store_transaction) noexcept;
+
+	//! Whether or not we can checkpoint
+	CheckpointDecision CanCheckpoint(DuckTransaction &transaction, unique_ptr<StorageLockKey> &checkpoint_lock,
+	                                 const UndoBufferProperties &properties);
 
 private:
 	//! The current start timestamp used by transactions
@@ -65,6 +84,8 @@ private:
 	atomic<transaction_t> lowest_active_id;
 	//! The lowest active transaction timestamp
 	atomic<transaction_t> lowest_active_start;
+	//! The last commit timestamp
+	atomic<transaction_t> last_commit;
 	//! Set of currently running transactions
 	vector<unique_ptr<DuckTransaction>> active_transactions;
 	//! Set of recently committed transactions
@@ -73,8 +94,10 @@ private:
 	vector<unique_ptr<DuckTransaction>> old_transactions;
 	//! The lock used for transaction operations
 	mutex transaction_lock;
-
-	bool thread_is_checkpointing;
+	//! The checkpoint lock
+	StorageLock checkpoint_lock;
+	//! Lock necessary to start transactions only - used by FORCE CHECKPOINT to prevent new transactions from starting
+	mutex start_transaction_lock;
 
 protected:
 	virtual void OnCommitCheckpointDecision(const CheckpointDecision &decision, DuckTransaction &transaction) {

package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp

@@ -12,19 +12,21 @@
 #include "duckdb/storage/table/table_index_list.hpp"
 #include "duckdb/storage/table/table_statistics.hpp"
 #include "duckdb/storage/optimistic_data_writer.hpp"
+#include "duckdb/common/reference_map.hpp"
 
 namespace duckdb {
 class AttachedDatabase;
+class Catalog;
 class DataTable;
 class Transaction;
 class WriteAheadLog;
 struct LocalAppendState;
 struct TableAppendState;
 
-class LocalTableStorage : public std::enable_shared_from_this<LocalTableStorage> {
+class LocalTableStorage : public enable_shared_from_this<LocalTableStorage> {
 public:
 	// Create a new LocalTableStorage
-	explicit LocalTableStorage(DataTable &table);
+	explicit LocalTableStorage(ClientContext &context, DataTable &table);
 	// Create a LocalTableStorage from an ALTER TYPE
 	LocalTableStorage(ClientContext &context, DataTable &table, LocalTableStorage &parent, idx_t changed_idx,
 	                  const LogicalType &target_type, const vector<column_t> &bound_columns, Expression &cast_expr);
@@ -32,7 +34,7 @@ public:
 	LocalTableStorage(DataTable &table, LocalTableStorage &parent, idx_t drop_idx);
 	// Create a LocalTableStorage from an ADD COLUMN
 	LocalTableStorage(ClientContext &context, DataTable &table, LocalTableStorage &parent, ColumnDefinition &new_column,
-	                  Expression &default_value);
+	                  ExpressionExecutor &default_executor);
 	~LocalTableStorage();
 
 	reference<DataTable> table_ref;
@@ -74,7 +76,7 @@ public:
 	shared_ptr<LocalTableStorage> MoveEntry(DataTable &table);
 	reference_map_t<DataTable, shared_ptr<LocalTableStorage>> MoveEntries();
 	optional_ptr<LocalTableStorage> GetStorage(DataTable &table);
-	LocalTableStorage &GetOrCreateStorage(DataTable &table);
+	LocalTableStorage &GetOrCreateStorage(ClientContext &context, DataTable &table);
 	idx_t EstimatedSize();
 	bool IsEmpty();
 	void InsertEntry(DataTable &table, shared_ptr<LocalTableStorage> entry);
@@ -143,7 +145,8 @@ public:
 
 	idx_t AddedRows(DataTable &table);
 
-	void AddColumn(DataTable &old_dt, DataTable &new_dt, ColumnDefinition &new_column, Expression &default_value);
+	void AddColumn(DataTable &old_dt, DataTable &new_dt, ColumnDefinition &new_column,
+	               ExpressionExecutor &default_executor);
 	void DropColumn(DataTable &old_dt, DataTable &new_dt, idx_t removed_column);
 	void ChangeType(DataTable &old_dt, DataTable &new_dt, idx_t changed_idx, const LogicalType &target_type,
 	                const vector<column_t> &bound_columns, Expression &cast_expr);

package/src/duckdb/src/include/duckdb/transaction/meta_transaction.hpp

@@ -46,6 +46,7 @@ public:
 	}
 
 	Transaction &GetTransaction(AttachedDatabase &db);
+	optional_ptr<Transaction> TryGetTransaction(AttachedDatabase &db);
 	void RemoveTransaction(AttachedDatabase &db);
 
 	ErrorData Commit();

package/src/duckdb/src/include/duckdb/transaction/transaction.hpp

@@ -13,6 +13,7 @@
 #include "duckdb/transaction/undo_buffer.hpp"
 #include "duckdb/common/atomic.hpp"
 #include "duckdb/transaction/transaction_data.hpp"
+#include "duckdb/common/shared_ptr.hpp"
 
 namespace duckdb {
 class SequenceCatalogEntry;
@@ -50,9 +51,13 @@ public:
 public:
 	DUCKDB_API static Transaction &Get(ClientContext &context, AttachedDatabase &db);
 	DUCKDB_API static Transaction &Get(ClientContext &context, Catalog &catalog);
+	//! Returns the transaction for the given context if it has already been started
+	DUCKDB_API static optional_ptr<Transaction> TryGet(ClientContext &context, AttachedDatabase &db);
 
 	//! Whether or not the transaction has made any modifications to the database so far
 	DUCKDB_API bool IsReadOnly();
+	//! Promotes the transaction to a read-write transaction
+	DUCKDB_API virtual void SetReadWrite();
 
 	virtual bool IsDuckTransaction() const {
 		return false;
@@ -69,6 +74,9 @@ public:
 		DynamicCastCheck<TARGET>(this);
 		return reinterpret_cast<const TARGET &>(*this);
 	}
+
+private:
+	bool is_read_only;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/transaction/undo_buffer.hpp

@@ -16,6 +16,14 @@ namespace duckdb {
 
 class WriteAheadLog;
 
+struct UndoBufferProperties {
+	idx_t estimated_size = 0;
+	bool has_updates = false;
+	bool has_deletes = false;
+	bool has_catalog_changes = false;
+	bool has_dropped_entries = false;
+};
+
 //! The undo buffer of a transaction is used to hold previous versions of tuples
 //! that might be required in the future (because of rollbacks or previous
 //! transactions accessing them)
@@ -35,7 +43,7 @@ public:
 	data_ptr_t CreateEntry(UndoFlags type, idx_t len);
 
 	bool ChangesMade();
-	idx_t EstimatedSize();
+	UndoBufferProperties GetProperties();
 
 	//! Cleanup the undo buffer
 	void Cleanup();