duckdb 0.10.3-dev0.0 → 0.10.3-dev13.0

package/src/duckdb/src/include/duckdb/storage/table/column_data.hpp

@@ -16,6 +16,7 @@
 #include "duckdb/storage/table/segment_tree.hpp"
 #include "duckdb/storage/table/column_segment_tree.hpp"
 #include "duckdb/common/mutex.hpp"
+#include "duckdb/common/enums/scan_vector_type.hpp"
 
 namespace duckdb {
 class ColumnData;
@@ -29,10 +30,17 @@ struct TransactionData;
 struct TableScanOptions;
 
 struct DataTableInfo;
+struct RowGroupWriteInfo;
 
 struct ColumnCheckpointInfo {
-	explicit ColumnCheckpointInfo(CompressionType compression_type_p) : compression_type(compression_type_p) {};
-	CompressionType compression_type;
+	ColumnCheckpointInfo(RowGroupWriteInfo &info, idx_t column_idx) : info(info), column_idx(column_idx) {
+	}
+
+	RowGroupWriteInfo &info;
+	idx_t column_idx;
+
+public:
+	CompressionType GetCompressionType();
 };
 
 class ColumnData {
@@ -46,7 +54,7 @@ public:
 	//! The start row
 	idx_t start;
 	//! The count of the column data
-	idx_t count;
+	atomic<idx_t> count;
 	//! The block manager
 	BlockManager &block_manager;
 	//! Table info for the column
@@ -76,15 +84,22 @@ public:
 	//! The root type of the column
 	const LogicalType &RootType() const;
 	//! Whether or not the column has any updates
-	virtual bool HasUpdates() const;
+	bool HasUpdates() const;
+	//! Whether or not we can scan an entire vector
+	virtual ScanVectorType GetVectorScanType(ColumnScanState &state, idx_t scan_count);
 
 	//! Initialize a scan of the column
 	virtual void InitializeScan(ColumnScanState &state);
 	//! Initialize a scan starting at the specified offset
 	virtual void InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx);
 	//! Scan the next vector from the column
-	virtual idx_t Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result);
-	virtual idx_t ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates);
+	idx_t Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result);
+	idx_t ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates);
+	virtual idx_t Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
+	                   idx_t scan_count);
+	virtual idx_t ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates,
+	                            idx_t scan_count);
+
 	virtual void ScanCommittedRange(idx_t row_group_start, idx_t offset_in_row_group, idx_t count, Vector &result);
 	virtual idx_t ScanCount(ColumnScanState &state, Vector &result, idx_t count);
 	//! Select
@@ -124,8 +139,7 @@ public:
 
 	virtual unique_ptr<ColumnCheckpointState> CreateCheckpointState(RowGroup &row_group,
 	                                                                PartialBlockManager &partial_block_manager);
-	virtual unique_ptr<ColumnCheckpointState>
-	Checkpoint(RowGroup &row_group, PartialBlockManager &partial_block_manager, ColumnCheckpointInfo &checkpoint_info);
+	virtual unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, ColumnCheckpointInfo &info);
 
 	virtual void CheckpointScan(ColumnSegment &segment, ColumnScanState &state, idx_t row_group_start, idx_t count,
 	                            Vector &scan_vector);
@@ -155,11 +169,12 @@ protected:
 	void AppendTransientSegment(SegmentLock &l, idx_t start_row);
 
 	//! Scans a base vector from the column
-	idx_t ScanVector(ColumnScanState &state, Vector &result, idx_t remaining, bool has_updates);
+	idx_t ScanVector(ColumnScanState &state, Vector &result, idx_t remaining, ScanVectorType scan_type);
 	//! Scans a vector from the column merged with any potential updates
 	//! If ALLOW_UPDATES is set to false, the function will instead throw an exception if any updates are found
 	template <bool SCAN_COMMITTED, bool ALLOW_UPDATES>
-	idx_t ScanVector(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result);
+	idx_t ScanVector(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
+	                 idx_t target_scan);
 
 	void ClearUpdates();
 	void FetchUpdates(TransactionData transaction, idx_t vector_index, Vector &result, idx_t scan_count,
@@ -168,6 +183,8 @@ protected:
 	void UpdateInternal(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids,
 	                    idx_t update_count, Vector &base_vector);
 
+	idx_t GetVectorCount(idx_t vector_index) const;
+
 protected:
 	//! The segments holding the data of this column segment
 	ColumnSegmentTree data;
@@ -175,6 +192,8 @@ protected:
 	mutable mutex update_lock;
 	//! The updates for this column segment
 	unique_ptr<UpdateSegment> updates;
+	//! The lock for the stats
+	mutable mutex stats_lock;
 	//! The stats of the root segment
 	unique_ptr<SegmentStatistics> stats;
 	//! Total transient allocation size

package/src/duckdb/src/include/duckdb/storage/table/column_segment.hpp

@@ -17,6 +17,7 @@
 #include "duckdb/function/compression_function.hpp"
 #include "duckdb/storage/table/segment_base.hpp"
 #include "duckdb/storage/buffer/block_handle.hpp"
+#include "duckdb/common/enums/scan_vector_type.hpp"
 
 namespace duckdb {
 class ColumnSegment;
@@ -64,7 +65,7 @@ public:
 public:
 	void InitializeScan(ColumnScanState &state);
 	//! Scan one vector from this segment
-	void Scan(ColumnScanState &state, idx_t scan_count, Vector &result, idx_t result_offset, bool entire_vector);
+	void Scan(ColumnScanState &state, idx_t scan_count, Vector &result, idx_t result_offset, ScanVectorType scan_type);
 	//! Fetch a value of the specific row id and append it to the result
 	void FetchRow(ColumnFetchState &state, row_t row_id, Vector &result, idx_t result_idx);
 

package/src/duckdb/src/include/duckdb/storage/table/data_table_info.hpp

@@ -11,25 +11,51 @@
 #include "duckdb/common/atomic.hpp"
 #include "duckdb/common/common.hpp"
 #include "duckdb/storage/table/table_index_list.hpp"
+#include "duckdb/storage/storage_lock.hpp"
 
 namespace duckdb {
 class DatabaseInstance;
 class TableIOManager;
 
 struct DataTableInfo {
+	friend class DataTable;
+
+public:
 	DataTableInfo(AttachedDatabase &db, shared_ptr<TableIOManager> table_io_manager_p, string schema, string table);
 
 	//! Initialize any unknown indexes whose types might now be present after an extension load, optionally throwing an
 	//! exception if an index can't be initialized
-	void InitializeIndexes(ClientContext &context, bool throw_on_failure = false);
+	void InitializeIndexes(ClientContext &context, const char *index_type = nullptr);
+
+	//! Whether or not the table is temporary
+	bool IsTemporary() const;
+
+	AttachedDatabase &GetDB() {
+		return db;
+	}
+
+	TableIOManager &GetIOManager() {
+		return *table_io_manager;
+	}
 
+	TableIndexList &GetIndexes() {
+		return indexes;
+	}
+	const vector<IndexStorageInfo> &GetIndexStorageInfo() const {
+		return index_storage_infos;
+	}
+
+	string GetSchemaName();
+	string GetTableName();
+	void SetTableName(string name);
+
+private:
 	//! The database instance of the table
 	AttachedDatabase &db;
 	//! The table IO manager
 	shared_ptr<TableIOManager> table_io_manager;
-	//! The amount of elements in the table. Note that this number signifies the amount of COMMITTED entries in the
-	//! table. It can be inaccurate inside of transactions. More work is needed to properly support that.
-	atomic<idx_t> cardinality;
+	//! Lock for modifying the name
+	mutex name_lock;
 	//! The schema of the table
 	string schema;
 	//! The name of the table
@@ -38,8 +64,8 @@ struct DataTableInfo {
 	TableIndexList indexes;
 	//! Index storage information of the indexes created by this table
 	vector<IndexStorageInfo> index_storage_infos;
-
-	bool IsTemporary() const;
+	//! Lock held while checkpointing
+	StorageLock checkpoint_lock;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/delete_state.hpp

@@ -0,0 +1,23 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/table/delete_state.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/storage/table/append_state.hpp"
+
+namespace duckdb {
+class TableCatalogEntry;
+
+struct TableDeleteState {
+	unique_ptr<ConstraintState> constraint_state;
+	bool has_delete_constraints = false;
+	DataChunk verify_chunk;
+	vector<column_t> col_ids;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/list_column_data.hpp

@@ -31,8 +31,10 @@ public:
 	void InitializeScan(ColumnScanState &state) override;
 	void InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx) override;
 
-	idx_t Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result) override;
-	idx_t ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates) override;
+	idx_t Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
+	           idx_t scan_count) override;
+	idx_t ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates,
+	                    idx_t scan_count) override;
 	idx_t ScanCount(ColumnScanState &state, Vector &result, idx_t count) override;
 
 	void Skip(ColumnScanState &state, idx_t count = STANDARD_VECTOR_SIZE) override;
@@ -53,8 +55,7 @@ public:
 
 	unique_ptr<ColumnCheckpointState> CreateCheckpointState(RowGroup &row_group,
 	                                                        PartialBlockManager &partial_block_manager) override;
-	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, PartialBlockManager &partial_block_manager,
-	                                             ColumnCheckpointInfo &checkpoint_info) override;
+	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, ColumnCheckpointInfo &info) override;
 
 	void DeserializeColumn(Deserializer &deserializer, BaseStatistics &target_stats) override;
 

package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp

@@ -17,6 +17,7 @@
 #include "duckdb/parser/column_list.hpp"
 #include "duckdb/storage/table/segment_base.hpp"
 #include "duckdb/storage/block.hpp"
+#include "duckdb/common/enums/checkpoint_type.hpp"
 
 namespace duckdb {
 class AttachedDatabase;
@@ -43,6 +44,17 @@ struct RowGroupAppendState;
 class MetadataManager;
 class RowVersionManager;
 
+struct RowGroupWriteInfo {
+	RowGroupWriteInfo(PartialBlockManager &manager, const vector<CompressionType> &compression_types,
+	                  CheckpointType checkpoint_type = CheckpointType::FULL_CHECKPOINT)
+	    : manager(manager), compression_types(compression_types), checkpoint_type(checkpoint_type) {
+	}
+
+	PartialBlockManager &manager;
+	const vector<CompressionType> &compression_types;
+	CheckpointType checkpoint_type;
+};
+
 struct RowGroupWriteData {
 	vector<unique_ptr<ColumnCheckpointState>> states;
 	vector<BaseStatistics> statistics;
@@ -77,7 +89,7 @@ public:
 	                               ExpressionExecutor &executor, CollectionScanState &scan_state,
 	                               DataChunk &scan_chunk);
 	unique_ptr<RowGroup> AddColumn(RowGroupCollection &collection, ColumnDefinition &new_column,
-	                               ExpressionExecutor &executor, Expression &default_value, Vector &intermediate);
+	                               ExpressionExecutor &executor, Vector &intermediate);
 	unique_ptr<RowGroup> RemoveColumn(RowGroupCollection &collection, idx_t removed_column);
 
 	void CommitDrop();
@@ -117,7 +129,7 @@ public:
 	//! Delete the given set of rows in the version manager
 	idx_t Delete(TransactionData transaction, DataTable &table, row_t *row_ids, idx_t count);
 
-	RowGroupWriteData WriteToDisk(PartialBlockManager &manager, const vector<CompressionType> &compression_types);
+	RowGroupWriteData WriteToDisk(RowGroupWriteInfo &info);
 	//! Returns the number of committed rows (count - committed deletes)
 	idx_t GetCommittedRowCount();
 	RowGroupWriteData WriteToDisk(RowGroupWriter &writer);
@@ -171,7 +183,6 @@ private:
 
 private:
 	mutex row_group_lock;
-	mutex stats_lock;
 	vector<MetaBlockPointer> column_pointers;
 	unique_ptr<atomic<bool>[]> is_loaded;
 	vector<MetaBlockPointer> deletes_pointers;

package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp

@@ -90,7 +90,8 @@ public:
 
 	void Checkpoint(TableDataWriter &writer, TableStatistics &global_stats);
 
-	void InitializeVacuumState(VacuumState &state, vector<SegmentNode<RowGroup>> &segments);
+	void InitializeVacuumState(CollectionCheckpointState &checkpoint_state, VacuumState &state,
+	                           vector<SegmentNode<RowGroup>> &segments);
 	bool ScheduleVacuumTasks(CollectionCheckpointState &checkpoint_state, VacuumState &state, idx_t segment_idx);
 	void ScheduleCheckpointTask(CollectionCheckpointState &checkpoint_state, idx_t segment_idx);
 
@@ -101,7 +102,7 @@ public:
 	const vector<LogicalType> &GetTypes() const;
 
 	shared_ptr<RowGroupCollection> AddColumn(ClientContext &context, ColumnDefinition &new_column,
-	                                         Expression &default_value);
+	                                         ExpressionExecutor &default_executor);
 	shared_ptr<RowGroupCollection> RemoveColumn(idx_t col_idx);
 	shared_ptr<RowGroupCollection> AlterType(ClientContext &context, idx_t changed_idx, const LogicalType &target_type,
 	                                         vector<column_t> bound_columns, Expression &cast_expr);

package/src/duckdb/src/include/duckdb/storage/table/scan_state.hpp

@@ -157,6 +157,8 @@ public:
 	CollectionScanState local_state;
 	//! Options for scanning
 	TableScanOptions options;
+	//! Shared lock over the checkpoint to prevent checkpoints while reading
+	unique_ptr<StorageLockKey> checkpoint_lock;
 
 public:
 	void Initialize(vector<storage_t> column_ids, TableFilterSet *table_filters = nullptr);
@@ -192,6 +194,8 @@ struct ParallelTableScanState {
 	ParallelCollectionScanState scan_state;
 	//! Parallel scan state for the transaction-local state
 	ParallelCollectionScanState local_state;
+	//! Shared lock over the checkpoint to prevent checkpoints while reading
+	unique_ptr<StorageLockKey> checkpoint_lock;
 };
 
 class CreateIndexScanState : public TableScanState {

package/src/duckdb/src/include/duckdb/storage/table/segment_tree.hpp

@@ -83,11 +83,11 @@ public:
 		if (index < 0) {
 			// load all segments
 			LoadAllSegments(l);
-			index = nodes.size() + index;
+			index += nodes.size();
 			if (index < 0) {
 				return nullptr;
 			}
-			return nodes[index].node.get();
+			return nodes[UnsafeNumericCast<idx_t>(index)].node.get();
 		} else {
 			// lazily load segments until we reach the specific segment
 			while (idx_t(index) >= nodes.size() && LoadNextSegment(l)) {
@@ -95,7 +95,7 @@ public:
 			if (idx_t(index) >= nodes.size()) {
 				return nullptr;
 			}
-			return nodes[index].node.get();
+			return nodes[UnsafeNumericCast<idx_t>(index)].node.get();
 		}
 	}
 	//! Gets the next segment
@@ -116,7 +116,7 @@ public:
 #ifdef DEBUG
 		D_ASSERT(nodes[segment->index].node.get() == segment);
 #endif
-		return GetSegmentByIndex(l, segment->index + 1);
+		return GetSegmentByIndex(l, UnsafeNumericCast<int64_t>(segment->index + 1));
 	}
 
 	//! Gets a pointer to the last segment. Useful for appends.
@@ -182,7 +182,7 @@ public:
 		if (segment_start >= nodes.size() - 1) {
 			return;
 		}
-		nodes.erase(nodes.begin() + segment_start + 1, nodes.end());
+		nodes.erase(nodes.begin() + UnsafeNumericCast<int64_t>(segment_start) + 1, nodes.end());
 	}
 
 	//! Get the segment index of the column segment for the given row

package/src/duckdb/src/include/duckdb/storage/table/standard_column_data.hpp

@@ -23,16 +23,17 @@ public:
 	ValidityColumnData validity;
 
 public:
-	bool HasUpdates() const override;
-
 	void SetStart(idx_t new_start) override;
 	bool CheckZonemap(ColumnScanState &state, TableFilter &filter) override;
 
+	ScanVectorType GetVectorScanType(ColumnScanState &state, idx_t scan_count) override;
 	void InitializeScan(ColumnScanState &state) override;
 	void InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx) override;
 
-	idx_t Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result) override;
-	idx_t ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates) override;
+	idx_t Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
+	           idx_t target_count) override;
+	idx_t ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates,
+	                    idx_t target_count) override;
 	idx_t ScanCount(ColumnScanState &state, Vector &result, idx_t count) override;
 
 	void InitializeAppend(ColumnAppendState &state) override;
@@ -51,8 +52,7 @@ public:
 
 	unique_ptr<ColumnCheckpointState> CreateCheckpointState(RowGroup &row_group,
 	                                                        PartialBlockManager &partial_block_manager) override;
-	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, PartialBlockManager &partial_block_manager,
-	                                             ColumnCheckpointInfo &checkpoint_info) override;
+	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, ColumnCheckpointInfo &info) override;
 	void CheckpointScan(ColumnSegment &segment, ColumnScanState &state, idx_t row_group_start, idx_t count,
 	                    Vector &scan_vector) override;
 

package/src/duckdb/src/include/duckdb/storage/table/struct_column_data.hpp

@@ -32,8 +32,10 @@ public:
 	void InitializeScan(ColumnScanState &state) override;
 	void InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx) override;
 
-	idx_t Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result) override;
-	idx_t ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates) override;
+	idx_t Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
+	           idx_t scan_count) override;
+	idx_t ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates,
+	                    idx_t scan_count) override;
 	idx_t ScanCount(ColumnScanState &state, Vector &result, idx_t count) override;
 
 	void Skip(ColumnScanState &state, idx_t count = STANDARD_VECTOR_SIZE) override;
@@ -54,8 +56,7 @@ public:
 
 	unique_ptr<ColumnCheckpointState> CreateCheckpointState(RowGroup &row_group,
 	                                                        PartialBlockManager &partial_block_manager) override;
-	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, PartialBlockManager &partial_block_manager,
-	                                             ColumnCheckpointInfo &checkpoint_info) override;
+	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, ColumnCheckpointInfo &info) override;
 
 	void DeserializeColumn(Deserializer &source, BaseStatistics &target_stats) override;
 

package/src/duckdb/src/include/duckdb/storage/table/table_index_list.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/common/mutex.hpp"
 #include "duckdb/storage/index.hpp"
+#include "duckdb/parser/constraint.hpp"
 
 namespace duckdb {
 
@@ -29,6 +30,29 @@ public:
 			}
 		}
 	}
+
+	//! Scan the indexes, invoking the callback method for every bound entry of a specific type
+	template <class T, class FUNC>
+	void ScanBound(FUNC &&callback) {
+		lock_guard<mutex> lock(indexes_lock);
+		for (auto &index : indexes) {
+			if (index->IsBound() && T::TYPE_NAME == index->GetIndexType()) {
+				if (callback(index->Cast<T>())) {
+					break;
+				}
+			}
+		}
+	}
+
+	// Bind any unbound indexes of the specified type and invoke the callback method for every bound entry of the
+	// specified type, regardless if it was bound before or not
+	template <class T, class FUNC>
+	void BindAndScan(ClientContext &context, DataTableInfo &table_info, FUNC &&callback) {
+		// FIXME: optimize this by only looping through the indexes once without re-acquiring the lock
+		InitializeIndexes(context, table_info, T::TYPE_NAME);
+		ScanBound<T>(callback);
+	}
+
 	//! Returns a reference to the indexes of this table
 	const vector<unique_ptr<Index>> &Indexes() const {
 		return indexes;
@@ -43,7 +67,7 @@ public:
 	bool NameIsUnique(const string &name);
 	//! Initializes unknown indexes that might now be present after an extension load, optionally throwing an exception
 	//! if a index cant be initialized
-	void InitializeIndexes(ClientContext &context, DataTableInfo &table_info, bool throw_on_failure = false);
+	void InitializeIndexes(ClientContext &context, DataTableInfo &table_info, const char *index_type = nullptr);
 	bool Empty();
 	idx_t Count();
 	void Move(TableIndexList &other);

package/src/duckdb/src/include/duckdb/storage/table/table_statistics.hpp

@@ -43,8 +43,11 @@ public:
 	void MergeStats(TableStatisticsLock &lock, idx_t i, BaseStatistics &stats);
 
 	void CopyStats(TableStatistics &other);
+	void CopyStats(TableStatisticsLock &lock, TableStatistics &other);
 	unique_ptr<BaseStatistics> CopyStats(idx_t i);
-	ColumnStatistics &GetStats(idx_t i);
+	//! Get a reference to the stats - this requires us to hold the lock.
+	//! The reference can only be safely accessed while the lock is held
+	ColumnStatistics &GetStats(TableStatisticsLock &lock, idx_t i);
 
 	bool Empty();
 
@@ -55,7 +58,7 @@ public:
 
 private:
 	//! The statistics lock
-	mutex stats_lock;
+	shared_ptr<mutex> stats_lock;
 	//! Column statistics
 	vector<shared_ptr<ColumnStatistics>> column_stats;
 	//! The table sample

package/src/duckdb/src/include/duckdb/storage/table/update_state.hpp

@@ -0,0 +1,20 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/table/update_state.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/storage/table/append_state.hpp"
+
+namespace duckdb {
+class TableCatalogEntry;
+
+struct TableUpdateState {
+	unique_ptr<ConstraintState> constraint_state;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/validity_column_data.hpp

@@ -20,6 +20,7 @@ public:
 
 public:
 	bool CheckZonemap(ColumnScanState &state, TableFilter &filter) override;
+	void AppendData(BaseStatistics &stats, ColumnAppendState &state, UnifiedVectorFormat &vdata, idx_t count) override;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/temporary_file_manager.hpp

@@ -23,8 +23,11 @@ namespace duckdb {
 // BlockIndexManager
 //===--------------------------------------------------------------------===//
 
+class TemporaryFileManager;
+
 struct BlockIndexManager {
 public:
+	explicit BlockIndexManager(TemporaryFileManager &manager);
 	BlockIndexManager();
 
 public:
@@ -37,12 +40,14 @@ public:
 	bool HasFreeBlocks();
 
 private:
+	void SetMaxIndex(idx_t blocks);
 	idx_t GetNewBlockIndexInternal();
 
 private:
 	idx_t max_index;
 	set<idx_t> free_indexes;
 	set<idx_t> indexes_in_use;
+	optional_ptr<TemporaryFileManager> manager;
 };
 
 //===--------------------------------------------------------------------===//
@@ -69,7 +74,8 @@ class TemporaryFileHandle {
 	constexpr static idx_t MAX_ALLOWED_INDEX_BASE = 4000;
 
 public:
-	TemporaryFileHandle(idx_t temp_file_count, DatabaseInstance &db, const string &temp_directory, idx_t index);
+	TemporaryFileHandle(idx_t temp_file_count, DatabaseInstance &db, const string &temp_directory, idx_t index,
+	                    TemporaryFileManager &manager);
 
 public:
 	struct TemporaryFileLock {
@@ -103,15 +109,13 @@ private:
 	BlockIndexManager index_manager;
 };
 
-class TemporaryFileManager;
-
 //===--------------------------------------------------------------------===//
 // TemporaryDirectoryHandle
 //===--------------------------------------------------------------------===//
 
 class TemporaryDirectoryHandle {
 public:
-	TemporaryDirectoryHandle(DatabaseInstance &db, string path_p);
+	TemporaryDirectoryHandle(DatabaseInstance &db, string path_p, optional_idx max_swap_space);
 	~TemporaryDirectoryHandle();
 
 	TemporaryFileManager &GetTempFile();
@@ -130,6 +134,7 @@ private:
 class TemporaryFileManager {
 public:
 	TemporaryFileManager(DatabaseInstance &db, const string &temp_directory_p);
+	~TemporaryFileManager();
 
 public:
 	struct TemporaryManagerLock {
@@ -145,6 +150,13 @@ public:
 	unique_ptr<FileBuffer> ReadTemporaryBuffer(block_id_t id, unique_ptr<FileBuffer> reusable_buffer);
 	void DeleteTemporaryBuffer(block_id_t id);
 	vector<TemporaryFileInformation> GetTemporaryFiles();
+	idx_t GetTotalUsedSpaceInBytes();
+	optional_idx GetMaxSwapSpace() const;
+	void SetMaxSwapSpace(optional_idx limit);
+	//! Register temporary file size growth
+	void IncreaseSizeOnDisk(idx_t amount);
+	//! Register temporary file size decrease
+	void DecreaseSizeOnDisk(idx_t amount);
 
 private:
 	void EraseUsedBlock(TemporaryManagerLock &lock, block_id_t id, TemporaryFileHandle *handle,
@@ -164,6 +176,10 @@ private:
 	unordered_map<block_id_t, TemporaryFileIndex> used_blocks;
 	//! Manager of in-use temporary file indexes
 	BlockIndexManager index_manager;
+	//! The size in bytes of the temporary files that are currently alive
+	atomic<idx_t> size_on_disk;
+	//! The max amount of disk space that can be used
+	idx_t max_swap_space;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/write_ahead_log.hpp

@@ -44,7 +44,7 @@ class WriteAheadLogDeserializer;
 class WriteAheadLog {
 public:
 	//! Initialize the WAL in the specified directory
-	explicit WriteAheadLog(AttachedDatabase &database, const string &path);
+	explicit WriteAheadLog(AttachedDatabase &database, const string &wal_path);
 	virtual ~WriteAheadLog();
 
 	//! Skip writing to the WAL
@@ -54,11 +54,16 @@ public:
 	//! Replay the WAL
 	static bool Replay(AttachedDatabase &database, unique_ptr<FileHandle> handle);
 
-	//! Returns the current size of the WAL in bytes
-	int64_t GetWALSize();
 	//! Gets the total bytes written to the WAL since startup
 	idx_t GetTotalWritten();
 
+	//! A WAL is initialized, if a writer to a file exists.
+	bool Initialized() {
+		return writer != nullptr;
+	}
+	//! Initializes the file of the WAL by creating the file writer.
+	BufferedFileWriter &Initialize();
+	//! Returns the WAL file writer.
 	BufferedFileWriter &GetWriter() {
 		return *writer;
 	}
@@ -76,7 +81,7 @@ public:
 
 	void WriteCreateSequence(const SequenceCatalogEntry &entry);
 	void WriteDropSequence(const SequenceCatalogEntry &entry);
-	void WriteSequenceValue(const SequenceCatalogEntry &entry, SequenceValue val);
+	void WriteSequenceValue(SequenceValue val);
 
 	void WriteCreateMacro(const ScalarMacroCatalogEntry &entry);
 	void WriteDropMacro(const ScalarMacroCatalogEntry &entry);
@@ -90,7 +95,7 @@ public:
 	void WriteCreateType(const TypeCatalogEntry &entry);
 	void WriteDropType(const TypeCatalogEntry &entry);
 	//! Sets the table used for subsequent insert/delete/update commands
-	void WriteSetTable(string &schema, string &table);
+	void WriteSetTable(const string &schema, const string &table);
 
 	void WriteAlter(const AlterInfo &info);