duckdb 1.1.2-dev6.0 → 1.1.4-dev11.0

package/src/duckdb/src/include/duckdb/storage/table/column_data.hpp

@@ -18,6 +18,7 @@
 #include "duckdb/common/mutex.hpp"
 #include "duckdb/common/enums/scan_vector_type.hpp"
 #include "duckdb/common/serializer/serialization_traits.hpp"
+#include "duckdb/common/atomic_ptr.hpp"
 
 namespace duckdb {
 class ColumnData;
@@ -34,6 +35,8 @@ struct TableScanOptions;
 struct TransactionData;
 struct PersistentColumnData;
 
+using column_segment_vector_t = vector<SegmentNode<ColumnSegment>>;
+
 struct ColumnCheckpointInfo {
 	ColumnCheckpointInfo(RowGroupWriteInfo &info, idx_t column_idx) : info(info), column_idx(column_idx) {
 	}
@@ -65,8 +68,6 @@ public:
 	idx_t column_index;
 	//! The type of the column
 	LogicalType type;
-	//! The parent column (if any)
-	optional_ptr<ColumnData> parent;
 
 public:
 	virtual FilterPropagateResult CheckZonemap(ColumnScanState &state, TableFilter &filter);
@@ -81,12 +82,24 @@ public:
 	idx_t GetAllocationSize() const {
 		return allocation_size;
 	}
+	optional_ptr<const CompressionFunction> GetCompressionFunction() const {
+		return compression.get();
+	}
+
+	bool HasParent() const {
+		return parent != nullptr;
+	}
+	const ColumnData &Parent() const {
+		D_ASSERT(HasParent());
+		return *parent;
+	}
 
 	virtual void SetStart(idx_t new_start);
 	//! The root type of the column
 	const LogicalType &RootType() const;
 	//! Whether or not the column has any updates
 	bool HasUpdates() const;
+	bool HasChanges(idx_t start_row, idx_t end_row) const;
 	//! Whether or not we can scan an entire vector
 	virtual ScanVectorType GetVectorScanType(ColumnScanState &state, idx_t scan_count, Vector &result);
 
@@ -108,12 +121,12 @@ public:
 	virtual idx_t ScanCount(ColumnScanState &state, Vector &result, idx_t count);
 
 	//! Select
-	virtual void Select(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
+	virtual void Filter(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
 	                    SelectionVector &sel, idx_t &count, const TableFilter &filter);
-	virtual void FilterScan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
-	                        SelectionVector &sel, idx_t count);
-	virtual void FilterScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, SelectionVector &sel,
-	                                 idx_t count, bool allow_updates);
+	virtual void Select(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
+	                    SelectionVector &sel, idx_t count);
+	virtual void SelectCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, SelectionVector &sel,
+	                             idx_t count, bool allow_updates);
 
 	//! Skip the scan forward by "count" rows
 	virtual void Skip(ColumnScanState &state, idx_t count = STANDARD_VECTOR_SIZE);
@@ -177,14 +190,20 @@ public:
 protected:
 	//! Append a transient segment
 	void AppendTransientSegment(SegmentLock &l, idx_t start_row);
+	void AppendSegment(SegmentLock &l, unique_ptr<ColumnSegment> segment);
 
+	void BeginScanVectorInternal(ColumnScanState &state);
 	//! Scans a base vector from the column
 	idx_t ScanVector(ColumnScanState &state, Vector &result, idx_t remaining, ScanVectorType scan_type);
 	//! Scans a vector from the column merged with any potential updates
-	//! If ALLOW_UPDATES is set to false, the function will instead throw an exception if any updates are found
-	template <bool SCAN_COMMITTED, bool ALLOW_UPDATES>
 	idx_t ScanVector(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
-	                 idx_t target_scan);
+	                 idx_t target_scan, ScanVectorType scan_type, ScanVectorMode mode);
+	idx_t ScanVector(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
+	                 idx_t target_scan, ScanVectorMode mode);
+	void SelectVector(ColumnScanState &state, Vector &result, idx_t target_count, const SelectionVector &sel,
+	                  idx_t sel_count);
+	void FilterVector(ColumnScanState &state, Vector &result, idx_t target_count, SelectionVector &sel,
+	                  idx_t &sel_count, const TableFilter &filter);
 
 	void ClearUpdates();
 	void FetchUpdates(TransactionData transaction, idx_t vector_index, Vector &result, idx_t scan_count,
@@ -195,6 +214,9 @@ protected:
 
 	idx_t GetVectorCount(idx_t vector_index) const;
 
+private:
+	void UpdateCompressionFunction(SegmentLock &l, const CompressionFunction &function);
+
 protected:
 	//! The segments holding the data of this column segment
 	ColumnSegmentTree data;
@@ -208,6 +230,13 @@ protected:
 	unique_ptr<SegmentStatistics> stats;
 	//! Total transient allocation size
 	idx_t allocation_size;
+
+private:
+	//! The parent column (if any)
+	optional_ptr<ColumnData> parent;
+	//!	The compression function used by the ColumnData
+	//! This is empty if the segments have mixed compression or the ColumnData is empty
+	atomic_ptr<const CompressionFunction> compression;
 };
 
 struct PersistentColumnData {

package/src/duckdb/src/include/duckdb/storage/table/column_data_checkpointer.hpp

@@ -15,37 +15,79 @@
 namespace duckdb {
 struct TableScanOptions;
 
-class ColumnDataCheckpointer {
+//! Holds state related to a single column during compression
+struct ColumnDataCheckpointData {
 public:
-	ColumnDataCheckpointer(ColumnData &col_data_p, RowGroup &row_group_p, ColumnCheckpointState &state_p,
-	                       ColumnCheckpointInfo &checkpoint_info);
+	//! Default constructor used when column data does not need to be checkpointed
+	ColumnDataCheckpointData() {
+	}
+	ColumnDataCheckpointData(ColumnCheckpointState &checkpoint_state, ColumnData &col_data, DatabaseInstance &db,
+	                         RowGroup &row_group, ColumnCheckpointInfo &checkpoint_info)
+	    : checkpoint_state(checkpoint_state), col_data(col_data), db(db), row_group(row_group),
+	      checkpoint_info(checkpoint_info) {
+	}
 
 public:
-	DatabaseInstance &GetDatabase();
+	CompressionFunction &GetCompressionFunction(CompressionType type);
 	const LogicalType &GetType() const;
 	ColumnData &GetColumnData();
 	RowGroup &GetRowGroup();
 	ColumnCheckpointState &GetCheckpointState();
+	DatabaseInstance &GetDatabase();
 
-	void Checkpoint(vector<SegmentNode<ColumnSegment>> nodes);
-	CompressionFunction &GetCompressionFunction(CompressionType type);
+private:
+	optional_ptr<ColumnCheckpointState> checkpoint_state;
+	optional_ptr<ColumnData> col_data;
+	optional_ptr<DatabaseInstance> db;
+	optional_ptr<RowGroup> row_group;
+	optional_ptr<ColumnCheckpointInfo> checkpoint_info;
+};
+
+struct CheckpointAnalyzeResult {
+public:
+	//! Default constructor, returned when the column data doesn't require checkpoint
+	CheckpointAnalyzeResult() {
+	}
+	CheckpointAnalyzeResult(unique_ptr<AnalyzeState> &&analyze_state, CompressionFunction &function)
+	    : analyze_state(std::move(analyze_state)), function(function) {
+	}
+
+public:
+	unique_ptr<AnalyzeState> analyze_state;
+	optional_ptr<CompressionFunction> function;
+};
+
+class ColumnDataCheckpointer {
+public:
+	ColumnDataCheckpointer(vector<reference<ColumnCheckpointState>> &states, DatabaseInstance &db, RowGroup &row_group,
+	                       ColumnCheckpointInfo &checkpoint_info);
+
+public:
+	void Checkpoint();
+	void FinalizeCheckpoint();
 
 private:
 	void ScanSegments(const std::function<void(Vector &, idx_t)> &callback);
-	unique_ptr<AnalyzeState> DetectBestCompressionMethod(idx_t &compression_idx);
+	vector<CheckpointAnalyzeResult> DetectBestCompressionMethod();
 	void WriteToDisk();
-	bool HasChanges();
-	void WritePersistentSegments();
+	bool HasChanges(ColumnData &col_data);
+	void WritePersistentSegments(ColumnCheckpointState &state);
+	void InitAnalyze();
+	void DropSegments();
+	bool ValidityCoveredByBasedata(vector<CheckpointAnalyzeResult> &result);
 
 private:
-	ColumnData &col_data;
+	vector<reference<ColumnCheckpointState>> &checkpoint_states;
+	DatabaseInstance &db;
 	RowGroup &row_group;
-	ColumnCheckpointState &state;
-	bool is_validity;
 	Vector intermediate;
-	vector<SegmentNode<ColumnSegment>> nodes;
-	vector<optional_ptr<CompressionFunction>> compression_functions;
 	ColumnCheckpointInfo &checkpoint_info;
+
+	vector<bool> has_changes;
+	//! For every column data that is being checkpointed, the applicable functions
+	vector<vector<optional_ptr<CompressionFunction>>> compression_functions;
+	//! For every column data that is being checkpointed, the analyze state of functions being tried
+	vector<vector<unique_ptr<AnalyzeState>>> analyze_states;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/column_segment.hpp

@@ -38,37 +38,37 @@ enum class ColumnSegmentType : uint8_t { TRANSIENT, PERSISTENT };
 
 class ColumnSegment : public SegmentBase<ColumnSegment> {
 public:
+	//! Construct a column segment.
+	ColumnSegment(DatabaseInstance &db, shared_ptr<BlockHandle> block, const LogicalType &type,
+	              const ColumnSegmentType segment_type, const idx_t start, const idx_t count,
+	              CompressionFunction &function_p, BaseStatistics statistics, const block_id_t block_id_p,
+	              const idx_t offset, const idx_t segment_size_p,
+	              unique_ptr<ColumnSegmentState> segment_state_p = nullptr);
+	//! Construct a column segment from another column segment.
+	//! The other column segment becomes invalid (std::move).
+	ColumnSegment(ColumnSegment &other, const idx_t start);
 	~ColumnSegment();
 
-	//! The database instance
-	DatabaseInstance &db;
-	//! The type stored in the column
-	LogicalType type;
-	//! The size of the type
-	idx_t type_size;
-	//! The column segment type (transient or persistent)
-	ColumnSegmentType segment_type;
-	//! The compression function
-	reference<CompressionFunction> function;
-	//! The statistics for the segment
-	SegmentStatistics stats;
-	//! The block that this segment relates to
-	shared_ptr<BlockHandle> block;
-
+public:
 	static unique_ptr<ColumnSegment> CreatePersistentSegment(DatabaseInstance &db, BlockManager &block_manager,
 	                                                         block_id_t id, idx_t offset, const LogicalType &type_p,
 	                                                         idx_t start, idx_t count, CompressionType compression_type,
 	                                                         BaseStatistics statistics,
 	                                                         unique_ptr<ColumnSegmentState> segment_state);
-	static unique_ptr<ColumnSegment> CreateTransientSegment(DatabaseInstance &db, const LogicalType &type,
-	                                                        const idx_t start, const idx_t segment_size,
-	                                                        const idx_t block_size);
+	static unique_ptr<ColumnSegment> CreateTransientSegment(DatabaseInstance &db, CompressionFunction &function,
+	                                                        const LogicalType &type, const idx_t start,
+	                                                        const idx_t segment_size, const idx_t block_size);
 
 public:
 	void InitializePrefetch(PrefetchState &prefetch_state, ColumnScanState &scan_state);
 	void InitializeScan(ColumnScanState &state);
 	//! Scan one vector from this segment
 	void Scan(ColumnScanState &state, idx_t scan_count, Vector &result, idx_t result_offset, ScanVectorType scan_type);
+	//! Scan a subset of a vector (defined by the selection vector)
+	void Select(ColumnScanState &state, idx_t scan_count, Vector &result, const SelectionVector &sel, idx_t sel_count);
+	//! Scan one vector while applying a filter to the vector, returning only the matching elements
+	void Filter(ColumnScanState &state, idx_t scan_count, Vector &result, SelectionVector &sel, idx_t &sel_count,
+	            const TableFilter &filter);
 	//! Fetch a value of the specific row id and append it to the result
 	void FetchRow(ColumnFetchState &state, row_t row_id, Vector &result, idx_t result_idx);
 
@@ -82,6 +82,7 @@ public:
 	idx_t SegmentSize() const;
 	//! Resize the block
 	void Resize(idx_t segment_size);
+	const CompressionFunction &GetCompressionFunction();
 
 	//! Initialize an append of this segment. Appends are only supported on transient segments.
 	void InitializeAppend(ColumnAppendState &state);
@@ -132,22 +133,27 @@ public:
 
 	void CommitDropSegment();
 
-public:
-	//! Construct a column segment.
-	ColumnSegment(DatabaseInstance &db, shared_ptr<BlockHandle> block, const LogicalType &type,
-	              const ColumnSegmentType segment_type, const idx_t start, const idx_t count,
-	              CompressionFunction &function_p, BaseStatistics statistics, const block_id_t block_id_p,
-	              const idx_t offset, const idx_t segment_size_p,
-	              unique_ptr<ColumnSegmentState> segment_state_p = nullptr);
-	//! Construct a column segment from another column segment.
-	//! The other column segment becomes invalid (std::move).
-	ColumnSegment(ColumnSegment &other, const idx_t start);
-
 private:
 	void Scan(ColumnScanState &state, idx_t scan_count, Vector &result);
 	void ScanPartial(ColumnScanState &state, idx_t scan_count, Vector &result, idx_t result_offset);
 
+public:
+	//! The database instance
+	DatabaseInstance &db;
+	//! The type stored in the column
+	LogicalType type;
+	//! The size of the type
+	idx_t type_size;
+	//! The column segment type (transient or persistent)
+	ColumnSegmentType segment_type;
+	//! The statistics for the segment
+	SegmentStatistics stats;
+	//! The block that this segment relates to
+	shared_ptr<BlockHandle> block;
+
 private:
+	//! The compression function
+	reference<CompressionFunction> function;
 	//! The block id that this segment relates to (persistent segment only)
 	block_id_t block_id;
 	//! The offset into the block (persistent segment only)

package/src/duckdb/src/include/duckdb/storage/table/delete_state.hpp

@@ -17,7 +17,7 @@ struct TableDeleteState {
 	unique_ptr<ConstraintState> constraint_state;
 	bool has_delete_constraints = false;
 	DataChunk verify_chunk;
-	vector<column_t> col_ids;
+	vector<StorageIndex> col_ids;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/list_column_data.hpp

@@ -21,7 +21,7 @@ public:
 
 	//! The child-column of the list
 	unique_ptr<ColumnData> child_column;
-	//! The validity column data of the struct
+	//! The validity column data of the list
 	ValidityColumnData validity;
 
 public:

package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp

@@ -18,6 +18,8 @@
 #include "duckdb/storage/table/segment_base.hpp"
 #include "duckdb/storage/block.hpp"
 #include "duckdb/common/enums/checkpoint_type.hpp"
+#include "duckdb/storage/storage_index.hpp"
+#include "duckdb/function/partition_stats.hpp"
 
 namespace duckdb {
 class AttachedDatabase;
@@ -123,7 +125,7 @@ public:
 	//! For a specific row, returns true if it should be used for the transaction and false otherwise.
 	bool Fetch(TransactionData transaction, idx_t row);
 	//! Fetch a specific row from the row_group and insert it into the result at the specified index
-	void FetchRow(TransactionData transaction, ColumnFetchState &state, const vector<column_t> &column_ids,
+	void FetchRow(TransactionData transaction, ColumnFetchState &state, const vector<StorageIndex> &column_ids,
 	              row_t row_id, DataChunk &result, idx_t result_idx);
 
 	//! Append count rows to the version info
@@ -162,6 +164,7 @@ public:
 	unique_ptr<BaseStatistics> GetStatistics(idx_t column_idx);
 
 	void GetColumnSegmentInfo(idx_t row_group_index, vector<ColumnSegmentInfo> &result);
+	PartitionStatistics GetPartitionStats() const;
 
 	idx_t GetAllocationSize() const {
 		return allocation_size;
@@ -178,6 +181,8 @@ public:
 	static void Serialize(RowGroupPointer &pointer, Serializer &serializer);
 	static RowGroupPointer Deserialize(Deserializer &deserializer);
 
+	idx_t GetRowGroupSize() const;
+
 private:
 	optional_ptr<RowVersionManager> GetVersionInfo();
 	shared_ptr<RowVersionManager> GetOrCreateVersionInfoPtr();
@@ -185,6 +190,7 @@ private:
 	void SetVersionInfo(shared_ptr<RowVersionManager> version);
 
 	ColumnData &GetColumn(storage_t c);
+	ColumnData &GetColumn(const StorageIndex &c);
 	idx_t GetColumnCount() const;
 	vector<shared_ptr<ColumnData>> &GetColumns();
 

package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp

@@ -12,6 +12,7 @@
 #include "duckdb/storage/table/segment_tree.hpp"
 #include "duckdb/storage/statistics/column_statistics.hpp"
 #include "duckdb/storage/table/table_statistics.hpp"
+#include "duckdb/storage/storage_index.hpp"
 
 namespace duckdb {
 
@@ -34,11 +35,14 @@ struct VacuumState;
 struct CollectionCheckpointState;
 struct PersistentCollectionData;
 class CheckpointTask;
+class TableIOManager;
 
 class RowGroupCollection {
 public:
-	RowGroupCollection(shared_ptr<DataTableInfo> info, BlockManager &block_manager, vector<LogicalType> types,
+	RowGroupCollection(shared_ptr<DataTableInfo> info, TableIOManager &io_manager, vector<LogicalType> types,
 	                   idx_t row_start, idx_t total_rows = 0);
+	RowGroupCollection(shared_ptr<DataTableInfo> info, BlockManager &block_manager, vector<LogicalType> types,
+	                   idx_t row_start, idx_t total_rows, idx_t row_group_size);
 
 public:
 	idx_t GetTotalRows() const;
@@ -55,20 +59,21 @@ public:
 	RowGroup *GetRowGroup(int64_t index);
 	void Verify();
 
-	void InitializeScan(CollectionScanState &state, const vector<column_t> &column_ids, TableFilterSet *table_filters);
+	void InitializeScan(CollectionScanState &state, const vector<StorageIndex> &column_ids,
+	                    TableFilterSet *table_filters);
 	void InitializeCreateIndexScan(CreateIndexScanState &state);
-	void InitializeScanWithOffset(CollectionScanState &state, const vector<column_t> &column_ids, idx_t start_row,
+	void InitializeScanWithOffset(CollectionScanState &state, const vector<StorageIndex> &column_ids, idx_t start_row,
 	                              idx_t end_row);
 	static bool InitializeScanInRowGroup(CollectionScanState &state, RowGroupCollection &collection,
 	                                     RowGroup &row_group, idx_t vector_index, idx_t max_row);
 	void InitializeParallelScan(ParallelCollectionScanState &state);
 	bool NextParallelScan(ClientContext &context, ParallelCollectionScanState &state, CollectionScanState &scan_state);
 
-	bool Scan(DuckTransaction &transaction, const vector<column_t> &column_ids,
+	bool Scan(DuckTransaction &transaction, const vector<StorageIndex> &column_ids,
 	          const std::function<bool(DataChunk &chunk)> &fun);
 	bool Scan(DuckTransaction &transaction, const std::function<bool(DataChunk &chunk)> &fun);
 
-	void Fetch(TransactionData transaction, DataChunk &result, const vector<column_t> &column_ids,
+	void Fetch(TransactionData transaction, DataChunk &result, const vector<StorageIndex> &column_ids,
 	           const Vector &row_identifiers, idx_t fetch_count, ColumnFetchState &state);
 
 	//! Initialize an append of a variable number of rows. FinalizeAppend must be called after appending is done.
@@ -106,6 +111,7 @@ public:
 	void CommitDropColumn(idx_t index);
 	void CommitDropTable();
 
+	vector<PartitionStatistics> GetPartitionStats() const;
 	vector<ColumnSegmentInfo> GetColumnSegmentInfo();
 	const vector<LogicalType> &GetTypes() const;
 
@@ -113,11 +119,12 @@ public:
 	                                         ExpressionExecutor &default_executor);
 	shared_ptr<RowGroupCollection> RemoveColumn(idx_t col_idx);
 	shared_ptr<RowGroupCollection> AlterType(ClientContext &context, idx_t changed_idx, const LogicalType &target_type,
-	                                         vector<column_t> bound_columns, Expression &cast_expr);
+	                                         vector<StorageIndex> bound_columns, Expression &cast_expr);
 	void VerifyNewConstraint(DataTable &parent, const BoundConstraint &constraint);
 
 	void CopyStats(TableStatistics &stats);
 	unique_ptr<BaseStatistics> CopyStats(column_t column_id);
+	unique_ptr<BlockingSample> GetSample();
 	void SetDistinct(column_t column_id, unique_ptr<DistinctStatistics> distinct_stats);
 
 	AttachedDatabase &GetAttached();
@@ -133,12 +140,18 @@ public:
 		return allocation_size;
 	}
 
+	idx_t GetRowGroupSize() const {
+		return row_group_size;
+	}
+
 private:
 	bool IsEmpty(SegmentLock &) const;
 
 private:
 	//! BlockManager
 	BlockManager &block_manager;
+	//! The row group size of the row group collection
+	const idx_t row_group_size;
 	//! The number of rows in the table
 	atomic<idx_t> total_rows;
 	//! The data table info

package/src/duckdb/src/include/duckdb/storage/table/row_version_manager.hpp

@@ -49,13 +49,14 @@ public:
 private:
 	mutex version_lock;
 	idx_t start;
-	unique_ptr<ChunkInfo> vector_info[Storage::ROW_GROUP_VECTOR_COUNT];
+	vector<unique_ptr<ChunkInfo>> vector_info;
 	bool has_changes;
 	vector<MetaBlockPointer> storage_pointers;
 
 private:
 	optional_ptr<ChunkInfo> GetChunkInfo(idx_t vector_idx);
 	ChunkVectorInfo &GetVectorInfo(idx_t vector_idx);
+	void FillVectorInfo(idx_t vector_idx);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/scan_state.hpp

@@ -13,8 +13,11 @@
 #include "duckdb/storage/buffer/buffer_handle.hpp"
 #include "duckdb/storage/storage_lock.hpp"
 #include "duckdb/common/enums/scan_options.hpp"
+#include "duckdb/common/random_engine.hpp"
 #include "duckdb/storage/table/segment_lock.hpp"
 #include "duckdb/common/types/data_chunk.hpp"
+#include "duckdb/parser/parsed_data/sample_options.hpp"
+#include "duckdb/storage/storage_index.hpp"
 
 namespace duckdb {
 class AdaptiveFilter;
@@ -36,6 +39,7 @@ class RowGroupSegmentTree;
 class TableFilter;
 struct AdaptiveFilterState;
 struct TableScanOptions;
+struct ScanSamplingInfo;
 
 struct SegmentScanState {
 	virtual ~SegmentScanState() {
@@ -93,10 +97,14 @@ struct ColumnScanState {
 	vector<unique_ptr<SegmentScanState>> previous_states;
 	//! The last read offset in the child state (used for LIST columns only)
 	idx_t last_offset = 0;
+	//! Whether or not we should scan a specific child column
+	vector<bool> scan_child_column;
 	//! Contains TableScan level config for scanning
 	optional_ptr<TableScanOptions> scan_options;
 
 public:
+	void Initialize(const LogicalType &type, const vector<StorageIndex> &children,
+	                optional_ptr<TableScanOptions> options);
 	void Initialize(const LogicalType &type, optional_ptr<TableScanOptions> options);
 	//! Move the scan state forward by "count" rows (including all child states)
 	void Next(idx_t count);
@@ -114,7 +122,7 @@ struct ColumnFetchState {
 };
 
 struct ScanFilter {
-	ScanFilter(idx_t index, const vector<column_t> &column_ids, TableFilter &filter);
+	ScanFilter(idx_t index, const vector<StorageIndex> &column_ids, TableFilter &filter);
 
 	idx_t scan_column_index;
 	idx_t table_column_index;
@@ -130,7 +138,7 @@ class ScanFilterInfo {
 public:
 	~ScanFilterInfo();
 
-	void Initialize(TableFilterSet &filters, const vector<column_t> &column_ids);
+	void Initialize(TableFilterSet &filters, const vector<StorageIndex> &column_ids);
 
 	const vector<ScanFilter> &GetFilterList() const {
 		return filter_list;
@@ -188,10 +196,13 @@ public:
 	//! The valid selection
 	SelectionVector valid_sel;
 
+	RandomEngine random;
+
 public:
 	void Initialize(const vector<LogicalType> &types);
-	const vector<storage_t> &GetColumnIds();
+	const vector<StorageIndex> &GetColumnIds();
 	ScanFilterInfo &GetFilterInfo();
+	ScanSamplingInfo &GetSamplingInfo();
 	TableScanOptions &GetOptions();
 	bool Scan(DuckTransaction &transaction, DataChunk &result);
 	bool ScanCommitted(DataChunk &result, TableScanType type);
@@ -201,6 +212,13 @@ private:
 	TableScanState &parent;
 };
 
+struct ScanSamplingInfo {
+	//! Whether or not to do a system sample during scanning
+	bool do_system_sample = false;
+	//! The sampling rate to use
+	double sample_rate;
+};
+
 struct TableScanOptions {
 	//! Fetch rows one-at-a-time instead of using the regular scans.
 	bool force_fetch_row = false;
@@ -230,17 +248,22 @@ public:
 	shared_ptr<CheckpointLock> checkpoint_lock;
 	//! Filter info
 	ScanFilterInfo filters;
+	//! Sampling info
+	ScanSamplingInfo sampling_info;
 
 public:
-	void Initialize(vector<storage_t> column_ids, optional_ptr<TableFilterSet> table_filters = nullptr);
+	void Initialize(vector<StorageIndex> column_ids, optional_ptr<TableFilterSet> table_filters = nullptr,
+	                optional_ptr<SampleOptions> table_sampling = nullptr);
 
-	const vector<storage_t> &GetColumnIds();
+	const vector<StorageIndex> &GetColumnIds();
 
 	ScanFilterInfo &GetFilterInfo();
 
+	ScanSamplingInfo &GetSamplingInfo();
+
 private:
 	//! The column identifiers of the scan
-	vector<storage_t> column_ids;
+	vector<StorageIndex> column_ids;
 };
 
 struct ParallelCollectionScanState {

package/src/duckdb/src/include/duckdb/storage/table/segment_tree.hpp

@@ -67,6 +67,16 @@ public:
 		auto l = Lock();
 		return MoveSegments(l);
 	}
+
+	const vector<SegmentNode<T>> &ReferenceSegments(SegmentLock &l) {
+		LoadAllSegments(l);
+		return nodes;
+	}
+	const vector<SegmentNode<T>> &ReferenceSegments() {
+		auto l = Lock();
+		return ReferenceSegments(l);
+	}
+
 	idx_t GetSegmentCount() {
 		auto l = Lock();
 		return GetSegmentCount(l);
@@ -167,16 +177,6 @@ public:
 		return segment->index < nodes.size() && nodes[segment->index].node.get() == segment;
 	}
 
-	//! Replace this tree with another tree, taking over its nodes in-place
-	void Replace(SegmentTree<T> &other) {
-		auto l = Lock();
-		Replace(l, other);
-	}
-	void Replace(SegmentLock &l, SegmentTree<T> &other) {
-		other.LoadAllSegments(l);
-		nodes = std::move(other.nodes);
-	}
-
 	//! Erase all segments after a specific segment
 	void EraseSegments(SegmentLock &l, idx_t segment_start) {
 		LoadAllSegments(l);

package/src/duckdb/src/include/duckdb/storage/table/standard_column_data.hpp

@@ -36,6 +36,11 @@ public:
 	                    idx_t target_count) override;
 	idx_t ScanCount(ColumnScanState &state, Vector &result, idx_t count) override;
 
+	void Filter(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
+	            SelectionVector &sel, idx_t &count, const TableFilter &filter) override;
+	void Select(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
+	            SelectionVector &sel, idx_t sel_count) override;
+
 	void InitializeAppend(ColumnAppendState &state) override;
 	void AppendData(BaseStatistics &stats, ColumnAppendState &state, UnifiedVectorFormat &vdata, idx_t count) override;
 	void RevertAppend(row_t start_row) override;