duckdb 0.7.2-dev12.0 → 0.7.2-dev1244.0

package/src/duckdb/src/include/duckdb/storage/table/column_data.hpp

@@ -9,13 +9,12 @@
 #pragma once
 
 #include "duckdb/common/types/data_chunk.hpp"
-#include "duckdb/storage/table/append_state.hpp"
-#include "duckdb/storage/table/scan_state.hpp"
 #include "duckdb/storage/statistics/base_statistics.hpp"
 #include "duckdb/storage/data_pointer.hpp"
 #include "duckdb/storage/table/persistent_table_data.hpp"
 #include "duckdb/storage/statistics/segment_statistics.hpp"
-#include "duckdb/storage/table/column_checkpoint_state.hpp"
+#include "duckdb/storage/table/segment_tree.hpp"
+#include "duckdb/storage/table/column_segment_tree.hpp"
 #include "duckdb/common/mutex.hpp"
 
 namespace duckdb {
@@ -31,7 +30,7 @@ struct TransactionData;
 struct DataTableInfo;
 
 struct ColumnCheckpointInfo {
-	ColumnCheckpointInfo(CompressionType compression_type_p) : compression_type(compression_type_p) {};
+	explicit ColumnCheckpointInfo(CompressionType compression_type_p) : compression_type(compression_type_p) {};
 	CompressionType compression_type;
 };
 
@@ -44,14 +43,16 @@ public:
 	ColumnData(ColumnData &other, idx_t start, ColumnData *parent);
 	virtual ~ColumnData();
 
+	//! The start row
+	const idx_t start;
+	//! The count of the column data
+	idx_t count;
 	//! The block manager
 	BlockManager &block_manager;
 	//! Table info for the column
 	DataTableInfo &info;
 	//! The column index of the column, either within the parent table or within the parent
 	idx_t column_index;
-	//! The start row
-	idx_t start;
 	//! The type of the column
 	LogicalType type;
 	//! The parent column (if any)
@@ -60,6 +61,9 @@ public:
 public:
 	virtual bool CheckZonemap(ColumnScanState &state, TableFilter &filter) = 0;
 
+	BlockManager &GetBlockManager() {
+		return block_manager;
+	}
 	DatabaseInstance &GetDatabase() const;
 	DataTableInfo &GetTableInfo() const;
 	virtual idx_t GetMaxEntry();
@@ -93,6 +97,8 @@ public:
 	virtual void InitializeAppend(ColumnAppendState &state);
 	//! Append a vector of type [type] to the end of the column
 	virtual void Append(BaseStatistics &stats, ColumnAppendState &state, Vector &vector, idx_t count);
+	//! Append a vector of type [type] to the end of the column
+	void Append(ColumnAppendState &state, Vector &vector, idx_t count);
 	virtual void AppendData(BaseStatistics &stats, ColumnAppendState &state, UnifiedVectorFormat &vdata, idx_t count);
 	//! Revert a set of appends to the ColumnData
 	virtual void RevertAppend(row_t start_row);
@@ -127,6 +133,8 @@ public:
 	virtual void GetStorageInfo(idx_t row_group_index, vector<idx_t> col_path, TableStorageInfo &result);
 	virtual void Verify(RowGroup &parent);
 
+	bool CheckZonemap(TableFilter &filter);
+
 	static shared_ptr<ColumnData> CreateColumn(BlockManager &block_manager, DataTableInfo &info, idx_t column_index,
 	                                           idx_t start_row, const LogicalType &type, ColumnData *parent = nullptr);
 	static shared_ptr<ColumnData> CreateColumn(ColumnData &other, idx_t start_row, ColumnData *parent = nullptr);
@@ -135,6 +143,10 @@ public:
 	                                                 ColumnData *parent = nullptr);
 	static unique_ptr<ColumnData> CreateColumnUnique(ColumnData &other, idx_t start_row, ColumnData *parent = nullptr);
 
+	void MergeStatistics(const BaseStatistics &other);
+	void MergeIntoStatistics(BaseStatistics &other);
+	unique_ptr<BaseStatistics> GetStatistics();
+
 protected:
 	//! Append a transient segment
 	void AppendTransientSegment(SegmentLock &l, idx_t start_row);
@@ -148,13 +160,15 @@ protected:
 
 protected:
 	//! The segments holding the data of this column segment
-	SegmentTree data;
+	ColumnSegmentTree data;
 	//! The lock for the updates
 	mutex update_lock;
 	//! The updates for this column segment
 	unique_ptr<UpdateSegment> updates;
 	//! The internal version of the column data
 	idx_t version;
+	//! The stats of the root segment
+	unique_ptr<SegmentStatistics> stats;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/column_data_checkpointer.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/storage/table/column_data.hpp"
 #include "duckdb/function/compression_function.hpp"
+#include "duckdb/storage/table/column_checkpoint_state.hpp"
 
 namespace duckdb {
 
@@ -25,7 +26,7 @@ public:
 	RowGroup &GetRowGroup();
 	ColumnCheckpointState &GetCheckpointState();
 
-	void Checkpoint(vector<SegmentNode> nodes);
+	void Checkpoint(vector<SegmentNode<ColumnSegment>> nodes);
 
 private:
 	void ScanSegments(const std::function<void(Vector &, idx_t)> &callback);
@@ -40,7 +41,7 @@ private:
 	ColumnCheckpointState &state;
 	bool is_validity;
 	Vector intermediate;
-	vector<SegmentNode> nodes;
+	vector<SegmentNode<ColumnSegment>> nodes;
 	vector<CompressionFunction *> compression_functions;
 	ColumnCheckpointInfo &checkpoint_info;
 };

package/src/duckdb/src/include/duckdb/storage/table/column_segment.hpp

@@ -9,14 +9,13 @@
 #pragma once
 
 #include "duckdb/storage/block.hpp"
-#include "duckdb/storage/table/segment_tree.hpp"
 #include "duckdb/common/types.hpp"
 #include "duckdb/common/types/vector.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
 #include "duckdb/storage/statistics/segment_statistics.hpp"
 #include "duckdb/storage/storage_lock.hpp"
-#include "duckdb/storage/table/scan_state.hpp"
 #include "duckdb/function/compression_function.hpp"
+#include "duckdb/storage/table/segment_base.hpp"
 
 namespace duckdb {
 class ColumnSegment;
@@ -35,9 +34,9 @@ struct ColumnAppendState;
 enum class ColumnSegmentType : uint8_t { TRANSIENT, PERSISTENT };
 //! TableFilter represents a filter pushed down into the table scan.
 
-class ColumnSegment : public SegmentBase {
+class ColumnSegment : public SegmentBase<ColumnSegment> {
 public:
-	~ColumnSegment() override;
+	~ColumnSegment();
 
 	//! The database instance
 	DatabaseInstance &db;
@@ -57,7 +56,7 @@ public:
 	static unique_ptr<ColumnSegment> CreatePersistentSegment(DatabaseInstance &db, BlockManager &block_manager,
 	                                                         block_id_t id, idx_t offset, const LogicalType &type_p,
 	                                                         idx_t start, idx_t count, CompressionType compression_type,
-	                                                         unique_ptr<BaseStatistics> statistics);
+	                                                         BaseStatistics statistics);
 	static unique_ptr<ColumnSegment> CreateTransientSegment(DatabaseInstance &db, const LogicalType &type, idx_t start,
 	                                                        idx_t segment_size = Storage::BLOCK_SIZE);
 	static unique_ptr<ColumnSegment> CreateSegment(ColumnSegment &other, idx_t start);
@@ -124,7 +123,7 @@ public:
 
 public:
 	ColumnSegment(DatabaseInstance &db, shared_ptr<BlockHandle> block, LogicalType type, ColumnSegmentType segment_type,
-	              idx_t start, idx_t count, CompressionFunction *function, unique_ptr<BaseStatistics> statistics,
+	              idx_t start, idx_t count, CompressionFunction *function, BaseStatistics statistics,
 	              block_id_t block_id, idx_t offset, idx_t segment_size);
 	ColumnSegment(ColumnSegment &other, idx_t start);
 

package/src/duckdb/src/include/duckdb/storage/table/column_segment_tree.hpp

@@ -0,0 +1,18 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/table/column_segment_tree.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/storage/table/segment_tree.hpp"
+#include "duckdb/storage/table/column_segment.hpp"
+
+namespace duckdb {
+
+class ColumnSegmentTree : public SegmentTree<ColumnSegment> {};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/list_column_data.hpp

@@ -61,7 +61,7 @@ public:
 	void GetStorageInfo(idx_t row_group_index, vector<idx_t> col_path, TableStorageInfo &result) override;
 
 private:
-	list_entry_t FetchListEntry(idx_t row_idx);
+	uint64_t FetchListOffset(idx_t row_idx);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/persistent_table_data.hpp

@@ -10,8 +10,8 @@
 
 #include "duckdb/common/constants.hpp"
 #include "duckdb/common/vector.hpp"
-#include "duckdb/storage/table/segment_tree.hpp"
 #include "duckdb/storage/data_pointer.hpp"
+#include "duckdb/storage/table/table_statistics.hpp"
 
 namespace duckdb {
 class BaseStatistics;
@@ -21,8 +21,11 @@ public:
 	explicit PersistentTableData(idx_t column_count);
 	~PersistentTableData();
 
-	vector<RowGroupPointer> row_groups;
-	vector<unique_ptr<BaseStatistics>> column_stats;
+	TableStatistics table_stats;
+	idx_t total_rows;
+	idx_t row_group_count;
+	block_id_t block_id;
+	idx_t offset;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp

@@ -9,14 +9,14 @@
 #pragma once
 
 #include "duckdb/common/vector_size.hpp"
-#include "duckdb/storage/table/segment_base.hpp"
 #include "duckdb/storage/table/chunk_info.hpp"
-#include "duckdb/storage/table/append_state.hpp"
-#include "duckdb/storage/table/scan_state.hpp"
 #include "duckdb/storage/statistics/segment_statistics.hpp"
+#include "duckdb/common/types/data_chunk.hpp"
 #include "duckdb/common/enums/scan_options.hpp"
 #include "duckdb/common/mutex.hpp"
 #include "duckdb/parser/column_list.hpp"
+#include "duckdb/storage/table/segment_base.hpp"
+#include "duckdb/storage/block.hpp"
 
 namespace duckdb {
 class AttachedDatabase;
@@ -27,21 +27,27 @@ class DataTable;
 class PartialBlockManager;
 struct DataTableInfo;
 class ExpressionExecutor;
+class RowGroupCollection;
 class RowGroupWriter;
 class UpdateSegment;
+class TableStatistics;
 class TableStorageInfo;
 class Vector;
 struct ColumnCheckpointState;
 struct RowGroupPointer;
 struct TransactionData;
 struct VersionNode;
+class CollectionScanState;
+class TableFilterSet;
+struct ColumnFetchState;
+struct RowGroupAppendState;
 
 struct RowGroupWriteData {
 	vector<unique_ptr<ColumnCheckpointState>> states;
-	vector<unique_ptr<BaseStatistics>> statistics;
+	vector<BaseStatistics> statistics;
 };
 
-class RowGroup : public SegmentBase {
+class RowGroup : public SegmentBase<RowGroup> {
 public:
 	friend class ColumnData;
 	friend class VersionDeleteState;
@@ -51,48 +57,33 @@ public:
 	static constexpr const idx_t ROW_GROUP_VECTOR_COUNT = ROW_GROUP_SIZE / STANDARD_VECTOR_SIZE;
 
 public:
-	RowGroup(AttachedDatabase &db, BlockManager &block_manager, DataTableInfo &table_info, idx_t start, idx_t count);
-	RowGroup(AttachedDatabase &db, BlockManager &block_manager, DataTableInfo &table_info,
-	         const vector<LogicalType> &types, RowGroupPointer &&pointer);
-	RowGroup(RowGroup &row_group, idx_t start);
+	RowGroup(RowGroupCollection &collection, idx_t start, idx_t count);
+	RowGroup(RowGroupCollection &collection, RowGroupPointer &&pointer);
+	RowGroup(RowGroup &row_group, RowGroupCollection &collection, idx_t start);
 	~RowGroup();
 
 private:
-	//! The database instance
-	AttachedDatabase &db;
-	//! The block manager
-	BlockManager &block_manager;
-	//! The table info of this row_group
-	DataTableInfo &table_info;
+	//! The RowGroupCollection this row-group is a part of
+	RowGroupCollection &collection;
 	//! The version info of the row_group (inserted and deleted tuple info)
 	shared_ptr<VersionNode> version_info;
 	//! The column data of the row_group
 	vector<shared_ptr<ColumnData>> columns;
-	//! The segment statistics for each of the columns
-	vector<shared_ptr<SegmentStatistics>> stats;
 
 public:
-	DatabaseInstance &GetDatabase();
-	BlockManager &GetBlockManager() {
-		return block_manager;
-	}
-	DataTableInfo &GetTableInfo() {
-		return table_info;
-	}
-	idx_t GetColumnIndex(ColumnData *data) {
-		for (idx_t i = 0; i < columns.size(); i++) {
-			if (columns[i].get() == data) {
-				return i;
-			}
-		}
-		return 0;
+	RowGroupCollection &GetCollection() {
+		return collection;
 	}
+	DatabaseInstance &GetDatabase();
+	BlockManager &GetBlockManager();
+	DataTableInfo &GetTableInfo();
 
-	unique_ptr<RowGroup> AlterType(const LogicalType &target_type, idx_t changed_idx, ExpressionExecutor &executor,
-	                               RowGroupScanState &scan_state, DataChunk &scan_chunk);
-	unique_ptr<RowGroup> AddColumn(ColumnDefinition &new_column, ExpressionExecutor &executor,
-	                               Expression *default_value, Vector &intermediate);
-	unique_ptr<RowGroup> RemoveColumn(idx_t removed_column);
+	unique_ptr<RowGroup> AlterType(RowGroupCollection &collection, const LogicalType &target_type, idx_t changed_idx,
+	                               ExpressionExecutor &executor, CollectionScanState &scan_state,
+	                               DataChunk &scan_chunk);
+	unique_ptr<RowGroup> AddColumn(RowGroupCollection &collection, ColumnDefinition &new_column,
+	                               ExpressionExecutor &executor, Expression *default_value, Vector &intermediate);
+	unique_ptr<RowGroup> RemoveColumn(RowGroupCollection &collection, idx_t removed_column);
 
 	void CommitDrop();
 	void CommitDropColumn(idx_t index);
@@ -100,16 +91,16 @@ public:
 	void InitializeEmpty(const vector<LogicalType> &types);
 
 	//! Initialize a scan over this row_group
-	bool InitializeScan(RowGroupScanState &state);
-	bool InitializeScanWithOffset(RowGroupScanState &state, idx_t vector_offset);
+	bool InitializeScan(CollectionScanState &state);
+	bool InitializeScanWithOffset(CollectionScanState &state, idx_t vector_offset);
 	//! Checks the given set of table filters against the row-group statistics. Returns false if the entire row group
 	//! can be skipped.
 	bool CheckZonemap(TableFilterSet &filters, const vector<column_t> &column_ids);
 	//! Checks the given set of table filters against the per-segment statistics. Returns false if any segments were
 	//! skipped.
-	bool CheckZonemapSegments(RowGroupScanState &state);
-	void Scan(TransactionData transaction, RowGroupScanState &state, DataChunk &result);
-	void ScanCommitted(RowGroupScanState &state, DataChunk &result, TableScanType type);
+	bool CheckZonemapSegments(CollectionScanState &state);
+	void Scan(TransactionData transaction, CollectionScanState &state, DataChunk &result);
+	void ScanCommitted(CollectionScanState &state, DataChunk &result, TableScanType type);
 
 	idx_t GetSelVector(TransactionData transaction, idx_t vector_idx, SelectionVector &sel_vector, idx_t max_count);
 	idx_t GetCommittedSelVector(transaction_t start_time, transaction_t transaction_id, idx_t vector_idx,
@@ -132,9 +123,9 @@ public:
 	idx_t Delete(TransactionData transaction, DataTable *table, row_t *row_ids, idx_t count);
 
 	RowGroupWriteData WriteToDisk(PartialBlockManager &manager, const vector<CompressionType> &compression_types);
-	RowGroupPointer Checkpoint(RowGroupWriter &writer, vector<unique_ptr<BaseStatistics>> &global_stats);
+	RowGroupPointer Checkpoint(RowGroupWriter &writer, TableStatistics &global_stats);
 	static void Serialize(RowGroupPointer &pointer, Serializer &serializer);
-	static RowGroupPointer Deserialize(Deserializer &source, const ColumnList &columns);
+	static RowGroupPointer Deserialize(Deserializer &source, const vector<LogicalType> &columns);
 
 	void InitializeAppend(RowGroupAppendState &append_state);
 	void Append(RowGroupAppendState &append_state, DataChunk &chunk, idx_t append_count);
@@ -154,13 +145,16 @@ public:
 
 	void Verify();
 
-	void NextVector(RowGroupScanState &state);
+	void NextVector(CollectionScanState &state);
 
 private:
 	ChunkInfo *GetChunkInfo(idx_t vector_idx);
+	ColumnData &GetColumn(idx_t c);
+	idx_t GetColumnCount() const;
+	vector<shared_ptr<ColumnData>> &GetColumns();
 
 	template <TableScanType TYPE>
-	void TemplatedScan(TransactionData transaction, RowGroupScanState &state, DataChunk &result);
+	void TemplatedScan(TransactionData transaction, CollectionScanState &state, DataChunk &result);
 
 	static void CheckpointDeletes(VersionNode *versions, Serializer &serializer);
 	static shared_ptr<VersionNode> DeserializeDeletes(Deserializer &source);
@@ -168,6 +162,8 @@ private:
 private:
 	mutex row_group_lock;
 	mutex stats_lock;
+	vector<BlockPointer> column_pointers;
+	unique_ptr<atomic<bool>[]> is_loaded;
 };
 
 struct VersionNode {

package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp

@@ -15,13 +15,17 @@
 
 namespace duckdb {
 struct ParallelTableScanState;
-
+struct ParallelCollectionScanState;
+class CreateIndexScanState;
+class CollectionScanState;
 class PersistentTableData;
 class TableDataWriter;
 class TableIndexList;
 class TableStatistics;
-
+struct TableAppendState;
+class DuckTransaction;
 class BoundConstraint;
+class RowGroupSegmentTree;
 
 class RowGroupCollection {
 public:
@@ -46,8 +50,8 @@ public:
 	void InitializeCreateIndexScan(CreateIndexScanState &state);
 	void InitializeScanWithOffset(CollectionScanState &state, const vector<column_t> &column_ids, idx_t start_row,
 	                              idx_t end_row);
-	static bool InitializeScanInRowGroup(CollectionScanState &state, RowGroup *row_group, idx_t vector_index,
-	                                     idx_t max_row);
+	static bool InitializeScanInRowGroup(CollectionScanState &state, RowGroupCollection &collection,
+	                                     RowGroup &row_group, idx_t vector_index, idx_t max_row);
 	void InitializeParallelScan(ParallelCollectionScanState &state);
 	bool NextParallelScan(ClientContext &context, ParallelCollectionScanState &state, CollectionScanState &scan_state);
 
@@ -78,7 +82,7 @@ public:
 	void UpdateColumn(TransactionData transaction, Vector &row_ids, const vector<column_t> &column_path,
 	                  DataChunk &updates);
 
-	void Checkpoint(TableDataWriter &writer, vector<unique_ptr<BaseStatistics>> &global_stats);
+	void Checkpoint(TableDataWriter &writer, TableStatistics &global_stats);
 
 	void CommitDropColumn(idx_t index);
 	void CommitDropTable();
@@ -93,8 +97,18 @@ public:
 	                                         vector<column_t> bound_columns, Expression &cast_expr);
 	void VerifyNewConstraint(DataTable &parent, const BoundConstraint &constraint);
 
+	void CopyStats(TableStatistics &stats);
 	unique_ptr<BaseStatistics> CopyStats(column_t column_id);
-	void SetStatistics(column_t column_id, const std::function<void(BaseStatistics &)> &set_fun);
+	void SetDistinct(column_t column_id, unique_ptr<DistinctStatistics> distinct_stats);
+
+	AttachedDatabase &GetAttached();
+	DatabaseInstance &GetDatabase();
+	BlockManager &GetBlockManager() {
+		return block_manager;
+	}
+	DataTableInfo &GetTableInfo() {
+		return *info;
+	}
 
 private:
 	bool IsEmpty(SegmentLock &) const;
@@ -104,11 +118,13 @@ private:
 	BlockManager &block_manager;
 	//! The number of rows in the table
 	atomic<idx_t> total_rows;
+	//! The data table info
 	shared_ptr<DataTableInfo> info;
+	//! The column types of the row group collection
 	vector<LogicalType> types;
 	idx_t row_start;
 	//! The segment trees holding the various row_groups of the table
-	shared_ptr<SegmentTree> row_groups;
+	shared_ptr<RowGroupSegmentTree> row_groups;
 	//! Table statistics
 	TableStatistics stats;
 };

package/src/duckdb/src/include/duckdb/storage/table/row_group_segment_tree.hpp

@@ -0,0 +1,35 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/table/row_group_segment_tree.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/storage/table/segment_tree.hpp"
+#include "duckdb/storage/table/row_group.hpp"
+
+namespace duckdb {
+struct DataTableInfo;
+class PersistentTableData;
+class MetaBlockReader;
+
+class RowGroupSegmentTree : public SegmentTree<RowGroup, true> {
+public:
+	RowGroupSegmentTree(RowGroupCollection &collection);
+	~RowGroupSegmentTree() override;
+
+	void Initialize(PersistentTableData &data);
+
+protected:
+	unique_ptr<RowGroup> LoadSegment() override;
+
+	RowGroupCollection &collection;
+	idx_t current_row_group;
+	idx_t max_row_group;
+	unique_ptr<MetaBlockReader> reader;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/scan_state.hpp

@@ -25,10 +25,12 @@ class RowGroupCollection;
 class UpdateSegment;
 class TableScanState;
 class ColumnSegment;
+class ColumnSegmentTree;
 class ValiditySegment;
 class TableFilterSet;
 class ColumnData;
 class DuckTransaction;
+class RowGroupSegmentTree;
 
 struct SegmentScanState {
 	virtual ~SegmentScanState() {
@@ -45,6 +47,8 @@ typedef unordered_map<block_id_t, BufferHandle> buffer_handle_set_t;
 struct ColumnScanState {
 	//! The column segment that is currently being scanned
 	ColumnSegment *current = nullptr;
+	//! Column segment tree
+	ColumnSegmentTree *segment_tree = nullptr;
 	//! The current row index of the scan
 	idx_t row_index = 0;
 	//! The internal row index (i.e. the position of the SegmentScanState)
@@ -64,14 +68,15 @@ struct ColumnScanState {
 	//! We initialize one SegmentScanState per segment, however, if scanning a DataChunk requires us to scan over more
 	//! than one Segment, we need to keep the scan states of the previous segments around
 	vector<unique_ptr<SegmentScanState>> previous_states;
+	//! The last read offset in the child state (used for LIST columns only)
+	idx_t last_offset = 0;
 
 public:
+	void Initialize(const LogicalType &type);
 	//! Move the scan state forward by "count" rows (including all child states)
 	void Next(idx_t count);
 	//! Move ONLY this state forward by "count" rows (i.e. not the child states)
 	void NextInternal(idx_t count);
-	//! Move the scan state forward by STANDARD_VECTOR_SIZE rows
-	void NextVector();
 };
 
 struct ColumnFetchState {
@@ -83,50 +88,33 @@ struct ColumnFetchState {
 	BufferHandle &GetOrInsertHandle(ColumnSegment &segment);
 };
 
-class RowGroupScanState {
+class CollectionScanState {
 public:
-	RowGroupScanState(CollectionScanState &parent_p)
-	    : row_group(nullptr), vector_index(0), max_row(0), parent(parent_p) {
-	}
+	CollectionScanState(TableScanState &parent_p);
 
 	//! The current row_group we are scanning
-	RowGroup *row_group = nullptr;
+	RowGroup *row_group;
 	//! The vector index within the row_group
-	idx_t vector_index = 0;
-	//! The maximum row index of this row_group scan
-	idx_t max_row = 0;
+	idx_t vector_index;
+	//! The maximum row within the row group
+	idx_t max_row_group_row;
 	//! Child column scans
 	unique_ptr<ColumnScanState[]> column_scans;
-
-public:
-	const vector<column_t> &GetColumnIds();
-	TableFilterSet *GetFilters();
-	AdaptiveFilter *GetAdaptiveFilter();
-	idx_t GetParentMaxRow();
-
-private:
-	//! The parent scan state
-	CollectionScanState &parent;
-};
-
-class CollectionScanState {
-public:
-	CollectionScanState(TableScanState &parent_p)
-	    : row_group_state(*this), max_row(0), batch_index(0), parent(parent_p) {};
-
-	//! The row_group scan state
-	RowGroupScanState row_group_state;
+	//! Row group segment tree
+	RowGroupSegmentTree *row_groups;
 	//! The total maximum row index
 	idx_t max_row;
 	//! The current batch index
 	idx_t batch_index;
 
 public:
+	void Initialize(const vector<LogicalType> &types);
 	const vector<column_t> &GetColumnIds();
 	TableFilterSet *GetFilters();
 	AdaptiveFilter *GetAdaptiveFilter();
 	bool Scan(DuckTransaction &transaction, DataChunk &result);
 	bool ScanCommitted(DataChunk &result, TableScanType type);
+	bool ScanCommitted(DataChunk &result, SegmentLock &l, TableScanType type);
 
 private:
 	TableScanState &parent;
@@ -158,12 +146,16 @@ private:
 };
 
 struct ParallelCollectionScanState {
+	ParallelCollectionScanState();
+
 	//! The row group collection we are scanning
 	RowGroupCollection *collection;
 	RowGroup *current_row_group;
 	idx_t vector_index;
 	idx_t max_row;
 	idx_t batch_index;
+	atomic<idx_t> processed_rows;
+	mutex lock;
 };
 
 struct ParallelTableScanState {

package/src/duckdb/src/include/duckdb/storage/table/segment_base.hpp

@@ -13,13 +13,12 @@
 
 namespace duckdb {
 
+template <class T>
 class SegmentBase {
 public:
 	SegmentBase(idx_t start, idx_t count) : start(start), count(count), next(nullptr) {
 	}
-	virtual ~SegmentBase() {
-	}
-	SegmentBase *Next() {
+	T *Next() {
 #ifndef DUCKDB_R_BUILD
 		return next.load();
 #else
@@ -32,12 +31,13 @@ public:
 	//! The amount of entries in this storage chunk
 	atomic<idx_t> count;
 	//! The next segment after this one
-
 #ifndef DUCKDB_R_BUILD
-	atomic<SegmentBase *> next;
+	atomic<T *> next;
 #else
-	SegmentBase *next;
+	T *next;
 #endif
+	//! The index within the segment tree
+	idx_t index;
 };
 
 } // namespace duckdb