duckdb 0.7.2-dev1034.0 → 0.7.2-dev1138.0

package/src/duckdb/src/include/duckdb/common/types/string_type.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/common/assert.hpp"
 #include "duckdb/common/constants.hpp"
+#include "duckdb/common/helper.hpp"
 
 #include <cstring>
 
@@ -117,10 +118,79 @@ public:
 
 	void Verify() const;
 	void VerifyNull() const;
+
+	struct StringComparisonOperators {
+		static inline bool Equals(const string_t &a, const string_t &b) {
+#ifdef DUCKDB_DEBUG_NO_INLINE
+			if (a.GetSize() != b.GetSize())
+				return false;
+			return (memcmp(a.GetDataUnsafe(), b.GetDataUnsafe(), a.GetSize()) == 0);
+#endif
+			uint64_t A = Load<uint64_t>((const_data_ptr_t)&a);
+			uint64_t B = Load<uint64_t>((const_data_ptr_t)&b);
+			if (A != B) {
+				// Either length or prefix are different -> not equal
+				return false;
+			}
+			// they have the same length and same prefix!
+			A = Load<uint64_t>((const_data_ptr_t)&a + 8u);
+			B = Load<uint64_t>((const_data_ptr_t)&b + 8u);
+			if (A == B) {
+				// either they are both inlined (so compare equal) or point to the same string (so compare equal)
+				return true;
+			}
+			if (!a.IsInlined()) {
+				// 'long' strings of the same length -> compare pointed value
+				if (memcmp(a.value.pointer.ptr, b.value.pointer.ptr, a.GetSize()) == 0) {
+					return true;
+				}
+			}
+			// either they are short string of same length but different content
+			//     or they point to string with different content
+			//     either way, they can't represent the same underlying string
+			return false;
+		}
+		// compare up to shared length. if still the same, compare lengths
+		static bool GreaterThan(const string_t &left, const string_t &right) {
+			const uint32_t left_length = left.GetSize();
+			const uint32_t right_length = right.GetSize();
+			const uint32_t min_length = std::min<uint32_t>(left_length, right_length);
+
+#ifndef DUCKDB_DEBUG_NO_INLINE
+			uint32_t A = Load<uint32_t>((const_data_ptr_t)left.GetPrefix());
+			uint32_t B = Load<uint32_t>((const_data_ptr_t)right.GetPrefix());
+
+			// Utility to move 0xa1b2c3d4 into 0xd4c3b2a1, basically inverting the order byte-a-byte
+			auto bswap = [](uint32_t v) -> uint32_t {
+				uint32_t t1 = (v >> 16u) | (v << 16u);
+				uint32_t t2 = t1 & 0x00ff00ff;
+				uint32_t t3 = t1 & 0xff00ff00;
+				return (t2 << 8u) | (t3 >> 8u);
+			};
+
+			// Check on prefix -----
+			// We dont' need to mask since:
+			//	if the prefix is greater(after bswap), it will stay greater regardless of the extra bytes
+			// 	if the prefix is smaller(after bswap), it will stay smaller regardless of the extra bytes
+			//	if the prefix is equal, the extra bytes are guaranteed to be /0 for the shorter one
+
+			if (A != B)
+				return bswap(A) > bswap(B);
+#endif
+			auto memcmp_res = memcmp(left.GetDataUnsafe(), right.GetDataUnsafe(), min_length);
+			return memcmp_res > 0 || (memcmp_res == 0 && left_length > right_length);
+		}
+	};
+
+	bool operator==(const string_t &r) const {
+		return StringComparisonOperators::Equals(*this, r);
+	}
+
+	bool operator>(const string_t &r) const {
+		return StringComparisonOperators::GreaterThan(*this, r);
+	}
 	bool operator<(const string_t &r) const {
-		auto this_str = this->GetString();
-		auto r_str = r.GetString();
-		return this_str < r_str;
+		return r > *this;
 	}
 
 private:

package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp

@@ -27,18 +27,7 @@
 namespace duckdb {
 
 class ConflictManager;
-
-struct ARTIndexScanState : public IndexScanState {
-
-	//! Scan predicates (single predicate scan or range scan)
-	Value values[2];
-	//! Expressions of the scan predicates
-	ExpressionType expressions[2];
-	bool checked = false;
-	//! All scanned row IDs
-	vector<row_t> result_ids;
-	Iterator iterator;
-};
+struct ARTIndexScanState;
 
 enum class VerifyExistenceType : uint8_t {
 	APPEND = 0,    // appends to a table

package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_reader_options.hpp

@@ -89,6 +89,8 @@ struct BufferedCSVReaderOptions {
 
 	//! How many leading rows to skip
 	idx_t skip_rows = 0;
+	//! Whether or not the skip_rows is set by the user
+	bool skip_rows_set = false;
 	//! Maximum CSV line size: specified because if we reach this amount, we likely have wrong delimiters (default: 2MB)
 	//! note that this is the guaranteed line length that will succeed, longer lines may be accepted if slightly above
 	idx_t maximum_line_size = 2097152;
@@ -116,6 +118,8 @@ struct BufferedCSVReaderOptions {
 	idx_t buffer_size = CSVBuffer::INITIAL_BUFFER_SIZE_COLOSSAL;
 	//! Decimal separator when reading as numeric
 	string decimal_separator = ".";
+	//! Whether or not to pad rows that do not have enough columns with NULL values
+	bool null_padding = true;
 
 	//===--------------------------------------------------------------------===//
 	// WriteCSVOptions

package/src/duckdb/src/include/duckdb/main/client_config.hpp

@@ -40,6 +40,8 @@ struct ClientConfig {
 	//! to output anything
 	bool emit_profiler_output = true;
 
+	//! system-wide progress bar disable.
+	const char *system_progress_bar_disable_reason = nullptr;
 	//! If the progress bar is enabled or not.
 	bool enable_progress_bar = false;
 	//! If the print of the progress bar is enabled

package/src/duckdb/src/include/duckdb/storage/compression/chimp/chimp_scan.hpp

@@ -24,6 +24,7 @@
 
 #include "duckdb/storage/compression/chimp/algorithm/flag_buffer.hpp"
 #include "duckdb/storage/compression/chimp/algorithm/leading_zero_buffer.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/include/duckdb/storage/compression/patas/patas_scan.hpp

@@ -21,6 +21,7 @@
 #include "duckdb/storage/table/column_data_checkpointer.hpp"
 #include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/common/operator/subtract.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/include/duckdb/storage/data_pointer.hpp

@@ -34,8 +34,6 @@ struct RowGroupPointer {
 	uint64_t tuple_count;
 	//! The data pointers of the column segments stored in the row group
 	vector<BlockPointer> data_pointers;
-	//! The per-column statistics of the row group
-	vector<BaseStatistics> statistics;
 	//! The versions information of the row group (if any)
 	shared_ptr<VersionNode> versions;
 };

package/src/duckdb/src/include/duckdb/storage/data_table.hpp

@@ -39,6 +39,7 @@ class Transaction;
 class WriteAheadLog;
 class TableDataWriter;
 class ConflictManager;
+class TableScanState;
 enum class VerifyExistenceType : uint8_t;
 
 //! DataTable represents a physical table on disk

package/src/duckdb/src/include/duckdb/storage/index.hpp

@@ -14,7 +14,6 @@
 #include "duckdb/common/sort/sort.hpp"
 #include "duckdb/parser/parsed_expression.hpp"
 #include "duckdb/planner/expression.hpp"
-#include "duckdb/storage/table/scan_state.hpp"
 #include "duckdb/storage/meta_block_writer.hpp"
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/common/types/constraint_conflict_info.hpp"
@@ -27,6 +26,7 @@ class Transaction;
 class ConflictManager;
 
 struct IndexLock;
+struct IndexScanState;
 
 //! The index is an abstract base class that serves as the basis for indexes
 class Index {

package/src/duckdb/src/include/duckdb/storage/string_uncompressed.hpp

@@ -9,7 +9,7 @@
 #include "duckdb/storage/buffer_manager.hpp"
 #include "duckdb/storage/checkpoint/string_checkpoint_state.hpp"
 #include "duckdb/storage/segment/uncompressed.hpp"
-
+#include "duckdb/storage/table/scan_state.hpp"
 #include "duckdb/storage/string_uncompressed.hpp"
 #include "duckdb/storage/table/append_state.hpp"
 #include "duckdb/storage/table/column_segment.hpp"

package/src/duckdb/src/include/duckdb/storage/table/column_data.hpp

@@ -9,14 +9,12 @@
 #pragma once
 
 #include "duckdb/common/types/data_chunk.hpp"
-#include "duckdb/storage/table/append_state.hpp"
-#include "duckdb/storage/table/scan_state.hpp"
 #include "duckdb/storage/statistics/base_statistics.hpp"
 #include "duckdb/storage/data_pointer.hpp"
 #include "duckdb/storage/table/persistent_table_data.hpp"
 #include "duckdb/storage/statistics/segment_statistics.hpp"
 #include "duckdb/storage/table/segment_tree.hpp"
-#include "duckdb/storage/table/column_segment.hpp"
+#include "duckdb/storage/table/column_segment_tree.hpp"
 #include "duckdb/common/mutex.hpp"
 
 namespace duckdb {
@@ -36,8 +34,6 @@ struct ColumnCheckpointInfo {
 	CompressionType compression_type;
 };
 
-class ColumnSegmentTree : public SegmentTree<ColumnSegment> {};
-
 class ColumnData {
 	friend class ColumnDataCheckpointer;
 
@@ -47,14 +43,16 @@ public:
 	ColumnData(ColumnData &other, idx_t start, ColumnData *parent);
 	virtual ~ColumnData();
 
+	//! The start row
+	const idx_t start;
+	//! The count of the column data
+	idx_t count;
 	//! The block manager
 	BlockManager &block_manager;
 	//! Table info for the column
 	DataTableInfo &info;
 	//! The column index of the column, either within the parent table or within the parent
 	idx_t column_index;
-	//! The start row
-	idx_t start;
 	//! The type of the column
 	LogicalType type;
 	//! The parent column (if any)
@@ -63,6 +61,9 @@ public:
 public:
 	virtual bool CheckZonemap(ColumnScanState &state, TableFilter &filter) = 0;
 
+	BlockManager &GetBlockManager() {
+		return block_manager;
+	}
 	DatabaseInstance &GetDatabase() const;
 	DataTableInfo &GetTableInfo() const;
 	virtual idx_t GetMaxEntry();
@@ -96,6 +97,8 @@ public:
 	virtual void InitializeAppend(ColumnAppendState &state);
 	//! Append a vector of type [type] to the end of the column
 	virtual void Append(BaseStatistics &stats, ColumnAppendState &state, Vector &vector, idx_t count);
+	//! Append a vector of type [type] to the end of the column
+	void Append(ColumnAppendState &state, Vector &vector, idx_t count);
 	virtual void AppendData(BaseStatistics &stats, ColumnAppendState &state, UnifiedVectorFormat &vdata, idx_t count);
 	//! Revert a set of appends to the ColumnData
 	virtual void RevertAppend(row_t start_row);
@@ -130,6 +133,8 @@ public:
 	virtual void GetStorageInfo(idx_t row_group_index, vector<idx_t> col_path, TableStorageInfo &result);
 	virtual void Verify(RowGroup &parent);
 
+	bool CheckZonemap(TableFilter &filter);
+
 	static shared_ptr<ColumnData> CreateColumn(BlockManager &block_manager, DataTableInfo &info, idx_t column_index,
 	                                           idx_t start_row, const LogicalType &type, ColumnData *parent = nullptr);
 	static shared_ptr<ColumnData> CreateColumn(ColumnData &other, idx_t start_row, ColumnData *parent = nullptr);
@@ -138,6 +143,10 @@ public:
 	                                                 ColumnData *parent = nullptr);
 	static unique_ptr<ColumnData> CreateColumnUnique(ColumnData &other, idx_t start_row, ColumnData *parent = nullptr);
 
+	void MergeStatistics(const BaseStatistics &other);
+	void MergeIntoStatistics(BaseStatistics &other);
+	unique_ptr<BaseStatistics> GetStatistics();
+
 protected:
 	//! Append a transient segment
 	void AppendTransientSegment(SegmentLock &l, idx_t start_row);
@@ -158,6 +167,8 @@ protected:
 	unique_ptr<UpdateSegment> updates;
 	//! The internal version of the column data
 	idx_t version;
+	//! The stats of the root segment
+	unique_ptr<SegmentStatistics> stats;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/column_segment.hpp

@@ -14,7 +14,6 @@
 #include "duckdb/storage/buffer_manager.hpp"
 #include "duckdb/storage/statistics/segment_statistics.hpp"
 #include "duckdb/storage/storage_lock.hpp"
-#include "duckdb/storage/table/scan_state.hpp"
 #include "duckdb/function/compression_function.hpp"
 #include "duckdb/storage/table/segment_base.hpp"
 
@@ -39,8 +38,6 @@ class ColumnSegment : public SegmentBase<ColumnSegment> {
 public:
 	~ColumnSegment();
 
-	//! The index within the segment tree
-	idx_t index;
 	//! The database instance
 	DatabaseInstance &db;
 	//! The type stored in the column

package/src/duckdb/src/include/duckdb/storage/table/column_segment_tree.hpp

@@ -0,0 +1,18 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/table/column_segment_tree.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/storage/table/segment_tree.hpp"
+#include "duckdb/storage/table/column_segment.hpp"
+
+namespace duckdb {
+
+class ColumnSegmentTree : public SegmentTree<ColumnSegment> {};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/persistent_table_data.hpp

@@ -10,7 +10,6 @@
 
 #include "duckdb/common/constants.hpp"
 #include "duckdb/common/vector.hpp"
-#include "duckdb/storage/table/segment_tree.hpp"
 #include "duckdb/storage/data_pointer.hpp"
 #include "duckdb/storage/table/table_statistics.hpp"
 

package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp

@@ -10,13 +10,13 @@
 
 #include "duckdb/common/vector_size.hpp"
 #include "duckdb/storage/table/chunk_info.hpp"
-#include "duckdb/storage/table/append_state.hpp"
-#include "duckdb/storage/table/scan_state.hpp"
 #include "duckdb/storage/statistics/segment_statistics.hpp"
+#include "duckdb/common/types/data_chunk.hpp"
 #include "duckdb/common/enums/scan_options.hpp"
 #include "duckdb/common/mutex.hpp"
 #include "duckdb/parser/column_list.hpp"
 #include "duckdb/storage/table/segment_base.hpp"
+#include "duckdb/storage/block.hpp"
 
 namespace duckdb {
 class AttachedDatabase;
@@ -27,6 +27,7 @@ class DataTable;
 class PartialBlockManager;
 struct DataTableInfo;
 class ExpressionExecutor;
+class RowGroupCollection;
 class RowGroupWriter;
 class UpdateSegment;
 class TableStatistics;
@@ -36,6 +37,10 @@ struct ColumnCheckpointState;
 struct RowGroupPointer;
 struct TransactionData;
 struct VersionNode;
+class CollectionScanState;
+class TableFilterSet;
+struct ColumnFetchState;
+struct RowGroupAppendState;
 
 struct RowGroupWriteData {
 	vector<unique_ptr<ColumnCheckpointState>> states;
@@ -52,51 +57,33 @@ public:
 	static constexpr const idx_t ROW_GROUP_VECTOR_COUNT = ROW_GROUP_SIZE / STANDARD_VECTOR_SIZE;
 
 public:
-	RowGroup(AttachedDatabase &db, BlockManager &block_manager, DataTableInfo &table_info, idx_t start, idx_t count);
-	RowGroup(AttachedDatabase &db, BlockManager &block_manager, DataTableInfo &table_info,
-	         const vector<LogicalType> &types, RowGroupPointer &&pointer);
-	RowGroup(RowGroup &row_group, idx_t start);
+	RowGroup(RowGroupCollection &collection, idx_t start, idx_t count);
+	RowGroup(RowGroupCollection &collection, RowGroupPointer &&pointer);
+	RowGroup(RowGroup &row_group, RowGroupCollection &collection, idx_t start);
 	~RowGroup();
 
-	//! The index within the segment tree
-	idx_t index;
-
 private:
-	//! The database instance
-	AttachedDatabase &db;
-	//! The block manager
-	BlockManager &block_manager;
-	//! The table info of this row_group
-	DataTableInfo &table_info;
+	//! The RowGroupCollection this row-group is a part of
+	RowGroupCollection &collection;
 	//! The version info of the row_group (inserted and deleted tuple info)
 	shared_ptr<VersionNode> version_info;
 	//! The column data of the row_group
 	vector<shared_ptr<ColumnData>> columns;
-	//! The segment statistics for each of the columns
-	vector<SegmentStatistics> stats;
 
 public:
-	DatabaseInstance &GetDatabase();
-	BlockManager &GetBlockManager() {
-		return block_manager;
-	}
-	DataTableInfo &GetTableInfo() {
-		return table_info;
-	}
-	idx_t GetColumnIndex(ColumnData *data) {
-		for (idx_t i = 0; i < columns.size(); i++) {
-			if (columns[i].get() == data) {
-				return i;
-			}
-		}
-		return 0;
+	RowGroupCollection &GetCollection() {
+		return collection;
 	}
+	DatabaseInstance &GetDatabase();
+	BlockManager &GetBlockManager();
+	DataTableInfo &GetTableInfo();
 
-	unique_ptr<RowGroup> AlterType(const LogicalType &target_type, idx_t changed_idx, ExpressionExecutor &executor,
-	                               RowGroupScanState &scan_state, DataChunk &scan_chunk);
-	unique_ptr<RowGroup> AddColumn(ColumnDefinition &new_column, ExpressionExecutor &executor,
-	                               Expression *default_value, Vector &intermediate);
-	unique_ptr<RowGroup> RemoveColumn(idx_t removed_column);
+	unique_ptr<RowGroup> AlterType(RowGroupCollection &collection, const LogicalType &target_type, idx_t changed_idx,
+	                               ExpressionExecutor &executor, CollectionScanState &scan_state,
+	                               DataChunk &scan_chunk);
+	unique_ptr<RowGroup> AddColumn(RowGroupCollection &collection, ColumnDefinition &new_column,
+	                               ExpressionExecutor &executor, Expression *default_value, Vector &intermediate);
+	unique_ptr<RowGroup> RemoveColumn(RowGroupCollection &collection, idx_t removed_column);
 
 	void CommitDrop();
 	void CommitDropColumn(idx_t index);
@@ -104,16 +91,16 @@ public:
 	void InitializeEmpty(const vector<LogicalType> &types);
 
 	//! Initialize a scan over this row_group
-	bool InitializeScan(RowGroupScanState &state);
-	bool InitializeScanWithOffset(RowGroupScanState &state, idx_t vector_offset);
+	bool InitializeScan(CollectionScanState &state);
+	bool InitializeScanWithOffset(CollectionScanState &state, idx_t vector_offset);
 	//! Checks the given set of table filters against the row-group statistics. Returns false if the entire row group
 	//! can be skipped.
 	bool CheckZonemap(TableFilterSet &filters, const vector<column_t> &column_ids);
 	//! Checks the given set of table filters against the per-segment statistics. Returns false if any segments were
 	//! skipped.
-	bool CheckZonemapSegments(RowGroupScanState &state);
-	void Scan(TransactionData transaction, RowGroupScanState &state, DataChunk &result);
-	void ScanCommitted(RowGroupScanState &state, DataChunk &result, TableScanType type);
+	bool CheckZonemapSegments(CollectionScanState &state);
+	void Scan(TransactionData transaction, CollectionScanState &state, DataChunk &result);
+	void ScanCommitted(CollectionScanState &state, DataChunk &result, TableScanType type);
 
 	idx_t GetSelVector(TransactionData transaction, idx_t vector_idx, SelectionVector &sel_vector, idx_t max_count);
 	idx_t GetCommittedSelVector(transaction_t start_time, transaction_t transaction_id, idx_t vector_idx,
@@ -158,13 +145,16 @@ public:
 
 	void Verify();
 
-	void NextVector(RowGroupScanState &state);
+	void NextVector(CollectionScanState &state);
 
 private:
 	ChunkInfo *GetChunkInfo(idx_t vector_idx);
+	ColumnData &GetColumn(idx_t c);
+	idx_t GetColumnCount() const;
+	vector<shared_ptr<ColumnData>> &GetColumns();
 
 	template <TableScanType TYPE>
-	void TemplatedScan(TransactionData transaction, RowGroupScanState &state, DataChunk &result);
+	void TemplatedScan(TransactionData transaction, CollectionScanState &state, DataChunk &result);
 
 	static void CheckpointDeletes(VersionNode *versions, Serializer &serializer);
 	static shared_ptr<VersionNode> DeserializeDeletes(Deserializer &source);
@@ -172,6 +162,8 @@ private:
 private:
 	mutex row_group_lock;
 	mutex stats_lock;
+	vector<BlockPointer> column_pointers;
+	unique_ptr<atomic<bool>[]> is_loaded;
 };
 
 struct VersionNode {

package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp

@@ -15,14 +15,16 @@
 
 namespace duckdb {
 struct ParallelTableScanState;
-
+struct ParallelCollectionScanState;
+class CreateIndexScanState;
+class CollectionScanState;
 class PersistentTableData;
 class TableDataWriter;
 class TableIndexList;
 class TableStatistics;
-
+struct TableAppendState;
+class DuckTransaction;
 class BoundConstraint;
-
 class RowGroupSegmentTree;
 
 class RowGroupCollection {
@@ -48,8 +50,8 @@ public:
 	void InitializeCreateIndexScan(CreateIndexScanState &state);
 	void InitializeScanWithOffset(CollectionScanState &state, const vector<column_t> &column_ids, idx_t start_row,
 	                              idx_t end_row);
-	static bool InitializeScanInRowGroup(CollectionScanState &state, ParallelCollectionScanState &parallel_state,
-	                                     idx_t vector_index, idx_t max_row);
+	static bool InitializeScanInRowGroup(CollectionScanState &state, RowGroupCollection &collection,
+	                                     RowGroup &row_group, idx_t vector_index, idx_t max_row);
 	void InitializeParallelScan(ParallelCollectionScanState &state);
 	bool NextParallelScan(ClientContext &context, ParallelCollectionScanState &state, CollectionScanState &scan_state);
 
@@ -99,6 +101,15 @@ public:
 	unique_ptr<BaseStatistics> CopyStats(column_t column_id);
 	void SetDistinct(column_t column_id, unique_ptr<DistinctStatistics> distinct_stats);
 
+	AttachedDatabase &GetAttached();
+	DatabaseInstance &GetDatabase();
+	BlockManager &GetBlockManager() {
+		return block_manager;
+	}
+	DataTableInfo &GetTableInfo() {
+		return *info;
+	}
+
 private:
 	bool IsEmpty(SegmentLock &) const;
 
@@ -107,7 +118,9 @@ private:
 	BlockManager &block_manager;
 	//! The number of rows in the table
 	atomic<idx_t> total_rows;
+	//! The data table info
 	shared_ptr<DataTableInfo> info;
+	//! The column types of the row group collection
 	vector<LogicalType> types;
 	idx_t row_start;
 	//! The segment trees holding the various row_groups of the table

package/src/duckdb/src/include/duckdb/storage/table/row_group_segment_tree.hpp

@@ -18,7 +18,7 @@ class MetaBlockReader;
 
 class RowGroupSegmentTree : public SegmentTree<RowGroup, true> {
 public:
-	RowGroupSegmentTree(DataTableInfo &table_info_p, BlockManager &block_manager_p, vector<LogicalType> column_types_p);
+	RowGroupSegmentTree(RowGroupCollection &collection);
 	~RowGroupSegmentTree() override;
 
 	void Initialize(PersistentTableData &data);
@@ -26,9 +26,7 @@ public:
 protected:
 	unique_ptr<RowGroup> LoadSegment() override;
 
-	DataTableInfo &info;
-	BlockManager &block_manager;
-	vector<LogicalType> column_types;
+	RowGroupCollection &collection;
 	idx_t current_row_group;
 	idx_t max_row_group;
 	unique_ptr<MetaBlockReader> reader;

package/src/duckdb/src/include/duckdb/storage/table/scan_state.hpp

@@ -72,12 +72,11 @@ struct ColumnScanState {
 	idx_t last_offset = 0;
 
 public:
+	void Initialize(const LogicalType &type);
 	//! Move the scan state forward by "count" rows (including all child states)
 	void Next(idx_t count);
 	//! Move ONLY this state forward by "count" rows (i.e. not the child states)
 	void NextInternal(idx_t count);
-	//! Move the scan state forward by STANDARD_VECTOR_SIZE rows
-	void NextVector();
 };
 
 struct ColumnFetchState {
@@ -89,39 +88,18 @@ struct ColumnFetchState {
 	BufferHandle &GetOrInsertHandle(ColumnSegment &segment);
 };
 
-class RowGroupScanState {
+class CollectionScanState {
 public:
-	RowGroupScanState(CollectionScanState &parent_p)
-	    : row_group(nullptr), vector_index(0), max_row(0), parent(parent_p) {
-	}
+	CollectionScanState(TableScanState &parent_p);
 
 	//! The current row_group we are scanning
-	RowGroup *row_group = nullptr;
+	RowGroup *row_group;
 	//! The vector index within the row_group
-	idx_t vector_index = 0;
-	//! The maximum row index of this row_group scan
-	idx_t max_row = 0;
+	idx_t vector_index;
+	//! The maximum row within the row group
+	idx_t max_row_group_row;
 	//! Child column scans
 	unique_ptr<ColumnScanState[]> column_scans;
-
-public:
-	const vector<column_t> &GetColumnIds();
-	TableFilterSet *GetFilters();
-	AdaptiveFilter *GetAdaptiveFilter();
-	idx_t GetParentMaxRow();
-
-private:
-	//! The parent scan state
-	CollectionScanState &parent;
-};
-
-class CollectionScanState {
-public:
-	CollectionScanState(TableScanState &parent_p)
-	    : row_group_state(*this), row_groups(nullptr), max_row(0), batch_index(0), parent(parent_p) {};
-
-	//! The row_group scan state
-	RowGroupScanState row_group_state;
 	//! Row group segment tree
 	RowGroupSegmentTree *row_groups;
 	//! The total maximum row index
@@ -130,6 +108,7 @@ public:
 	idx_t batch_index;
 
 public:
+	void Initialize(const vector<LogicalType> &types);
 	const vector<column_t> &GetColumnIds();
 	TableFilterSet *GetFilters();
 	AdaptiveFilter *GetAdaptiveFilter();
@@ -167,12 +146,16 @@ private:
 };
 
 struct ParallelCollectionScanState {
+	ParallelCollectionScanState();
+
 	//! The row group collection we are scanning
 	RowGroupCollection *collection;
 	RowGroup *current_row_group;
 	idx_t vector_index;
 	idx_t max_row;
 	idx_t batch_index;
+	atomic<idx_t> processed_rows;
+	mutex lock;
 };
 
 struct ParallelTableScanState {