duckdb 0.7.2-dev0.0 → 0.7.2-dev1138.0

package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp

@@ -0,0 +1,74 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/statistics/string_stats.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+#include "duckdb/common/exception.hpp"
+#include "duckdb/common/types/hugeint.hpp"
+#include "duckdb/common/enums/filter_propagate_result.hpp"
+#include "duckdb/common/enums/expression_type.hpp"
+#include "duckdb/common/operator/comparison_operators.hpp"
+
+namespace duckdb {
+class BaseStatistics;
+class FieldWriter;
+class FieldReader;
+struct SelectionVector;
+class Vector;
+
+struct StringStatsData {
+	constexpr static uint32_t MAX_STRING_MINMAX_SIZE = 8;
+
+	//! The minimum value of the segment, potentially truncated
+	data_t min[MAX_STRING_MINMAX_SIZE];
+	//! The maximum value of the segment, potentially truncated
+	data_t max[MAX_STRING_MINMAX_SIZE];
+	//! Whether or not the column can contain unicode characters
+	bool has_unicode;
+	//! Whether or not the maximum string length is known
+	bool has_max_string_length;
+	//! The maximum string length in bytes
+	uint32_t max_string_length;
+};
+
+struct StringStats {
+	//! Unknown statistics - i.e. "has_unicode" is true, "max_string_length" is unknown, "min" is \0, max is \xFF
+	DUCKDB_API static BaseStatistics CreateUnknown(LogicalType type);
+	//! Empty statistics - i.e. "has_unicode" is false, "max_string_length" is 0, "min" is \xFF, max is \x00
+	DUCKDB_API static BaseStatistics CreateEmpty(LogicalType type);
+	//! Whether or not the statistics have a maximum string length defined
+	DUCKDB_API static bool HasMaxStringLength(const BaseStatistics &stats);
+	//! Returns the maximum string length, or throws an exception if !HasMaxStringLength()
+	DUCKDB_API static uint32_t MaxStringLength(const BaseStatistics &stats);
+	//! Whether or not the strings can contain unicode
+	DUCKDB_API static bool CanContainUnicode(const BaseStatistics &stats);
+
+	//! Resets the max string length so HasMaxStringLength() is false
+	DUCKDB_API static void ResetMaxStringLength(BaseStatistics &stats);
+	//! FIXME: make this part of Set on statistics
+	DUCKDB_API static void SetContainsUnicode(BaseStatistics &stats);
+
+	DUCKDB_API static void Serialize(const BaseStatistics &stats, FieldWriter &writer);
+	DUCKDB_API static BaseStatistics Deserialize(FieldReader &reader, LogicalType type);
+
+	DUCKDB_API static string ToString(const BaseStatistics &stats);
+
+	DUCKDB_API static FilterPropagateResult CheckZonemap(const BaseStatistics &stats, ExpressionType comparison_type,
+	                                                     const string &value);
+
+	DUCKDB_API static void Update(BaseStatistics &stats, const string_t &value);
+	DUCKDB_API static void Merge(BaseStatistics &stats, const BaseStatistics &other);
+	DUCKDB_API static void Verify(const BaseStatistics &stats, Vector &vector, const SelectionVector &sel, idx_t count);
+
+private:
+	static StringStatsData &GetDataUnsafe(BaseStatistics &stats);
+	static const StringStatsData &GetDataUnsafe(const BaseStatistics &stats);
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/statistics/struct_stats.hpp

@@ -0,0 +1,42 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/statistics/struct_stats.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+#include "duckdb/common/exception.hpp"
+
+namespace duckdb {
+class BaseStatistics;
+class FieldWriter;
+class FieldReader;
+struct SelectionVector;
+class Vector;
+
+struct StructStats {
+	DUCKDB_API static void Construct(BaseStatistics &stats);
+	DUCKDB_API static BaseStatistics CreateUnknown(LogicalType type);
+	DUCKDB_API static BaseStatistics CreateEmpty(LogicalType type);
+
+	DUCKDB_API static const BaseStatistics *GetChildStats(const BaseStatistics &stats);
+	DUCKDB_API static const BaseStatistics &GetChildStats(const BaseStatistics &stats, idx_t i);
+	DUCKDB_API static BaseStatistics &GetChildStats(BaseStatistics &stats, idx_t i);
+	DUCKDB_API static void SetChildStats(BaseStatistics &stats, idx_t i, const BaseStatistics &new_stats);
+	DUCKDB_API static void SetChildStats(BaseStatistics &stats, idx_t i, unique_ptr<BaseStatistics> new_stats);
+
+	DUCKDB_API static void Serialize(const BaseStatistics &stats, FieldWriter &writer);
+	DUCKDB_API static BaseStatistics Deserialize(FieldReader &reader, LogicalType type);
+
+	DUCKDB_API static string ToString(const BaseStatistics &stats);
+
+	DUCKDB_API static void Merge(BaseStatistics &stats, const BaseStatistics &other);
+	DUCKDB_API static void Copy(BaseStatistics &stats, const BaseStatistics &other);
+	DUCKDB_API static void Verify(const BaseStatistics &stats, Vector &vector, const SelectionVector &sel, idx_t count);
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/string_uncompressed.hpp

@@ -9,7 +9,7 @@
 #include "duckdb/storage/buffer_manager.hpp"
 #include "duckdb/storage/checkpoint/string_checkpoint_state.hpp"
 #include "duckdb/storage/segment/uncompressed.hpp"
-#include "duckdb/storage/statistics/string_statistics.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
 #include "duckdb/storage/string_uncompressed.hpp"
 #include "duckdb/storage/table/append_state.hpp"
 #include "duckdb/storage/table/column_segment.hpp"
@@ -173,8 +173,7 @@ public:
 
 public:
 	static inline void UpdateStringStats(SegmentStatistics &stats, const string_t &new_value) {
-		auto &sstats = (StringStatistics &)*stats.statistics;
-		sstats.Update(new_value);
+		StringStats::Update(stats.statistics, new_value);
 	}
 
 	static void SetDictionary(ColumnSegment &segment, BufferHandle &handle, StringDictionaryContainer dict);

package/src/duckdb/src/include/duckdb/storage/table/column_checkpoint_state.hpp

@@ -13,6 +13,7 @@
 #include "duckdb/storage/data_pointer.hpp"
 #include "duckdb/storage/statistics/segment_statistics.hpp"
 #include "duckdb/storage/table/column_segment.hpp"
+#include "duckdb/storage/table/column_data.hpp"
 #include "duckdb/common/unordered_set.hpp"
 
 namespace duckdb {
@@ -28,7 +29,7 @@ struct ColumnCheckpointState {
 
 	RowGroup &row_group;
 	ColumnData &column_data;
-	SegmentTree new_tree;
+	ColumnSegmentTree new_tree;
 	vector<DataPointer> data_pointers;
 	unique_ptr<BaseStatistics> global_stats;
 

package/src/duckdb/src/include/duckdb/storage/table/column_data.hpp

@@ -9,13 +9,12 @@
 #pragma once
 
 #include "duckdb/common/types/data_chunk.hpp"
-#include "duckdb/storage/table/append_state.hpp"
-#include "duckdb/storage/table/scan_state.hpp"
 #include "duckdb/storage/statistics/base_statistics.hpp"
 #include "duckdb/storage/data_pointer.hpp"
 #include "duckdb/storage/table/persistent_table_data.hpp"
 #include "duckdb/storage/statistics/segment_statistics.hpp"
-#include "duckdb/storage/table/column_checkpoint_state.hpp"
+#include "duckdb/storage/table/segment_tree.hpp"
+#include "duckdb/storage/table/column_segment_tree.hpp"
 #include "duckdb/common/mutex.hpp"
 
 namespace duckdb {
@@ -31,7 +30,7 @@ struct TransactionData;
 struct DataTableInfo;
 
 struct ColumnCheckpointInfo {
-	ColumnCheckpointInfo(CompressionType compression_type_p) : compression_type(compression_type_p) {};
+	explicit ColumnCheckpointInfo(CompressionType compression_type_p) : compression_type(compression_type_p) {};
 	CompressionType compression_type;
 };
 
@@ -44,14 +43,16 @@ public:
 	ColumnData(ColumnData &other, idx_t start, ColumnData *parent);
 	virtual ~ColumnData();
 
+	//! The start row
+	const idx_t start;
+	//! The count of the column data
+	idx_t count;
 	//! The block manager
 	BlockManager &block_manager;
 	//! Table info for the column
 	DataTableInfo &info;
 	//! The column index of the column, either within the parent table or within the parent
 	idx_t column_index;
-	//! The start row
-	idx_t start;
 	//! The type of the column
 	LogicalType type;
 	//! The parent column (if any)
@@ -60,6 +61,9 @@ public:
 public:
 	virtual bool CheckZonemap(ColumnScanState &state, TableFilter &filter) = 0;
 
+	BlockManager &GetBlockManager() {
+		return block_manager;
+	}
 	DatabaseInstance &GetDatabase() const;
 	DataTableInfo &GetTableInfo() const;
 	virtual idx_t GetMaxEntry();
@@ -93,6 +97,8 @@ public:
 	virtual void InitializeAppend(ColumnAppendState &state);
 	//! Append a vector of type [type] to the end of the column
 	virtual void Append(BaseStatistics &stats, ColumnAppendState &state, Vector &vector, idx_t count);
+	//! Append a vector of type [type] to the end of the column
+	void Append(ColumnAppendState &state, Vector &vector, idx_t count);
 	virtual void AppendData(BaseStatistics &stats, ColumnAppendState &state, UnifiedVectorFormat &vdata, idx_t count);
 	//! Revert a set of appends to the ColumnData
 	virtual void RevertAppend(row_t start_row);
@@ -127,6 +133,8 @@ public:
 	virtual void GetStorageInfo(idx_t row_group_index, vector<idx_t> col_path, TableStorageInfo &result);
 	virtual void Verify(RowGroup &parent);
 
+	bool CheckZonemap(TableFilter &filter);
+
 	static shared_ptr<ColumnData> CreateColumn(BlockManager &block_manager, DataTableInfo &info, idx_t column_index,
 	                                           idx_t start_row, const LogicalType &type, ColumnData *parent = nullptr);
 	static shared_ptr<ColumnData> CreateColumn(ColumnData &other, idx_t start_row, ColumnData *parent = nullptr);
@@ -135,6 +143,10 @@ public:
 	                                                 ColumnData *parent = nullptr);
 	static unique_ptr<ColumnData> CreateColumnUnique(ColumnData &other, idx_t start_row, ColumnData *parent = nullptr);
 
+	void MergeStatistics(const BaseStatistics &other);
+	void MergeIntoStatistics(BaseStatistics &other);
+	unique_ptr<BaseStatistics> GetStatistics();
+
 protected:
 	//! Append a transient segment
 	void AppendTransientSegment(SegmentLock &l, idx_t start_row);
@@ -148,13 +160,15 @@ protected:
 
 protected:
 	//! The segments holding the data of this column segment
-	SegmentTree data;
+	ColumnSegmentTree data;
 	//! The lock for the updates
 	mutex update_lock;
 	//! The updates for this column segment
 	unique_ptr<UpdateSegment> updates;
 	//! The internal version of the column data
 	idx_t version;
+	//! The stats of the root segment
+	unique_ptr<SegmentStatistics> stats;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/column_data_checkpointer.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/storage/table/column_data.hpp"
 #include "duckdb/function/compression_function.hpp"
+#include "duckdb/storage/table/column_checkpoint_state.hpp"
 
 namespace duckdb {
 
@@ -25,7 +26,7 @@ public:
 	RowGroup &GetRowGroup();
 	ColumnCheckpointState &GetCheckpointState();
 
-	void Checkpoint(vector<SegmentNode> nodes);
+	void Checkpoint(vector<SegmentNode<ColumnSegment>> nodes);
 
 private:
 	void ScanSegments(const std::function<void(Vector &, idx_t)> &callback);
@@ -40,7 +41,7 @@ private:
 	ColumnCheckpointState &state;
 	bool is_validity;
 	Vector intermediate;
-	vector<SegmentNode> nodes;
+	vector<SegmentNode<ColumnSegment>> nodes;
 	vector<CompressionFunction *> compression_functions;
 	ColumnCheckpointInfo &checkpoint_info;
 };

package/src/duckdb/src/include/duckdb/storage/table/column_segment.hpp

@@ -9,14 +9,13 @@
 #pragma once
 
 #include "duckdb/storage/block.hpp"
-#include "duckdb/storage/table/segment_tree.hpp"
 #include "duckdb/common/types.hpp"
 #include "duckdb/common/types/vector.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
 #include "duckdb/storage/statistics/segment_statistics.hpp"
 #include "duckdb/storage/storage_lock.hpp"
-#include "duckdb/storage/table/scan_state.hpp"
 #include "duckdb/function/compression_function.hpp"
+#include "duckdb/storage/table/segment_base.hpp"
 
 namespace duckdb {
 class ColumnSegment;
@@ -35,9 +34,9 @@ struct ColumnAppendState;
 enum class ColumnSegmentType : uint8_t { TRANSIENT, PERSISTENT };
 //! TableFilter represents a filter pushed down into the table scan.
 
-class ColumnSegment : public SegmentBase {
+class ColumnSegment : public SegmentBase<ColumnSegment> {
 public:
-	~ColumnSegment() override;
+	~ColumnSegment();
 
 	//! The database instance
 	DatabaseInstance &db;
@@ -57,7 +56,7 @@ public:
 	static unique_ptr<ColumnSegment> CreatePersistentSegment(DatabaseInstance &db, BlockManager &block_manager,
 	                                                         block_id_t id, idx_t offset, const LogicalType &type_p,
 	                                                         idx_t start, idx_t count, CompressionType compression_type,
-	                                                         unique_ptr<BaseStatistics> statistics);
+	                                                         BaseStatistics statistics);
 	static unique_ptr<ColumnSegment> CreateTransientSegment(DatabaseInstance &db, const LogicalType &type, idx_t start,
 	                                                        idx_t segment_size = Storage::BLOCK_SIZE);
 	static unique_ptr<ColumnSegment> CreateSegment(ColumnSegment &other, idx_t start);
@@ -124,7 +123,7 @@ public:
 
 public:
 	ColumnSegment(DatabaseInstance &db, shared_ptr<BlockHandle> block, LogicalType type, ColumnSegmentType segment_type,
-	              idx_t start, idx_t count, CompressionFunction *function, unique_ptr<BaseStatistics> statistics,
+	              idx_t start, idx_t count, CompressionFunction *function, BaseStatistics statistics,
 	              block_id_t block_id, idx_t offset, idx_t segment_size);
 	ColumnSegment(ColumnSegment &other, idx_t start);
 

package/src/duckdb/src/include/duckdb/storage/table/column_segment_tree.hpp

@@ -0,0 +1,18 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/table/column_segment_tree.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/storage/table/segment_tree.hpp"
+#include "duckdb/storage/table/column_segment.hpp"
+
+namespace duckdb {
+
+class ColumnSegmentTree : public SegmentTree<ColumnSegment> {};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/list_column_data.hpp

@@ -61,7 +61,7 @@ public:
 	void GetStorageInfo(idx_t row_group_index, vector<idx_t> col_path, TableStorageInfo &result) override;
 
 private:
-	list_entry_t FetchListEntry(idx_t row_idx);
+	uint64_t FetchListOffset(idx_t row_idx);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/persistent_table_data.hpp

@@ -10,8 +10,8 @@
 
 #include "duckdb/common/constants.hpp"
 #include "duckdb/common/vector.hpp"
-#include "duckdb/storage/table/segment_tree.hpp"
 #include "duckdb/storage/data_pointer.hpp"
+#include "duckdb/storage/table/table_statistics.hpp"
 
 namespace duckdb {
 class BaseStatistics;
@@ -21,8 +21,11 @@ public:
 	explicit PersistentTableData(idx_t column_count);
 	~PersistentTableData();
 
-	vector<RowGroupPointer> row_groups;
-	vector<unique_ptr<BaseStatistics>> column_stats;
+	TableStatistics table_stats;
+	idx_t total_rows;
+	idx_t row_group_count;
+	block_id_t block_id;
+	idx_t offset;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp

@@ -9,14 +9,14 @@
 #pragma once
 
 #include "duckdb/common/vector_size.hpp"
-#include "duckdb/storage/table/segment_base.hpp"
 #include "duckdb/storage/table/chunk_info.hpp"
-#include "duckdb/storage/table/append_state.hpp"
-#include "duckdb/storage/table/scan_state.hpp"
 #include "duckdb/storage/statistics/segment_statistics.hpp"
+#include "duckdb/common/types/data_chunk.hpp"
 #include "duckdb/common/enums/scan_options.hpp"
 #include "duckdb/common/mutex.hpp"
 #include "duckdb/parser/column_list.hpp"
+#include "duckdb/storage/table/segment_base.hpp"
+#include "duckdb/storage/block.hpp"
 
 namespace duckdb {
 class AttachedDatabase;
@@ -27,21 +27,27 @@ class DataTable;
 class PartialBlockManager;
 struct DataTableInfo;
 class ExpressionExecutor;
+class RowGroupCollection;
 class RowGroupWriter;
 class UpdateSegment;
+class TableStatistics;
 class TableStorageInfo;
 class Vector;
 struct ColumnCheckpointState;
 struct RowGroupPointer;
 struct TransactionData;
 struct VersionNode;
+class CollectionScanState;
+class TableFilterSet;
+struct ColumnFetchState;
+struct RowGroupAppendState;
 
 struct RowGroupWriteData {
 	vector<unique_ptr<ColumnCheckpointState>> states;
-	vector<unique_ptr<BaseStatistics>> statistics;
+	vector<BaseStatistics> statistics;
 };
 
-class RowGroup : public SegmentBase {
+class RowGroup : public SegmentBase<RowGroup> {
 public:
 	friend class ColumnData;
 	friend class VersionDeleteState;
@@ -51,48 +57,33 @@ public:
 	static constexpr const idx_t ROW_GROUP_VECTOR_COUNT = ROW_GROUP_SIZE / STANDARD_VECTOR_SIZE;
 
 public:
-	RowGroup(AttachedDatabase &db, BlockManager &block_manager, DataTableInfo &table_info, idx_t start, idx_t count);
-	RowGroup(AttachedDatabase &db, BlockManager &block_manager, DataTableInfo &table_info,
-	         const vector<LogicalType> &types, RowGroupPointer &&pointer);
-	RowGroup(RowGroup &row_group, idx_t start);
+	RowGroup(RowGroupCollection &collection, idx_t start, idx_t count);
+	RowGroup(RowGroupCollection &collection, RowGroupPointer &&pointer);
+	RowGroup(RowGroup &row_group, RowGroupCollection &collection, idx_t start);
 	~RowGroup();
 
 private:
-	//! The database instance
-	AttachedDatabase &db;
-	//! The block manager
-	BlockManager &block_manager;
-	//! The table info of this row_group
-	DataTableInfo &table_info;
+	//! The RowGroupCollection this row-group is a part of
+	RowGroupCollection &collection;
 	//! The version info of the row_group (inserted and deleted tuple info)
 	shared_ptr<VersionNode> version_info;
 	//! The column data of the row_group
 	vector<shared_ptr<ColumnData>> columns;
-	//! The segment statistics for each of the columns
-	vector<shared_ptr<SegmentStatistics>> stats;
 
 public:
-	DatabaseInstance &GetDatabase();
-	BlockManager &GetBlockManager() {
-		return block_manager;
-	}
-	DataTableInfo &GetTableInfo() {
-		return table_info;
-	}
-	idx_t GetColumnIndex(ColumnData *data) {
-		for (idx_t i = 0; i < columns.size(); i++) {
-			if (columns[i].get() == data) {
-				return i;
-			}
-		}
-		return 0;
+	RowGroupCollection &GetCollection() {
+		return collection;
 	}
+	DatabaseInstance &GetDatabase();
+	BlockManager &GetBlockManager();
+	DataTableInfo &GetTableInfo();
 
-	unique_ptr<RowGroup> AlterType(const LogicalType &target_type, idx_t changed_idx, ExpressionExecutor &executor,
-	                               RowGroupScanState &scan_state, DataChunk &scan_chunk);
-	unique_ptr<RowGroup> AddColumn(ColumnDefinition &new_column, ExpressionExecutor &executor,
-	                               Expression *default_value, Vector &intermediate);
-	unique_ptr<RowGroup> RemoveColumn(idx_t removed_column);
+	unique_ptr<RowGroup> AlterType(RowGroupCollection &collection, const LogicalType &target_type, idx_t changed_idx,
+	                               ExpressionExecutor &executor, CollectionScanState &scan_state,
+	                               DataChunk &scan_chunk);
+	unique_ptr<RowGroup> AddColumn(RowGroupCollection &collection, ColumnDefinition &new_column,
+	                               ExpressionExecutor &executor, Expression *default_value, Vector &intermediate);
+	unique_ptr<RowGroup> RemoveColumn(RowGroupCollection &collection, idx_t removed_column);
 
 	void CommitDrop();
 	void CommitDropColumn(idx_t index);
@@ -100,16 +91,16 @@ public:
 	void InitializeEmpty(const vector<LogicalType> &types);
 
 	//! Initialize a scan over this row_group
-	bool InitializeScan(RowGroupScanState &state);
-	bool InitializeScanWithOffset(RowGroupScanState &state, idx_t vector_offset);
+	bool InitializeScan(CollectionScanState &state);
+	bool InitializeScanWithOffset(CollectionScanState &state, idx_t vector_offset);
 	//! Checks the given set of table filters against the row-group statistics. Returns false if the entire row group
 	//! can be skipped.
 	bool CheckZonemap(TableFilterSet &filters, const vector<column_t> &column_ids);
 	//! Checks the given set of table filters against the per-segment statistics. Returns false if any segments were
 	//! skipped.
-	bool CheckZonemapSegments(RowGroupScanState &state);
-	void Scan(TransactionData transaction, RowGroupScanState &state, DataChunk &result);
-	void ScanCommitted(RowGroupScanState &state, DataChunk &result, TableScanType type);
+	bool CheckZonemapSegments(CollectionScanState &state);
+	void Scan(TransactionData transaction, CollectionScanState &state, DataChunk &result);
+	void ScanCommitted(CollectionScanState &state, DataChunk &result, TableScanType type);
 
 	idx_t GetSelVector(TransactionData transaction, idx_t vector_idx, SelectionVector &sel_vector, idx_t max_count);
 	idx_t GetCommittedSelVector(transaction_t start_time, transaction_t transaction_id, idx_t vector_idx,
@@ -132,9 +123,9 @@ public:
 	idx_t Delete(TransactionData transaction, DataTable *table, row_t *row_ids, idx_t count);
 
 	RowGroupWriteData WriteToDisk(PartialBlockManager &manager, const vector<CompressionType> &compression_types);
-	RowGroupPointer Checkpoint(RowGroupWriter &writer, vector<unique_ptr<BaseStatistics>> &global_stats);
+	RowGroupPointer Checkpoint(RowGroupWriter &writer, TableStatistics &global_stats);
 	static void Serialize(RowGroupPointer &pointer, Serializer &serializer);
-	static RowGroupPointer Deserialize(Deserializer &source, const ColumnList &columns);
+	static RowGroupPointer Deserialize(Deserializer &source, const vector<LogicalType> &columns);
 
 	void InitializeAppend(RowGroupAppendState &append_state);
 	void Append(RowGroupAppendState &append_state, DataChunk &chunk, idx_t append_count);
@@ -154,13 +145,16 @@ public:
 
 	void Verify();
 
-	void NextVector(RowGroupScanState &state);
+	void NextVector(CollectionScanState &state);
 
 private:
 	ChunkInfo *GetChunkInfo(idx_t vector_idx);
+	ColumnData &GetColumn(idx_t c);
+	idx_t GetColumnCount() const;
+	vector<shared_ptr<ColumnData>> &GetColumns();
 
 	template <TableScanType TYPE>
-	void TemplatedScan(TransactionData transaction, RowGroupScanState &state, DataChunk &result);
+	void TemplatedScan(TransactionData transaction, CollectionScanState &state, DataChunk &result);
 
 	static void CheckpointDeletes(VersionNode *versions, Serializer &serializer);
 	static shared_ptr<VersionNode> DeserializeDeletes(Deserializer &source);
@@ -168,6 +162,8 @@ private:
 private:
 	mutex row_group_lock;
 	mutex stats_lock;
+	vector<BlockPointer> column_pointers;
+	unique_ptr<atomic<bool>[]> is_loaded;
 };
 
 struct VersionNode {

package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp

@@ -15,13 +15,17 @@
 
 namespace duckdb {
 struct ParallelTableScanState;
-
+struct ParallelCollectionScanState;
+class CreateIndexScanState;
+class CollectionScanState;
 class PersistentTableData;
 class TableDataWriter;
 class TableIndexList;
 class TableStatistics;
-
+struct TableAppendState;
+class DuckTransaction;
 class BoundConstraint;
+class RowGroupSegmentTree;
 
 class RowGroupCollection {
 public:
@@ -46,8 +50,8 @@ public:
 	void InitializeCreateIndexScan(CreateIndexScanState &state);
 	void InitializeScanWithOffset(CollectionScanState &state, const vector<column_t> &column_ids, idx_t start_row,
 	                              idx_t end_row);
-	static bool InitializeScanInRowGroup(CollectionScanState &state, RowGroup *row_group, idx_t vector_index,
-	                                     idx_t max_row);
+	static bool InitializeScanInRowGroup(CollectionScanState &state, RowGroupCollection &collection,
+	                                     RowGroup &row_group, idx_t vector_index, idx_t max_row);
 	void InitializeParallelScan(ParallelCollectionScanState &state);
 	bool NextParallelScan(ClientContext &context, ParallelCollectionScanState &state, CollectionScanState &scan_state);
 
@@ -78,7 +82,7 @@ public:
 	void UpdateColumn(TransactionData transaction, Vector &row_ids, const vector<column_t> &column_path,
 	                  DataChunk &updates);
 
-	void Checkpoint(TableDataWriter &writer, vector<unique_ptr<BaseStatistics>> &global_stats);
+	void Checkpoint(TableDataWriter &writer, TableStatistics &global_stats);
 
 	void CommitDropColumn(idx_t index);
 	void CommitDropTable();
@@ -93,8 +97,18 @@ public:
 	                                         vector<column_t> bound_columns, Expression &cast_expr);
 	void VerifyNewConstraint(DataTable &parent, const BoundConstraint &constraint);
 
+	void CopyStats(TableStatistics &stats);
 	unique_ptr<BaseStatistics> CopyStats(column_t column_id);
-	void SetStatistics(column_t column_id, const std::function<void(BaseStatistics &)> &set_fun);
+	void SetDistinct(column_t column_id, unique_ptr<DistinctStatistics> distinct_stats);
+
+	AttachedDatabase &GetAttached();
+	DatabaseInstance &GetDatabase();
+	BlockManager &GetBlockManager() {
+		return block_manager;
+	}
+	DataTableInfo &GetTableInfo() {
+		return *info;
+	}
 
 private:
 	bool IsEmpty(SegmentLock &) const;
@@ -104,11 +118,13 @@ private:
 	BlockManager &block_manager;
 	//! The number of rows in the table
 	atomic<idx_t> total_rows;
+	//! The data table info
 	shared_ptr<DataTableInfo> info;
+	//! The column types of the row group collection
 	vector<LogicalType> types;
 	idx_t row_start;
 	//! The segment trees holding the various row_groups of the table
-	shared_ptr<SegmentTree> row_groups;
+	shared_ptr<RowGroupSegmentTree> row_groups;
 	//! Table statistics
 	TableStatistics stats;
 };

package/src/duckdb/src/include/duckdb/storage/table/row_group_segment_tree.hpp

@@ -0,0 +1,35 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/table/row_group_segment_tree.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/storage/table/segment_tree.hpp"
+#include "duckdb/storage/table/row_group.hpp"
+
+namespace duckdb {
+struct DataTableInfo;
+class PersistentTableData;
+class MetaBlockReader;
+
+class RowGroupSegmentTree : public SegmentTree<RowGroup, true> {
+public:
+	RowGroupSegmentTree(RowGroupCollection &collection);
+	~RowGroupSegmentTree() override;
+
+	void Initialize(PersistentTableData &data);
+
+protected:
+	unique_ptr<RowGroup> LoadSegment() override;
+
+	RowGroupCollection &collection;
+	idx_t current_row_group;
+	idx_t max_row_group;
+	unique_ptr<MetaBlockReader> reader;
+};
+
+} // namespace duckdb