duckdb 1.1.4-dev2.0 → 1.1.4-dev9.0

package/src/duckdb/src/storage/table/row_group_collection.cpp

@@ -1,5 +1,4 @@
 #include "duckdb/storage/table/row_group_collection.hpp"
-
 #include "duckdb/common/serializer/binary_deserializer.hpp"
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/execution/index/bound_index.hpp"
@@ -53,10 +52,17 @@ unique_ptr<RowGroup> RowGroupSegmentTree::LoadSegment() {
 //===--------------------------------------------------------------------===//
 // Row Group Collection
 //===--------------------------------------------------------------------===//
+RowGroupCollection::RowGroupCollection(shared_ptr<DataTableInfo> info_p, TableIOManager &io_manager,
+                                       vector<LogicalType> types_p, idx_t row_start, idx_t total_rows)
+    : RowGroupCollection(std::move(info_p), io_manager.GetBlockManagerForRowData(), std::move(types_p), row_start,
+                         total_rows, io_manager.GetRowGroupSize()) {
+}
+
 RowGroupCollection::RowGroupCollection(shared_ptr<DataTableInfo> info_p, BlockManager &block_manager,
-                                       vector<LogicalType> types_p, idx_t row_start_p, idx_t total_rows_p)
-    : block_manager(block_manager), total_rows(total_rows_p), info(std::move(info_p)), types(std::move(types_p)),
-      row_start(row_start_p), allocation_size(0) {
+                                       vector<LogicalType> types_p, idx_t row_start_p, idx_t total_rows_p,
+                                       idx_t row_group_size_p)
+    : block_manager(block_manager), row_group_size(row_group_size_p), total_rows(total_rows_p), info(std::move(info_p)),
+      types(std::move(types_p)), row_start(row_start_p), allocation_size(0) {
 	row_groups = make_shared_ptr<RowGroupSegmentTree>(*this);
 }
 
@@ -134,7 +140,7 @@ void RowGroupCollection::Verify() {
 //===--------------------------------------------------------------------===//
 // Scan
 //===--------------------------------------------------------------------===//
-void RowGroupCollection::InitializeScan(CollectionScanState &state, const vector<column_t> &column_ids,
+void RowGroupCollection::InitializeScan(CollectionScanState &state, const vector<StorageIndex> &column_ids,
                                         TableFilterSet *table_filters) {
 	auto row_group = row_groups->GetRootSegment();
 	D_ASSERT(row_group);
@@ -150,7 +156,7 @@ void RowGroupCollection::InitializeCreateIndexScan(CreateIndexScanState &state)
 	state.segment_lock = row_groups->Lock();
 }
 
-void RowGroupCollection::InitializeScanWithOffset(CollectionScanState &state, const vector<column_t> &column_ids,
+void RowGroupCollection::InitializeScanWithOffset(CollectionScanState &state, const vector<StorageIndex> &column_ids,
                                                   idx_t start_row, idx_t end_row) {
 	auto row_group = row_groups->GetSegment(start_row);
 	D_ASSERT(row_group);
@@ -235,11 +241,11 @@ bool RowGroupCollection::NextParallelScan(ClientContext &context, ParallelCollec
 	return false;
 }
 
-bool RowGroupCollection::Scan(DuckTransaction &transaction, const vector<column_t> &column_ids,
+bool RowGroupCollection::Scan(DuckTransaction &transaction, const vector<StorageIndex> &column_ids,
                               const std::function<bool(DataChunk &chunk)> &fun) {
 	vector<LogicalType> scan_types;
 	for (idx_t i = 0; i < column_ids.size(); i++) {
-		scan_types.push_back(types[column_ids[i]]);
+		scan_types.push_back(types[column_ids[i].GetPrimaryIndex()]);
 	}
 	DataChunk chunk;
 	chunk.Initialize(GetAllocator(), scan_types);
@@ -262,10 +268,10 @@ bool RowGroupCollection::Scan(DuckTransaction &transaction, const vector<column_
 }
 
 bool RowGroupCollection::Scan(DuckTransaction &transaction, const std::function<bool(DataChunk &chunk)> &fun) {
-	vector<column_t> column_ids;
+	vector<StorageIndex> column_ids;
 	column_ids.reserve(types.size());
 	for (idx_t i = 0; i < types.size(); i++) {
-		column_ids.push_back(i);
+		column_ids.emplace_back(i);
 	}
 	return Scan(transaction, column_ids, fun);
 }
@@ -273,7 +279,7 @@ bool RowGroupCollection::Scan(DuckTransaction &transaction, const std::function<
 //===--------------------------------------------------------------------===//
 // Fetch
 //===--------------------------------------------------------------------===//
-void RowGroupCollection::Fetch(TransactionData transaction, DataChunk &result, const vector<column_t> &column_ids,
+void RowGroupCollection::Fetch(TransactionData transaction, DataChunk &result, const vector<StorageIndex> &column_ids,
                                const Vector &row_identifiers, idx_t fetch_count, ColumnFetchState &state) {
 	// figure out which row_group to fetch from
 	auto row_ids = FlatVector::GetData<row_t>(row_identifiers);
@@ -303,7 +309,8 @@ void RowGroupCollection::Fetch(TransactionData transaction, DataChunk &result, c
 // Append
 //===--------------------------------------------------------------------===//
 TableAppendState::TableAppendState()
-    : row_group_append_state(*this), total_append_count(0), start_row_group(nullptr), transaction(0, 0) {
+    : row_group_append_state(*this), total_append_count(0), start_row_group(nullptr), transaction(0, 0),
+      hashes(LogicalType::HASH) {
 }
 
 TableAppendState::~TableAppendState() {
@@ -345,6 +352,7 @@ void RowGroupCollection::InitializeAppend(TableAppendState &state) {
 }
 
 bool RowGroupCollection::Append(DataChunk &chunk, TableAppendState &state) {
+	const idx_t row_group_size = GetRowGroupSize();
 	D_ASSERT(chunk.ColumnCount() == types.size());
 	chunk.Verify();
 
@@ -356,7 +364,7 @@ bool RowGroupCollection::Append(DataChunk &chunk, TableAppendState &state) {
 		auto current_row_group = state.row_group_append_state.row_group;
 		// check how much we can fit into the current row_group
 		idx_t append_count =
-		    MinValue<idx_t>(remaining, Storage::ROW_GROUP_SIZE - state.row_group_append_state.offset_in_row_group);
+		    MinValue<idx_t>(remaining, row_group_size - state.row_group_append_state.offset_in_row_group);
 		if (append_count > 0) {
 			auto previous_allocation_size = current_row_group->GetAllocationSize();
 			current_row_group->Append(state.row_group_append_state, chunk, append_count);
@@ -388,18 +396,30 @@ bool RowGroupCollection::Append(DataChunk &chunk, TableAppendState &state) {
 		}
 	}
 	state.current_row += row_t(total_append_count);
+
 	auto local_stats_lock = state.stats.GetLock();
+
 	for (idx_t col_idx = 0; col_idx < types.size(); col_idx++) {
-		state.stats.GetStats(*local_stats_lock, col_idx).UpdateDistinctStatistics(chunk.data[col_idx], chunk.size());
+		auto &column_stats = state.stats.GetStats(*local_stats_lock, col_idx);
+		column_stats.UpdateDistinctStatistics(chunk.data[col_idx], chunk.size(), state.hashes);
+	}
+
+	auto &table_sample = state.stats.GetTableSampleRef(*local_stats_lock);
+	if (!table_sample.destroyed) {
+		D_ASSERT(table_sample.type == SampleType::RESERVOIR_SAMPLE);
+		table_sample.AddToReservoir(chunk);
 	}
+
 	return new_row_group;
 }
 
 void RowGroupCollection::FinalizeAppend(TransactionData transaction, TableAppendState &state) {
+	const idx_t row_group_size = GetRowGroupSize();
+
 	auto remaining = state.total_append_count;
 	auto row_group = state.start_row_group;
 	while (remaining > 0) {
-		auto append_count = MinValue<idx_t>(remaining, Storage::ROW_GROUP_SIZE - row_group->count);
+		auto append_count = MinValue<idx_t>(remaining, row_group_size - row_group->count);
 		row_group->AppendVersionInfo(transaction, append_count);
 		remaining -= append_count;
 		row_group = row_groups->GetNextSegment(row_group);
@@ -409,8 +429,8 @@ void RowGroupCollection::FinalizeAppend(TransactionData transaction, TableAppend
 	state.total_append_count = 0;
 	state.start_row_group = nullptr;
 
-	auto global_stats_lock = stats.GetLock();
 	auto local_stats_lock = state.stats.GetLock();
+	auto global_stats_lock = stats.GetLock();
 	for (idx_t col_idx = 0; col_idx < types.size(); col_idx++) {
 		auto &global_stats = stats.GetStats(*global_stats_lock, col_idx);
 		if (!global_stats.HasDistinctStats()) {
@@ -423,6 +443,22 @@ void RowGroupCollection::FinalizeAppend(TransactionData transaction, TableAppend
 		global_stats.DistinctStats().Merge(local_stats.DistinctStats());
 	}
 
+	auto local_sample = state.stats.GetTableSample(*local_stats_lock);
+	auto global_sample = stats.GetTableSample(*global_stats_lock);
+
+	if (local_sample && global_sample) {
+		D_ASSERT(global_sample->type == SampleType::RESERVOIR_SAMPLE);
+		auto &reservoir_sample = global_sample->Cast<ReservoirSample>();
+		reservoir_sample.Merge(std::move(local_sample));
+		// initialize the thread local sample again
+		auto new_local_sample = make_uniq<ReservoirSample>(reservoir_sample.GetSampleCount());
+		state.stats.SetTableSample(*local_stats_lock, std::move(new_local_sample));
+		stats.SetTableSample(*global_stats_lock, std::move(global_sample));
+	} else {
+		state.stats.SetTableSample(*local_stats_lock, std::move(local_sample));
+		stats.SetTableSample(*global_stats_lock, std::move(global_sample));
+	}
+
 	Verify();
 }
 
@@ -570,6 +606,11 @@ idx_t RowGroupCollection::Delete(TransactionData transaction, DataTable &table,
 		}
 		delete_count += row_group->Delete(transaction, table, ids + start, pos - start);
 	} while (pos < count);
+
+	// When deleting destroy the sample.
+	auto stats_guard = stats.GetLock();
+	stats.DestroyTableSample(*stats_guard);
+
 	return delete_count;
 }
 
@@ -607,6 +648,9 @@ void RowGroupCollection::Update(TransactionData transaction, row_t *ids, const v
 			stats.MergeStats(*l, column_id.index, *row_group->GetStatistics(column_id.index));
 		}
 	} while (pos < updates.size());
+	// on update destroy the sample
+	auto stats_guard = stats.GetLock();
+	stats.DestroyTableSample(*stats_guard);
 }
 
 void RowGroupCollection::RemoveFromIndexes(TableIndexList &indexes, Vector &row_identifiers, idx_t count) {
@@ -615,10 +659,10 @@ void RowGroupCollection::RemoveFromIndexes(TableIndexList &indexes, Vector &row_
 	// initialize the fetch state
 	// FIXME: we do not need to fetch all columns, only the columns required by the indices!
 	TableScanState state;
-	vector<column_t> column_ids;
+	vector<StorageIndex> column_ids;
 	column_ids.reserve(types.size());
 	for (idx_t i = 0; i < types.size(); i++) {
-		column_ids.push_back(i);
+		column_ids.emplace_back(i);
 	}
 	state.Initialize(std::move(column_ids));
 	state.table_state.max_row = row_start + total_rows;
@@ -758,6 +802,7 @@ public:
 
 	void ExecuteTask() override {
 		auto &collection = checkpoint_state.collection;
+		const idx_t row_group_size = collection.GetRowGroupSize();
 		auto &types = collection.GetTypes();
 		// create the new set of target row groups (initially empty)
 		vector<unique_ptr<RowGroup>> new_row_groups;
@@ -765,7 +810,7 @@ public:
 		idx_t row_group_rows = merge_rows;
 		idx_t start = row_start;
 		for (idx_t target_idx = 0; target_idx < target_count; target_idx++) {
-			idx_t current_row_group_rows = MinValue<idx_t>(row_group_rows, Storage::ROW_GROUP_SIZE);
+			idx_t current_row_group_rows = MinValue<idx_t>(row_group_rows, row_group_size);
 			auto new_row_group = make_uniq<RowGroup>(collection, start, current_row_group_rows);
 			new_row_group->InitializeEmpty(types);
 			new_row_groups.push_back(std::move(new_row_group));
@@ -778,9 +823,9 @@ public:
 		DataChunk scan_chunk;
 		scan_chunk.Initialize(Allocator::DefaultAllocator(), types);
 
-		vector<column_t> column_ids;
+		vector<StorageIndex> column_ids;
 		for (idx_t c = 0; c < types.size(); c++) {
-			column_ids.push_back(c);
+			column_ids.emplace_back(c);
 		}
 
 		idx_t current_append_idx = 0;
@@ -815,13 +860,12 @@ public:
 				scan_chunk.Flatten();
 				idx_t remaining = scan_chunk.size();
 				while (remaining > 0) {
-					idx_t append_count =
-					    MinValue<idx_t>(remaining, Storage::ROW_GROUP_SIZE - append_counts[current_append_idx]);
+					idx_t append_count = MinValue<idx_t>(remaining, row_group_size - append_counts[current_append_idx]);
 					new_row_groups[current_append_idx]->Append(append_state.row_group_append_state, scan_chunk,
 					                                           append_count);
 					append_counts[current_append_idx] += append_count;
 					remaining -= append_count;
-					const bool row_group_full = append_counts[current_append_idx] == Storage::ROW_GROUP_SIZE;
+					const bool row_group_full = append_counts[current_append_idx] == row_group_size;
 					const bool last_row_group = current_append_idx + 1 >= new_row_groups.size();
 					if (remaining > 0 || (row_group_full && !last_row_group)) {
 						// move to the next row group
@@ -917,8 +961,9 @@ bool RowGroupCollection::ScheduleVacuumTasks(CollectionCheckpointState &checkpoi
 	// hence we target_count should be less than merge_count for a marge to be worth it
 	// we greedily prefer to merge to the lowest target_count
 	// i.e. we prefer to merge 2 row groups into 1, than 3 row groups into 2
+	const idx_t row_group_size = GetRowGroupSize();
 	for (target_count = 1; target_count <= MAX_MERGE_COUNT; target_count++) {
-		auto total_target_size = target_count * Storage::ROW_GROUP_SIZE;
+		auto total_target_size = target_count * row_group_size;
 		merge_count = 0;
 		merge_rows = 0;
 		for (next_idx = segment_idx; next_idx < checkpoint_state.segments.size(); next_idx++) {
@@ -969,29 +1014,38 @@ void RowGroupCollection::Checkpoint(TableDataWriter &writer, TableStatistics &gl
 
 	VacuumState vacuum_state;
 	InitializeVacuumState(checkpoint_state, vacuum_state, segments);
-	// schedule tasks
-	idx_t total_vacuum_tasks = 0;
-	auto &config = DBConfig::GetConfig(writer.GetDatabase());
-	for (idx_t segment_idx = 0; segment_idx < segments.size(); segment_idx++) {
-		auto &entry = segments[segment_idx];
-		auto vacuum_tasks = ScheduleVacuumTasks(checkpoint_state, vacuum_state, segment_idx,
-		                                        total_vacuum_tasks < config.options.max_vacuum_tasks);
-		if (vacuum_tasks) {
-			// vacuum tasks were scheduled - don't schedule a checkpoint task yet
-			total_vacuum_tasks++;
-			continue;
-		}
-		if (!entry.node) {
-			// row group was vacuumed/dropped - skip
-			continue;
+
+	try {
+		// schedule tasks
+		idx_t total_vacuum_tasks = 0;
+		auto &config = DBConfig::GetConfig(writer.GetDatabase());
+
+		for (idx_t segment_idx = 0; segment_idx < segments.size(); segment_idx++) {
+			auto &entry = segments[segment_idx];
+			auto vacuum_tasks = ScheduleVacuumTasks(checkpoint_state, vacuum_state, segment_idx,
+			                                        total_vacuum_tasks < config.options.max_vacuum_tasks);
+			if (vacuum_tasks) {
+				// vacuum tasks were scheduled - don't schedule a checkpoint task yet
+				total_vacuum_tasks++;
+				continue;
+			}
+			if (!entry.node) {
+				// row group was vacuumed/dropped - skip
+				continue;
+			}
+			// schedule a checkpoint task for this row group
+			entry.node->MoveToCollection(*this, vacuum_state.row_start);
+			auto checkpoint_task = GetCheckpointTask(checkpoint_state, segment_idx);
+			checkpoint_state.executor.ScheduleTask(std::move(checkpoint_task));
+			vacuum_state.row_start += entry.node->count;
 		}
-		// schedule a checkpoint task for this row group
-		entry.node->MoveToCollection(*this, vacuum_state.row_start);
-		auto checkpoint_task = GetCheckpointTask(checkpoint_state, segment_idx);
-		checkpoint_state.executor.ScheduleTask(std::move(checkpoint_task));
-		vacuum_state.row_start += entry.node->count;
+	} catch (const std::exception &e) {
+		ErrorData error(e);
+		checkpoint_state.executor.PushError(std::move(error));
+		checkpoint_state.executor.WorkOnTasks(); // ensure all tasks have completed first before rethrowing
+		throw;
 	}
-	// all tasks have been scheduled - execute tasks until we are done
+	// all tasks have been successfully scheduled - execute tasks until we are done
 	checkpoint_state.executor.WorkOnTasks();
 
 	// no errors - finalize the row groups
@@ -1031,6 +1085,17 @@ void RowGroupCollection::CommitDropTable() {
 	}
 }
 
+//===--------------------------------------------------------------------===//
+// GetPartitionStats
+//===--------------------------------------------------------------------===//
+vector<PartitionStatistics> RowGroupCollection::GetPartitionStats() const {
+	vector<PartitionStatistics> result;
+	for (auto &row_group : row_groups->Segments()) {
+		result.push_back(row_group.GetPartitionStats());
+	}
+	return result;
+}
+
 //===--------------------------------------------------------------------===//
 // GetColumnSegmentInfo
 //===--------------------------------------------------------------------===//
@@ -1050,8 +1115,8 @@ shared_ptr<RowGroupCollection> RowGroupCollection::AddColumn(ClientContext &cont
 	idx_t new_column_idx = types.size();
 	auto new_types = types;
 	new_types.push_back(new_column.GetType());
-	auto result =
-	    make_shared_ptr<RowGroupCollection>(info, block_manager, std::move(new_types), row_start, total_rows.load());
+	auto result = make_shared_ptr<RowGroupCollection>(info, block_manager, std::move(new_types), row_start,
+	                                                  total_rows.load(), row_group_size);
 
 	DataChunk dummy_chunk;
 	Vector default_vector(new_column.GetType());
@@ -1069,6 +1134,9 @@ shared_ptr<RowGroupCollection> RowGroupCollection::AddColumn(ClientContext &cont
 
 		result->row_groups->AppendSegment(std::move(new_row_group));
 	}
+	// When adding a column destroy the sample
+	stats.DestroyTableSample(*lock);
+
 	return result;
 }
 
@@ -1077,10 +1145,13 @@ shared_ptr<RowGroupCollection> RowGroupCollection::RemoveColumn(idx_t col_idx) {
 	auto new_types = types;
 	new_types.erase_at(col_idx);
 
-	auto result =
-	    make_shared_ptr<RowGroupCollection>(info, block_manager, std::move(new_types), row_start, total_rows.load());
+	auto result = make_shared_ptr<RowGroupCollection>(info, block_manager, std::move(new_types), row_start,
+	                                                  total_rows.load(), row_group_size);
 	result->stats.InitializeRemoveColumn(stats, col_idx);
 
+	auto result_lock = result->stats.GetLock();
+	result->stats.DestroyTableSample(*result_lock);
+
 	for (auto &current_row_group : row_groups->Segments()) {
 		auto new_row_group = current_row_group.RemoveColumn(*result, col_idx);
 		result->row_groups->AppendSegment(std::move(new_row_group));
@@ -1090,21 +1161,22 @@ shared_ptr<RowGroupCollection> RowGroupCollection::RemoveColumn(idx_t col_idx) {
 
 shared_ptr<RowGroupCollection> RowGroupCollection::AlterType(ClientContext &context, idx_t changed_idx,
                                                              const LogicalType &target_type,
-                                                             vector<column_t> bound_columns, Expression &cast_expr) {
+                                                             vector<StorageIndex> bound_columns,
+                                                             Expression &cast_expr) {
 	D_ASSERT(changed_idx < types.size());
 	auto new_types = types;
 	new_types[changed_idx] = target_type;
 
-	auto result =
-	    make_shared_ptr<RowGroupCollection>(info, block_manager, std::move(new_types), row_start, total_rows.load());
+	auto result = make_shared_ptr<RowGroupCollection>(info, block_manager, std::move(new_types), row_start,
+	                                                  total_rows.load(), row_group_size);
 	result->stats.InitializeAlterType(stats, changed_idx, target_type);
 
 	vector<LogicalType> scan_types;
 	for (idx_t i = 0; i < bound_columns.size(); i++) {
-		if (bound_columns[i] == COLUMN_IDENTIFIER_ROW_ID) {
+		if (bound_columns[i].IsRowIdColumn()) {
 			scan_types.emplace_back(LogicalType::ROW_TYPE);
 		} else {
-			scan_types.push_back(types[bound_columns[i]]);
+			scan_types.push_back(types[bound_columns[i].GetPrimaryIndex()]);
 		}
 	}
 	DataChunk scan_chunk;
@@ -1126,7 +1198,6 @@ shared_ptr<RowGroupCollection> RowGroupCollection::AlterType(ClientContext &cont
 		new_row_group->MergeIntoStatistics(changed_idx, changed_stats.Statistics());
 		result->row_groups->AppendSegment(std::move(new_row_group));
 	}
-
 	return result;
 }
 
@@ -1134,40 +1205,46 @@ void RowGroupCollection::VerifyNewConstraint(DataTable &parent, const BoundConst
 	if (total_rows == 0) {
 		return;
 	}
-	// scan the original table, check if there's any null value
+
+	// Scan the original table for NULL values.
 	auto &not_null_constraint = constraint.Cast<BoundNotNullConstraint>();
 	vector<LogicalType> scan_types;
 	auto physical_index = not_null_constraint.index.index;
 	D_ASSERT(physical_index < types.size());
+
 	scan_types.push_back(types[physical_index]);
 	DataChunk scan_chunk;
 	scan_chunk.Initialize(GetAllocator(), scan_types);
 
+	vector<StorageIndex> column_ids;
+	column_ids.emplace_back(physical_index);
+
+	// Use SCAN_COMMITTED to scan the latest data.
 	CreateIndexScanState state;
-	vector<column_t> cids;
-	cids.push_back(physical_index);
-	// Use ScanCommitted to scan the latest committed data
-	state.Initialize(cids, nullptr);
-	InitializeScan(state.table_state, cids, nullptr);
+	auto scan_type = TableScanType::TABLE_SCAN_COMMITTED_ROWS_OMIT_PERMANENTLY_DELETED;
+	state.Initialize(column_ids, nullptr);
+	InitializeScan(state.table_state, column_ids, nullptr);
+
 	InitializeCreateIndexScan(state);
+
 	while (true) {
 		scan_chunk.Reset();
-		state.table_state.ScanCommitted(scan_chunk, state.segment_lock,
-		                                TableScanType::TABLE_SCAN_COMMITTED_ROWS_OMIT_PERMANENTLY_DELETED);
+		state.table_state.ScanCommitted(scan_chunk, state.segment_lock, scan_type);
 		if (scan_chunk.size() == 0) {
 			break;
 		}
-		// Check constraint
+
+		// Verify the NOT NULL constraint.
 		if (VectorOperations::HasNull(scan_chunk.data[0], scan_chunk.size())) {
-			throw ConstraintException("NOT NULL constraint failed: %s.%s", info->GetTableName(),
-			                          parent.Columns()[physical_index].GetName());
+			auto name = parent.Columns()[physical_index].GetName();
+			throw ConstraintException("NOT NULL constraint failed: %s.%s", info->GetTableName(), name);
 		}
 	}
 }
 
 //===--------------------------------------------------------------------===//
 // Statistics
-//===--------------------------------------------------------------------===//
+//===---------------------------------------------------------------r-----===//
 void RowGroupCollection::CopyStats(TableStatistics &other_stats) {
 	stats.CopyStats(other_stats);
 }
@@ -1176,6 +1253,18 @@ unique_ptr<BaseStatistics> RowGroupCollection::CopyStats(column_t column_id) {
 	return stats.CopyStats(column_id);
 }
 
+unique_ptr<BlockingSample> RowGroupCollection::GetSample() {
+	auto lock = stats.GetLock();
+	auto &sample = stats.GetTableSampleRef(*lock);
+	if (!sample.destroyed) {
+		D_ASSERT(sample.type == SampleType::RESERVOIR_SAMPLE);
+		auto ret = sample.Copy();
+		ret->Cast<ReservoirSample>().EvictOverBudgetSamples();
+		return ret;
+	}
+	return nullptr;
+}
+
 void RowGroupCollection::SetDistinct(column_t column_id, unique_ptr<DistinctStatistics> distinct_stats) {
 	D_ASSERT(column_id != COLUMN_IDENTIFIER_ROW_ID);
 	auto stats_lock = stats.GetLock();

package/src/duckdb/src/storage/table/row_version_manager.cpp

@@ -14,9 +14,9 @@ void RowVersionManager::SetStart(idx_t new_start) {
 	lock_guard<mutex> l(version_lock);
 	this->start = new_start;
 	idx_t current_start = start;
-	for (idx_t i = 0; i < Storage::ROW_GROUP_VECTOR_COUNT; i++) {
-		if (vector_info[i]) {
-			vector_info[i]->start = current_start;
+	for (auto &info : vector_info) {
+		if (info) {
+			info->start = current_start;
 		}
 		current_start += STANDARD_VECTOR_SIZE;
 	}
@@ -26,7 +26,7 @@ idx_t RowVersionManager::GetCommittedDeletedCount(idx_t count) {
 	lock_guard<mutex> l(version_lock);
 	idx_t deleted_count = 0;
 	for (idx_t r = 0, i = 0; r < count; r += STANDARD_VECTOR_SIZE, i++) {
-		if (!vector_info[i]) {
+		if (i >= vector_info.size() || !vector_info[i]) {
 			continue;
 		}
 		idx_t max_count = MinValue<idx_t>(STANDARD_VECTOR_SIZE, count - r);
@@ -39,6 +39,9 @@ idx_t RowVersionManager::GetCommittedDeletedCount(idx_t count) {
 }
 
 optional_ptr<ChunkInfo> RowVersionManager::GetChunkInfo(idx_t vector_idx) {
+	if (vector_idx >= vector_info.size()) {
+		return nullptr;
+	}
 	return vector_info[vector_idx].get();
 }
 
@@ -72,12 +75,27 @@ bool RowVersionManager::Fetch(TransactionData transaction, idx_t row) {
 	return info->Fetch(transaction, UnsafeNumericCast<row_t>(row - vector_index * STANDARD_VECTOR_SIZE));
 }
 
+void RowVersionManager::FillVectorInfo(idx_t vector_idx) {
+	if (vector_idx < vector_info.size()) {
+		return;
+	}
+	vector_info.reserve(vector_idx + 1);
+	for (idx_t i = vector_info.size(); i <= vector_idx; i++) {
+		vector_info.emplace_back();
+	}
+}
+
 void RowVersionManager::AppendVersionInfo(TransactionData transaction, idx_t count, idx_t row_group_start,
                                           idx_t row_group_end) {
 	lock_guard<mutex> lock(version_lock);
 	has_changes = true;
 	idx_t start_vector_idx = row_group_start / STANDARD_VECTOR_SIZE;
 	idx_t end_vector_idx = (row_group_end - 1) / STANDARD_VECTOR_SIZE;
+
+	// fill-up vector_info
+	FillVectorInfo(end_vector_idx);
+
+	// insert the version info nodes
 	for (idx_t vector_idx = start_vector_idx; vector_idx <= end_vector_idx; vector_idx++) {
 		idx_t vector_start =
 		    vector_idx == start_vector_idx ? row_group_start - start_vector_idx * STANDARD_VECTOR_SIZE : 0;
@@ -143,7 +161,7 @@ void RowVersionManager::CleanupAppend(transaction_t lowest_active_transaction, i
 			// not written fully - skip
 			continue;
 		}
-		if (!vector_info[vector_idx]) {
+		if (vector_idx >= vector_info.size() || !vector_info[vector_idx]) {
 			// already vacuumed - skip
 			continue;
 		}
@@ -160,12 +178,14 @@ void RowVersionManager::CleanupAppend(transaction_t lowest_active_transaction, i
 void RowVersionManager::RevertAppend(idx_t start_row) {
 	lock_guard<mutex> lock(version_lock);
 	idx_t start_vector_idx = (start_row + (STANDARD_VECTOR_SIZE - 1)) / STANDARD_VECTOR_SIZE;
-	for (idx_t vector_idx = start_vector_idx; vector_idx < Storage::ROW_GROUP_VECTOR_COUNT; vector_idx++) {
+	for (idx_t vector_idx = start_vector_idx; vector_idx < vector_info.size(); vector_idx++) {
 		vector_info[vector_idx].reset();
 	}
 }
 
 ChunkVectorInfo &RowVersionManager::GetVectorInfo(idx_t vector_idx) {
+	FillVectorInfo(vector_idx);
+
 	if (!vector_info[vector_idx]) {
 		// no info yet: create it
 		vector_info[vector_idx] = make_uniq<ChunkVectorInfo>(start + vector_idx * STANDARD_VECTOR_SIZE);
@@ -206,7 +226,7 @@ vector<MetaBlockPointer> RowVersionManager::Checkpoint(MetadataManager &manager)
 	}
 	// first count how many ChunkInfo's we need to deserialize
 	vector<pair<idx_t, reference<ChunkInfo>>> to_serialize;
-	for (idx_t vector_idx = 0; vector_idx < Storage::ROW_GROUP_VECTOR_COUNT; vector_idx++) {
+	for (idx_t vector_idx = 0; vector_idx < vector_info.size(); vector_idx++) {
 		auto chunk_info = vector_info[vector_idx].get();
 		if (!chunk_info) {
 			continue;
@@ -248,10 +268,13 @@ shared_ptr<RowVersionManager> RowVersionManager::Deserialize(MetaBlockPointer de
 	D_ASSERT(chunk_count > 0);
 	for (idx_t i = 0; i < chunk_count; i++) {
 		idx_t vector_index = source.Read<idx_t>();
-		if (vector_index >= Storage::ROW_GROUP_VECTOR_COUNT) {
-			throw InternalException(
-			    "In DeserializeDeletes, vector_index is out of range for the row group. Corrupted file?");
+		if (vector_index * STANDARD_VECTOR_SIZE >= Storage::MAX_ROW_GROUP_SIZE) {
+			throw IOException("In DeserializeDeletes, vector_index %llu is out of range for the max row group size of "
+			                  "%llu. Corrupted file?",
+			                  vector_index, Storage::MAX_ROW_GROUP_SIZE);
 		}
+
+		version_info->FillVectorInfo(vector_index);
 		version_info->vector_info[vector_index] = ChunkInfo::Read(source);
 	}
 	version_info->has_changes = false;

package/src/duckdb/src/storage/table/scan_state.cpp

@@ -16,14 +16,19 @@ TableScanState::TableScanState() : table_state(*this), local_state(*this) {
 TableScanState::~TableScanState() {
 }
 
-void TableScanState::Initialize(vector<column_t> column_ids_p, optional_ptr<TableFilterSet> table_filters) {
+void TableScanState::Initialize(vector<StorageIndex> column_ids_p, optional_ptr<TableFilterSet> table_filters,
+                                optional_ptr<SampleOptions> table_sampling) {
 	this->column_ids = std::move(column_ids_p);
 	if (table_filters) {
 		filters.Initialize(*table_filters, column_ids);
 	}
+	if (table_sampling) {
+		sampling_info.do_system_sample = table_sampling->method == SampleMethod::SYSTEM_SAMPLE;
+		sampling_info.sample_rate = table_sampling->sample_size.GetValue<double>() / 100.0;
+	}
 }
 
-const vector<column_t> &TableScanState::GetColumnIds() {
+const vector<StorageIndex> &TableScanState::GetColumnIds() {
 	D_ASSERT(!column_ids.empty());
 	return column_ids;
 }
@@ -35,11 +40,16 @@ ScanFilterInfo &TableScanState::GetFilterInfo() {
 	return filters;
 }
 
-ScanFilter::ScanFilter(idx_t index, const vector<column_t> &column_ids, TableFilter &filter)
-    : scan_column_index(index), table_column_index(column_ids[index]), filter(filter), always_true(false) {
+ScanSamplingInfo &TableScanState::GetSamplingInfo() {
+	return sampling_info;
+}
+
+ScanFilter::ScanFilter(idx_t index, const vector<StorageIndex> &column_ids, TableFilter &filter)
+    : scan_column_index(index), table_column_index(column_ids[index].GetPrimaryIndex()), filter(filter),
+      always_true(false) {
 }
 
-void ScanFilterInfo::Initialize(TableFilterSet &filters, const vector<column_t> &column_ids) {
+void ScanFilterInfo::Initialize(TableFilterSet &filters, const vector<StorageIndex> &column_ids) {
 	D_ASSERT(!filters.filters.empty());
 	table_filters = &filters;
 	adaptive_filter = make_uniq<AdaptiveFilter>(filters);
@@ -133,7 +143,7 @@ void ColumnScanState::Next(idx_t count) {
 	}
 }
 
-const vector<storage_t> &CollectionScanState::GetColumnIds() {
+const vector<StorageIndex> &CollectionScanState::GetColumnIds() {
 	return parent.GetColumnIds();
 }
 
@@ -143,6 +153,10 @@ ScanFilterInfo &CollectionScanState::GetFilterInfo() {
 	return parent.GetFilterInfo();
 }
 
+ScanSamplingInfo &CollectionScanState::GetSamplingInfo() {
+	return parent.GetSamplingInfo();
+}
+
 TableScanOptions &CollectionScanState::GetOptions() {
 	return parent.options;
 }
@@ -153,7 +167,7 @@ ParallelCollectionScanState::ParallelCollectionScanState()
 
 CollectionScanState::CollectionScanState(TableScanState &parent_p)
     : row_group(nullptr), vector_index(0), max_row_group_row(0), row_groups(nullptr), max_row(0), batch_index(0),
-      valid_sel(STANDARD_VECTOR_SIZE), parent(parent_p) {
+      valid_sel(STANDARD_VECTOR_SIZE), random(-1), parent(parent_p) {
 }
 
 bool CollectionScanState::Scan(DuckTransaction &transaction, DataChunk &result) {