duckdb 1.1.4-dev9.0 → 1.2.0

package/src/duckdb/src/storage/single_file_block_manager.cpp

@@ -4,11 +4,13 @@
 #include "duckdb/common/checksum.hpp"
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/serializer/memory_stream.hpp"
+#include "duckdb/main/attached_database.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/main/database.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
 #include "duckdb/storage/metadata/metadata_reader.hpp"
 #include "duckdb/storage/metadata/metadata_writer.hpp"
+#include "duckdb/storage/storage_manager.hpp"
 
 #include <algorithm>
 #include <cstring>
@@ -59,27 +61,25 @@ MainHeader MainHeader::Read(ReadStream &source) {
 	}
 	header.version_number = source.Read<uint64_t>();
 	// check the version number
-	if (header.version_number != VERSION_NUMBER) {
+	if (header.version_number < VERSION_NUMBER_LOWER || header.version_number > VERSION_NUMBER_UPPER) {
 		auto version = GetDuckDBVersion(header.version_number);
 		string version_text;
 		if (!version.empty()) {
 			// known version
 			version_text = "DuckDB version " + string(version);
 		} else {
-			version_text = string("an ") + (VERSION_NUMBER > header.version_number ? "older development" : "newer") +
+			version_text = string("an ") +
+			               (VERSION_NUMBER_UPPER > header.version_number ? "older development" : "newer") +
 			               string(" version of DuckDB");
 		}
 		throw IOException(
-		    "Trying to read a database file with version number %lld, but we can only read version %lld.\n"
+		    "Trying to read a database file with version number %lld, but we can only read versions between %lld and "
+		    "%lld.\n"
 		    "The database file was created with %s.\n\n"
-		    "The storage of DuckDB is not yet stable; newer versions of DuckDB cannot read old database files and "
-		    "vice versa.\n"
-		    "The storage will be stabilized when version 1.0 releases.\n\n"
-		    "For now, we recommend that you load the database file in a supported version of DuckDB, and use the "
-		    "EXPORT DATABASE command "
-		    "followed by IMPORT DATABASE on the current version of DuckDB.\n\n"
-		    "See the storage page for more information: https://duckdb.org/internals/storage",
-		    header.version_number, VERSION_NUMBER, version_text);
+		    "Newer DuckDB version might introduce backward incompatible changes (possibly guarded by compatibility "
+		    "settings)"
+		    "See the storage page for migration strategy and more information: https://duckdb.org/internals/storage",
+		    header.version_number, VERSION_NUMBER_LOWER, VERSION_NUMBER_UPPER, version_text);
 	}
 	// read the flags
 	for (idx_t i = 0; i < FLAG_COUNT; i++) {
@@ -97,15 +97,15 @@ void DatabaseHeader::Write(WriteStream &ser) {
 	ser.Write<uint64_t>(block_count);
 	ser.Write<idx_t>(block_alloc_size);
 	ser.Write<idx_t>(vector_size);
+	ser.Write<idx_t>(serialization_compatibility);
 }
 
-DatabaseHeader DatabaseHeader::Read(ReadStream &source) {
+DatabaseHeader DatabaseHeader::Read(const MainHeader &main_header, ReadStream &source) {
 	DatabaseHeader header;
 	header.iteration = source.Read<uint64_t>();
 	header.meta_block = source.Read<idx_t>();
 	header.free_list = source.Read<idx_t>();
 	header.block_count = source.Read<uint64_t>();
-
 	header.block_alloc_size = source.Read<idx_t>();
 
 	// backwards compatibility
@@ -123,7 +123,13 @@ DatabaseHeader DatabaseHeader::Read(ReadStream &source) {
 		                  "vector size of %llu bytes.",
 		                  STANDARD_VECTOR_SIZE, header.vector_size);
 	}
-
+	if (main_header.version_number == 64) {
+		// version number 64 does not have the serialization compatibility in the file - default to 1
+		header.serialization_compatibility = 1;
+	} else {
+		// read from the file
+		header.serialization_compatibility = source.Read<idx_t>();
+	}
 	return header;
 }
 
@@ -133,10 +139,14 @@ void SerializeHeaderStructure(T header, data_ptr_t ptr) {
 	header.Write(ser);
 }
 
-template <class T>
-T DeserializeHeaderStructure(data_ptr_t ptr) {
+MainHeader DeserializeMainHeader(data_ptr_t ptr) {
+	MemoryStream source(ptr, Storage::FILE_HEADER_SIZE);
+	return MainHeader::Read(source);
+}
+
+DatabaseHeader DeserializeDatabaseHeader(const MainHeader &main_header, data_ptr_t ptr) {
 	MemoryStream source(ptr, Storage::FILE_HEADER_SIZE);
-	return T::Read(source);
+	return DatabaseHeader::Read(main_header, source);
 }
 
 SingleFileBlockManager::SingleFileBlockManager(AttachedDatabase &db, const string &path_p,
@@ -166,6 +176,25 @@ FileOpenFlags SingleFileBlockManager::GetFileFlags(bool create_new) const {
 	return result;
 }
 
+void SingleFileBlockManager::AddStorageVersionTag() {
+	db.tags["storage_version"] = GetStorageVersionName(options.storage_version.GetIndex());
+}
+
+uint64_t SingleFileBlockManager::GetVersionNumber() {
+	uint64_t version_number = VERSION_NUMBER;
+	if (options.storage_version.GetIndex() >= 4) {
+		version_number = 65;
+	}
+	return version_number;
+}
+
+MainHeader ConstructMainHeader(idx_t version_number) {
+	MainHeader main_header;
+	main_header.version_number = version_number;
+	memset(main_header.flags, 0, sizeof(uint64_t) * MainHeader::FLAG_COUNT);
+	return main_header;
+}
+
 void SingleFileBlockManager::CreateNewDatabase() {
 	auto flags = GetFileFlags(true);
 
@@ -177,10 +206,11 @@ void SingleFileBlockManager::CreateNewDatabase() {
 	// first fill in the new header
 	header_buffer.Clear();
 
-	MainHeader main_header;
-	main_header.version_number = VERSION_NUMBER;
-	memset(main_header.flags, 0, sizeof(uint64_t) * MainHeader::FLAG_COUNT);
+	options.version_number = GetVersionNumber();
+	db.GetStorageManager().SetStorageVersion(options.storage_version.GetIndex());
+	AddStorageVersionTag();
 
+	MainHeader main_header = ConstructMainHeader(options.version_number.GetIndex());
 	SerializeHeaderStructure<MainHeader>(main_header, header_buffer.buffer);
 	// now write the header to the file
 	ChecksumAndWrite(header_buffer, 0);
@@ -198,6 +228,7 @@ void SingleFileBlockManager::CreateNewDatabase() {
 	// We create the SingleFileBlockManager with the desired block allocation size before calling CreateNewDatabase.
 	h1.block_alloc_size = GetBlockAllocSize();
 	h1.vector_size = STANDARD_VECTOR_SIZE;
+	h1.serialization_compatibility = options.storage_version.GetIndex();
 	SerializeHeaderStructure<DatabaseHeader>(h1, header_buffer.buffer);
 	ChecksumAndWrite(header_buffer, Storage::FILE_HEADER_SIZE);
 
@@ -210,6 +241,7 @@ void SingleFileBlockManager::CreateNewDatabase() {
 	// We create the SingleFileBlockManager with the desired block allocation size before calling CreateNewDatabase.
 	h2.block_alloc_size = GetBlockAllocSize();
 	h2.vector_size = STANDARD_VECTOR_SIZE;
+	h2.serialization_compatibility = options.storage_version.GetIndex();
 	SerializeHeaderStructure<DatabaseHeader>(h2, header_buffer.buffer);
 	ChecksumAndWrite(header_buffer, Storage::FILE_HEADER_SIZE * 2ULL);
 
@@ -235,16 +267,17 @@ void SingleFileBlockManager::LoadExistingDatabase() {
 	MainHeader::CheckMagicBytes(*handle);
 	// otherwise, we check the metadata of the file
 	ReadAndChecksum(header_buffer, 0);
-	DeserializeHeaderStructure<MainHeader>(header_buffer.buffer);
+	MainHeader main_header = DeserializeMainHeader(header_buffer.buffer);
+	options.version_number = main_header.version_number;
 
 	// read the database headers from disk
 	DatabaseHeader h1;
 	ReadAndChecksum(header_buffer, Storage::FILE_HEADER_SIZE);
-	h1 = DeserializeHeaderStructure<DatabaseHeader>(header_buffer.buffer);
+	h1 = DeserializeDatabaseHeader(main_header, header_buffer.buffer);
 
 	DatabaseHeader h2;
 	ReadAndChecksum(header_buffer, Storage::FILE_HEADER_SIZE * 2ULL);
-	h2 = DeserializeHeaderStructure<DatabaseHeader>(header_buffer.buffer);
+	h2 = DeserializeDatabaseHeader(main_header, header_buffer.buffer);
 
 	// check the header with the highest iteration count
 	if (h1.iteration > h2.iteration) {
@@ -256,6 +289,7 @@ void SingleFileBlockManager::LoadExistingDatabase() {
 		active_header = 1;
 		Initialize(h2, GetOptionalBlockAllocSize());
 	}
+	AddStorageVersionTag();
 	LoadFreeList();
 }
 
@@ -288,11 +322,32 @@ void SingleFileBlockManager::Initialize(const DatabaseHeader &header, const opti
 	meta_block = header.meta_block;
 	iteration_count = header.iteration;
 	max_block = NumericCast<block_id_t>(header.block_count);
+	if (options.storage_version.IsValid()) {
+		// storage version specified explicity - use requested storage version
+		auto requested_compat_version = options.storage_version.GetIndex();
+		if (requested_compat_version < header.serialization_compatibility) {
+			throw InvalidInputException(
+			    "Error opening \"%s\": cannot initialize database with storage version %d - which is lower than what "
+			    "the database itself uses (%d). The storage version of an existing database cannot be lowered.",
+			    path, requested_compat_version, header.serialization_compatibility);
+		}
+	} else {
+		// load storage version from header
+		options.storage_version = header.serialization_compatibility;
+	}
+	if (header.serialization_compatibility > SerializationCompatibility::Latest().serialization_version) {
+		throw InvalidInputException(
+		    "Error opening \"%s\": file was written with a storage version greater than the latest version supported "
+		    "by this DuckDB instance. Try opening the file with a newer version of DuckDB.",
+		    path);
+	}
+	db.GetStorageManager().SetStorageVersion(options.storage_version.GetIndex());
 
 	if (block_alloc_size.IsValid() && block_alloc_size.GetIndex() != header.block_alloc_size) {
-		throw InvalidInputException("cannot initialize the same database with a different block size: provided block "
-		                            "size: %llu, file block size: %llu",
-		                            GetBlockAllocSize(), header.block_alloc_size);
+		throw InvalidInputException(
+		    "Error opening \"%s\": cannot initialize the same database with a different block size: provided block "
+		    "size: %llu, file block size: %llu",
+		    path, GetBlockAllocSize(), header.block_alloc_size);
 	}
 	SetBlockAllocSize(header.block_alloc_size);
 }
@@ -666,6 +721,7 @@ void SingleFileBlockManager::WriteHeader(DatabaseHeader header) {
 	}
 	metadata_manager.Flush();
 	header.block_count = NumericCast<idx_t>(max_block);
+	header.serialization_compatibility = options.storage_version.GetIndex();
 
 	auto &config = DBConfig::Get(db);
 	if (config.options.checkpoint_abort == CheckpointAbort::DEBUG_ABORT_AFTER_FREE_LIST_WRITE) {
@@ -675,9 +731,20 @@ void SingleFileBlockManager::WriteHeader(DatabaseHeader header) {
 	// We need to fsync BEFORE we write the header to ensure that all the previous blocks are written as well
 	handle->Sync();
 
-	// set the header inside the buffer
 	header_buffer.Clear();
-	MemoryStream serializer;
+	// if we are upgrading the database from version 64 -> version 65, we need to re-write the main header
+	if (options.version_number.GetIndex() == 64 && options.storage_version.GetIndex() >= 4) {
+		// rewrite the main header
+		options.version_number = 65;
+		MainHeader main_header = ConstructMainHeader(options.version_number.GetIndex());
+		SerializeHeaderStructure<MainHeader>(main_header, header_buffer.buffer);
+		// now write the header to the file
+		ChecksumAndWrite(header_buffer, 0);
+		header_buffer.Clear();
+	}
+
+	// set the header inside the buffer
+	MemoryStream serializer(Allocator::Get(db));
 	header.Write(serializer);
 	memcpy(header_buffer.buffer, serializer.GetData(), serializer.GetPosition());
 	// now write the header to the file, active_header determines whether we write to h1 or h2

package/src/duckdb/src/storage/storage_info.cpp

@@ -6,6 +6,11 @@
 namespace duckdb {
 
 const uint64_t VERSION_NUMBER = 64;
+const uint64_t VERSION_NUMBER_LOWER = 64;
+const uint64_t VERSION_NUMBER_UPPER = 65;
+
+static_assert(VERSION_NUMBER_LOWER <= VERSION_NUMBER, "Check on VERSION_NUMBER lower bound");
+static_assert(VERSION_NUMBER <= VERSION_NUMBER_UPPER, "Check on VERSION_NUMBER upper bound");
 
 struct StorageVersionInfo {
 	const char *version_name;
@@ -71,6 +76,7 @@ static const StorageVersionInfo storage_version_info[] = {
 	{"v1.1.1", 64},
 	{"v1.1.2", 64},
 	{"v1.1.3", 64},
+	{"v1.2.0", 65},
 	{nullptr, 0}
 };
 // END OF STORAGE VERSION INFO
@@ -99,6 +105,38 @@ static const SerializationVersionInfo serialization_version_info[] = {
 static_assert(DEFAULT_SERIALIZATION_VERSION_INFO <= LATEST_SERIALIZATION_VERSION_INFO,
               "Check on SERIALIZATION_VERSION_INFO");
 
+string GetStorageVersionName(idx_t serialization_version) {
+	if (serialization_version < 4) {
+		// special handling for lower serialization versions
+		return "v1.0.0 - v1.1.3";
+	}
+	optional_idx min_idx;
+	optional_idx max_idx;
+	for (idx_t i = 0; serialization_version_info[i].version_name; i++) {
+		if (strcmp(serialization_version_info[i].version_name, "latest") == 0) {
+			continue;
+		}
+		if (serialization_version_info[i].serialization_version != serialization_version) {
+			continue;
+		}
+		if (!min_idx.IsValid()) {
+			min_idx = i;
+		} else {
+			max_idx = i;
+		}
+	}
+	if (!min_idx.IsValid()) {
+		D_ASSERT(0);
+		return "--UNKNOWN--";
+	}
+	auto min_name = serialization_version_info[min_idx.GetIndex()].version_name;
+	if (!max_idx.IsValid()) {
+		return min_name;
+	}
+	auto max_name = serialization_version_info[max_idx.GetIndex()].version_name;
+	return string(min_name) + " - " + string(max_name);
+}
+
 optional_idx GetStorageVersion(const char *version_string) {
 	for (idx_t i = 0; storage_version_info[i].version_name; i++) {
 		if (!strcmp(storage_version_info[i].version_name, version_string)) {

package/src/duckdb/src/storage/storage_manager.cpp

@@ -145,6 +145,7 @@ void SingleFileStorageManager::LoadDatabase(StorageOptions storage_options) {
 	options.read_only = read_only;
 	options.use_direct_io = config.options.use_direct_io;
 	options.debug_initialize = config.options.debug_initialize;
+	options.storage_version = storage_options.storage_version;
 
 	idx_t row_group_size = DEFAULT_ROW_GROUP_SIZE;
 	if (storage_options.row_group_size.IsValid()) {
@@ -182,6 +183,10 @@ void SingleFileStorageManager::LoadDatabase(StorageOptions storage_options) {
 			// No explicit option provided: use the default option.
 			options.block_alloc_size = config.options.default_block_alloc_size;
 		}
+		if (!options.storage_version.IsValid()) {
+			// when creating a new database we default to the serialization version specified in the config
+			options.storage_version = config.options.serialization_compatibility.serialization_version;
+		}
 
 		// Initialize the block manager before creating a new database.
 		auto sf_block_manager = make_uniq<SingleFileBlockManager>(db, path, options);
@@ -218,6 +223,12 @@ void SingleFileStorageManager::LoadDatabase(StorageOptions storage_options) {
 		auto wal_path = GetWALPath();
 		wal = WriteAheadLog::Replay(fs, db, wal_path);
 	}
+	if (row_group_size > 122880ULL && GetStorageVersion() < 4) {
+		throw InvalidInputException("Unsupported row group size %llu - row group sizes >= 122_880 are only supported "
+		                            "with STORAGE_VERSION '1.2.0' or above.\nExplicitly specify a newer storage "
+		                            "version when creating the database to enable larger row groups",
+		                            row_group_size);
+	}
 
 	load_complete = true;
 }

package/src/duckdb/src/storage/table/column_data.cpp

@@ -51,6 +51,10 @@ DataTableInfo &ColumnData::GetTableInfo() const {
 	return info;
 }
 
+StorageManager &ColumnData::GetStorageManager() const {
+	return info.GetDB().GetStorageManager();
+}
+
 const LogicalType &ColumnData::RootType() const {
 	if (parent) {
 		return parent->RootType();

package/src/duckdb/src/storage/table/column_data_checkpointer.cpp

@@ -250,9 +250,9 @@ vector<CheckpointAnalyzeResult> ColumnDataCheckpointer::DetectBestCompressionMet
 		D_ASSERT(compression_idx != DConstants::INVALID_INDEX);
 
 		auto &best_function = *functions[compression_idx];
-		Logger::Info(db, "FinalAnalyze(%s) result for %s.%s.%d(%s): %d", EnumUtil::ToString(best_function.type),
-		             col_data.info.GetSchemaName(), col_data.info.GetTableName(), col_data.column_index,
-		             col_data.type.ToString(), best_score);
+		DUCKDB_LOG_INFO(db, "duckdb.ColumnDataCheckPointer", "FinalAnalyze(%s) result for %s.%s.%d(%s): %d",
+		                EnumUtil::ToString(best_function.type), col_data.info.GetSchemaName(),
+		                col_data.info.GetTableName(), col_data.column_index, col_data.type.ToString(), best_score);
 		result[i] = CheckpointAnalyzeResult(std::move(chosen_state), best_function);
 	}
 	return result;

package/src/duckdb/src/storage/table/row_group_collection.cpp

@@ -404,12 +404,6 @@ bool RowGroupCollection::Append(DataChunk &chunk, TableAppendState &state) {
 		column_stats.UpdateDistinctStatistics(chunk.data[col_idx], chunk.size(), state.hashes);
 	}
 
-	auto &table_sample = state.stats.GetTableSampleRef(*local_stats_lock);
-	if (!table_sample.destroyed) {
-		D_ASSERT(table_sample.type == SampleType::RESERVOIR_SAMPLE);
-		table_sample.AddToReservoir(chunk);
-	}
-
 	return new_row_group;
 }
 
@@ -443,22 +437,6 @@ void RowGroupCollection::FinalizeAppend(TransactionData transaction, TableAppend
 		global_stats.DistinctStats().Merge(local_stats.DistinctStats());
 	}
 
-	auto local_sample = state.stats.GetTableSample(*local_stats_lock);
-	auto global_sample = stats.GetTableSample(*global_stats_lock);
-
-	if (local_sample && global_sample) {
-		D_ASSERT(global_sample->type == SampleType::RESERVOIR_SAMPLE);
-		auto &reservoir_sample = global_sample->Cast<ReservoirSample>();
-		reservoir_sample.Merge(std::move(local_sample));
-		// initialize the thread local sample again
-		auto new_local_sample = make_uniq<ReservoirSample>(reservoir_sample.GetSampleCount());
-		state.stats.SetTableSample(*local_stats_lock, std::move(new_local_sample));
-		stats.SetTableSample(*global_stats_lock, std::move(global_sample));
-	} else {
-		state.stats.SetTableSample(*local_stats_lock, std::move(local_sample));
-		stats.SetTableSample(*global_stats_lock, std::move(global_sample));
-	}
-
 	Verify();
 }
 
@@ -607,10 +585,6 @@ idx_t RowGroupCollection::Delete(TransactionData transaction, DataTable &table,
 		delete_count += row_group->Delete(transaction, table, ids + start, pos - start);
 	} while (pos < count);
 
-	// When deleting destroy the sample.
-	auto stats_guard = stats.GetLock();
-	stats.DestroyTableSample(*stats_guard);
-
 	return delete_count;
 }
 
@@ -619,6 +593,7 @@ idx_t RowGroupCollection::Delete(TransactionData transaction, DataTable &table,
 //===--------------------------------------------------------------------===//
 void RowGroupCollection::Update(TransactionData transaction, row_t *ids, const vector<PhysicalIndex> &column_ids,
                                 DataChunk &updates) {
+	D_ASSERT(updates.size() >= 1);
 	idx_t pos = 0;
 	do {
 		idx_t start = pos;
@@ -648,72 +623,106 @@ void RowGroupCollection::Update(TransactionData transaction, row_t *ids, const v
 			stats.MergeStats(*l, column_id.index, *row_group->GetStatistics(column_id.index));
 		}
 	} while (pos < updates.size());
-	// on update destroy the sample
-	auto stats_guard = stats.GetLock();
-	stats.DestroyTableSample(*stats_guard);
 }
 
 void RowGroupCollection::RemoveFromIndexes(TableIndexList &indexes, Vector &row_identifiers, idx_t count) {
 	auto row_ids = FlatVector::GetData<row_t>(row_identifiers);
 
-	// initialize the fetch state
-	// FIXME: we do not need to fetch all columns, only the columns required by the indices!
-	TableScanState state;
+	// Collect all indexed columns.
+	unordered_set<column_t> indexed_column_id_set;
+	indexes.Scan([&](Index &index) {
+		D_ASSERT(index.IsBound());
+		auto &set = index.GetColumnIdSet();
+		indexed_column_id_set.insert(set.begin(), set.end());
+		return false;
+	});
 	vector<StorageIndex> column_ids;
-	column_ids.reserve(types.size());
-	for (idx_t i = 0; i < types.size(); i++) {
-		column_ids.emplace_back(i);
+	for (auto &col : indexed_column_id_set) {
+		column_ids.emplace_back(col);
 	}
+	sort(column_ids.begin(), column_ids.end());
+
+	vector<LogicalType> column_types;
+	for (auto &col : column_ids) {
+		column_types.push_back(types[col.GetPrimaryIndex()]);
+	}
+
+	// Initialize the fetch state. Only use indexed columns.
+	TableScanState state;
 	state.Initialize(std::move(column_ids));
 	state.table_state.max_row = row_start + total_rows;
 
-	// initialize the fetch chunk
-	DataChunk result;
-	result.Initialize(GetAllocator(), types);
+	// Used for scanning data. Only contains the indexed columns.
+	DataChunk fetch_chunk;
+	fetch_chunk.Initialize(GetAllocator(), column_types);
+
+	// Used for index value removal.
+	// Contains all columns but only initializes indexed ones.
+	DataChunk result_chunk;
+	auto fetched_columns = vector<bool>(types.size(), false);
+	result_chunk.Initialize(GetAllocator(), types, fetched_columns);
+
+	// Now set all to-be-fetched columns.
+	for (auto &col : indexed_column_id_set) {
+		fetched_columns[col] = true;
+	}
 
+	// Iterate over the row ids.
 	SelectionVector sel(STANDARD_VECTOR_SIZE);
-	// now iterate over the row ids
 	for (idx_t r = 0; r < count;) {
-		result.Reset();
-		// figure out which row_group to fetch from
+		fetch_chunk.Reset();
+		result_chunk.Reset();
+
+		// Figure out which row_group to fetch from.
 		auto row_id = row_ids[r];
 		auto row_group = row_groups->GetSegment(UnsafeNumericCast<idx_t>(row_id));
 		auto row_group_vector_idx = (UnsafeNumericCast<idx_t>(row_id) - row_group->start) / STANDARD_VECTOR_SIZE;
 		auto base_row_id = row_group_vector_idx * STANDARD_VECTOR_SIZE + row_group->start;
 
-		// fetch the current vector
+		// Fetch the current vector into fetch_chunk.
 		state.table_state.Initialize(GetTypes());
 		row_group->InitializeScanWithOffset(state.table_state, row_group_vector_idx);
-		row_group->ScanCommitted(state.table_state, result, TableScanType::TABLE_SCAN_COMMITTED_ROWS);
-		result.Verify();
+		row_group->ScanCommitted(state.table_state, fetch_chunk, TableScanType::TABLE_SCAN_COMMITTED_ROWS);
+		fetch_chunk.Verify();
 
-		// check for any remaining row ids if they also fall into this vector
-		// we try to fetch handle as many rows as possible at the same time
+		// Check for any remaining row ids, if they also fall into this vector.
+		// We try to fetch as many rows as possible at the same time.
 		idx_t sel_count = 0;
 		for (; r < count; r++) {
 			idx_t current_row = idx_t(row_ids[r]);
-			if (current_row < base_row_id || current_row >= base_row_id + result.size()) {
-				// this row-id does not fall into the current chunk - break
+			if (current_row < base_row_id || current_row >= base_row_id + fetch_chunk.size()) {
+				// This row id does not fall into the current chunk.
 				break;
 			}
 			auto row_in_vector = current_row - base_row_id;
-			D_ASSERT(row_in_vector < result.size());
+			D_ASSERT(row_in_vector < fetch_chunk.size());
 			sel.set_index(sel_count++, row_in_vector);
 		}
 		D_ASSERT(sel_count > 0);
-		// slice the vector with all rows that are present in this vector and erase from the index
-		result.Slice(sel, sel_count);
 
+		// Reference the necessary columns of the fetch_chunk.
+		idx_t fetch_idx = 0;
+		for (idx_t j = 0; j < types.size(); j++) {
+			if (fetched_columns[j]) {
+				result_chunk.data[j].Reference(fetch_chunk.data[fetch_idx++]);
+				continue;
+			}
+			result_chunk.data[j].Reference(Value(types[j]));
+		}
+		result_chunk.SetCardinality(fetch_chunk);
+
+		// Slice the vector with all rows that are present in this vector.
+		// Then, erase all values from the indexes.
+		result_chunk.Slice(sel, sel_count);
 		indexes.Scan([&](Index &index) {
 			if (index.IsBound()) {
-				index.Cast<BoundIndex>().Delete(result, row_identifiers);
-			} else {
-				throw MissingExtensionException(
-				    "Cannot delete from index '%s', unknown index type '%s'. You need to load the "
-				    "extension that provides this index type before table '%s' can be modified.",
-				    index.GetIndexName(), index.GetIndexType(), info->GetTableName());
+				index.Cast<BoundIndex>().Delete(result_chunk, row_identifiers);
+				return false;
 			}
-			return false;
+			throw MissingExtensionException(
+			    "Cannot delete from index '%s', unknown index type '%s'. You need to load the "
+			    "extension that provides this index type before table '%s' can be modified.",
+			    index.GetIndexName(), index.GetIndexType(), info->GetTableName());
 		});
 	}
 }
@@ -1134,8 +1143,6 @@ shared_ptr<RowGroupCollection> RowGroupCollection::AddColumn(ClientContext &cont
 
 		result->row_groups->AppendSegment(std::move(new_row_group));
 	}
-	// When adding a column destroy the sample
-	stats.DestroyTableSample(*lock);
 
 	return result;
 }
@@ -1254,14 +1261,6 @@ unique_ptr<BaseStatistics> RowGroupCollection::CopyStats(column_t column_id) {
 }
 
 unique_ptr<BlockingSample> RowGroupCollection::GetSample() {
-	auto lock = stats.GetLock();
-	auto &sample = stats.GetTableSampleRef(*lock);
-	if (!sample.destroyed) {
-		D_ASSERT(sample.type == SampleType::RESERVOIR_SAMPLE);
-		auto ret = sample.Copy();
-		ret->Cast<ReservoirSample>().EvictOverBudgetSamples();
-		return ret;
-	}
 	return nullptr;
 }
 

package/src/duckdb/src/storage/table/table_statistics.cpp

@@ -142,10 +142,10 @@ ColumnStatistics &TableStatistics::GetStats(TableStatisticsLock &lock, idx_t i)
 	return *column_stats[i];
 }
 
-BlockingSample &TableStatistics::GetTableSampleRef(TableStatisticsLock &lock) {
-	D_ASSERT(table_sample);
-	return *table_sample;
-}
+// BlockingSample &TableStatistics::GetTableSampleRef(TableStatisticsLock &lock) {
+//	D_ASSERT(table_sample);
+//	return *table_sample;
+//}
 
 unique_ptr<BlockingSample> TableStatistics::GetTableSample(TableStatisticsLock &lock) {
 	return std::move(table_sample);