duckdb 0.8.2-dev4314.0 → 0.8.2-dev4376.0

package/src/duckdb/src/include/duckdb/transaction/undo_buffer.hpp

@@ -48,7 +48,6 @@ public:
 	void Rollback() noexcept;
 
 private:
-	ClientContext &context;
 	ArenaAllocator allocator;
 
 private:

package/src/duckdb/src/main/settings/settings.cpp

@@ -726,17 +726,12 @@ Value ForceCompressionSetting::GetSetting(ClientContext &context) {
 //===--------------------------------------------------------------------===//
 void ForceBitpackingModeSetting::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) {
 	auto mode_str = StringUtil::Lower(input.ToString());
-	if (mode_str == "none") {
-		config.options.force_bitpacking_mode = BitpackingMode::AUTO;
-	} else {
-		auto mode = BitpackingModeFromString(mode_str);
-		if (mode == BitpackingMode::AUTO) {
-			throw ParserException(
-			    "Unrecognized option for force_bitpacking_mode, expected none, constant, constant_delta, "
-			    "delta_for, or for");
-		}
-		config.options.force_bitpacking_mode = mode;
+	auto mode = BitpackingModeFromString(mode_str);
+	if (mode == BitpackingMode::INVALID) {
+		throw ParserException("Unrecognized option for force_bitpacking_mode, expected none, constant, constant_delta, "
+		                      "delta_for, or for");
 	}
+	config.options.force_bitpacking_mode = mode;
 }
 
 void ForceBitpackingModeSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) {

package/src/duckdb/src/optimizer/statistics/expression/propagate_cast.cpp

@@ -22,6 +22,20 @@ static unique_ptr<BaseStatistics> StatisticsOperationsNumericNumericCast(const B
 }
 
 static unique_ptr<BaseStatistics> StatisticsNumericCastSwitch(const BaseStatistics &input, const LogicalType &target) {
+	//	Downcasting timestamps to times is not a truncation operation
+	switch (target.id()) {
+	case LogicalTypeId::TIME:
+		switch (input.GetType().id()) {
+		case LogicalTypeId::TIMESTAMP:
+		case LogicalTypeId::TIMESTAMP_TZ:
+			return nullptr;
+		default:
+			break;
+		}
+	default:
+		break;
+	}
+
 	switch (target.InternalType()) {
 	case PhysicalType::INT8:
 	case PhysicalType::INT16:

package/src/duckdb/src/storage/checkpoint/table_data_writer.cpp

@@ -66,7 +66,6 @@ void SingleFileTableDataWriter::FinalizeTable(TableStatistics &&global_stats, Da
 		row_group_serializer.End();
 	}
 
-	// TODO: Serialize this:
 	auto index_pointers = info->indexes.SerializeIndexes(table_data_writer);
 
 	// Now begin the metadata as a unit

package/src/duckdb/src/storage/compression/bitpacking.cpp

@@ -23,8 +23,7 @@ static constexpr const idx_t BITPACKING_METADATA_GROUP_SIZE = STANDARD_VECTOR_SI
 
 BitpackingMode BitpackingModeFromString(const string &str) {
 	auto mode = StringUtil::Lower(str);
-
-	if (mode == "auto") {
+	if (mode == "auto" || mode == "none") {
 		return BitpackingMode::AUTO;
 	} else if (mode == "constant") {
 		return BitpackingMode::CONSTANT;
@@ -35,21 +34,21 @@ BitpackingMode BitpackingModeFromString(const string &str) {
 	} else if (mode == "for") {
 		return BitpackingMode::FOR;
 	} else {
-		return BitpackingMode::AUTO;
+		return BitpackingMode::INVALID;
 	}
 }
 
 string BitpackingModeToString(const BitpackingMode &mode) {
 	switch (mode) {
-	case (BitpackingMode::AUTO):
+	case BitpackingMode::AUTO:
 		return "auto";
-	case (BitpackingMode::CONSTANT):
+	case BitpackingMode::CONSTANT:
 		return "constant";
-	case (BitpackingMode::CONSTANT_DELTA):
+	case BitpackingMode::CONSTANT_DELTA:
 		return "constant_delta";
-	case (BitpackingMode::DELTA_FOR):
+	case BitpackingMode::DELTA_FOR:
 		return "delta_for";
-	case (BitpackingMode::FOR):
+	case BitpackingMode::FOR:
 		return "for";
 	default:
 		throw NotImplementedException("Unknown bitpacking mode: " + to_string((uint8_t)mode) + "\n");
@@ -161,7 +160,7 @@ public:
 		// Don't delta encoding 1 value makes no sense
 		if (compression_buffer_idx < 2) {
 			return;
-		};
+		}
 
 		// TODO: handle NULLS here?
 		// Currently we cannot handle nulls because we would need an additional step of patching for this.
@@ -686,48 +685,57 @@ public:
 	}
 
 	void Skip(ColumnSegment &segment, idx_t skip_count) {
-		while (skip_count > 0) {
-			if (current_group_offset + skip_count < BITPACKING_METADATA_GROUP_SIZE) {
-				// Skipping Delta FOR requires a bit of decoding to figure out the new delta
-				if (current_group.mode == BitpackingMode::DELTA_FOR) {
-					// if current_group_offset points into the middle of a
-					// BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE, we need to scan a few
-					// values before current_group_offset to align with the algorithm groups
-					idx_t extra_count = current_group_offset % BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE;
-
-					// Calculate total offset and count to bitunpack
-					idx_t base_decompress_count = BitpackingPrimitives::RoundUpToAlgorithmGroupSize(skip_count);
-					idx_t decompress_count = base_decompress_count + extra_count;
-					idx_t decompress_offset = current_group_offset - extra_count;
-					bool skip_sign_extension = true;
-
-					BitpackingPrimitives::UnPackBuffer<T>(data_ptr_cast(decompression_buffer),
-					                                      current_group_ptr + decompress_offset, decompress_count,
-					                                      current_width, skip_sign_extension);
-
-					ApplyFrameOfReference<T_S>(reinterpret_cast<T_S *>(decompression_buffer + extra_count),
-					                           current_frame_of_reference, skip_count);
-					DeltaDecode<T_S>(reinterpret_cast<T_S *>(decompression_buffer + extra_count),
-					                 static_cast<T_S>(current_delta_offset), skip_count);
-					current_delta_offset = decompression_buffer[extra_count + skip_count - 1];
-
-					current_group_offset += skip_count;
-				} else {
-					current_group_offset += skip_count;
-				}
-				break;
-			} else {
-				auto left_in_this_group = BITPACKING_METADATA_GROUP_SIZE - current_group_offset;
-				auto number_of_groups_to_skip = (skip_count - left_in_this_group) / BITPACKING_METADATA_GROUP_SIZE;
-
-				current_group_offset = 0;
-				bitpacking_metadata_ptr -= number_of_groups_to_skip * sizeof(bitpacking_metadata_encoded_t);
+		bool skip_sign_extend = true;
 
+		idx_t skipped = 0;
+		while (skipped < skip_count) {
+			// Exhausted this metadata group, move pointers to next group and load metadata for next group.
+			if (current_group_offset >= BITPACKING_METADATA_GROUP_SIZE) {
 				LoadNextGroup();
+			}
 
-				skip_count -= left_in_this_group;
-				skip_count -= number_of_groups_to_skip * BITPACKING_METADATA_GROUP_SIZE;
+			idx_t offset_in_compression_group =
+			    current_group_offset % BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE;
+
+			if (current_group.mode == BitpackingMode::CONSTANT) {
+				idx_t remaining = skip_count - skipped;
+				idx_t to_skip = MinValue(remaining, BITPACKING_METADATA_GROUP_SIZE - current_group_offset);
+				skipped += to_skip;
+				current_group_offset += to_skip;
+				continue;
+			}
+			if (current_group.mode == BitpackingMode::CONSTANT_DELTA) {
+				idx_t remaining = skip_count - skipped;
+				idx_t to_skip = MinValue(remaining, BITPACKING_METADATA_GROUP_SIZE - current_group_offset);
+				skipped += to_skip;
+				current_group_offset += to_skip;
+				continue;
+			}
+			D_ASSERT(current_group.mode == BitpackingMode::FOR || current_group.mode == BitpackingMode::DELTA_FOR);
+
+			idx_t to_skip =
+			    MinValue<idx_t>(skip_count - skipped,
+			                    BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE - offset_in_compression_group);
+			// Calculate start of compression algorithm group
+			if (current_group.mode == BitpackingMode::DELTA_FOR) {
+				data_ptr_t current_position_ptr = current_group_ptr + current_group_offset * current_width / 8;
+				data_ptr_t decompression_group_start_pointer =
+				    current_position_ptr - offset_in_compression_group * current_width / 8;
+
+				BitpackingPrimitives::UnPackBlock<T>(data_ptr_cast(decompression_buffer),
+				                                     decompression_group_start_pointer, current_width,
+				                                     skip_sign_extend);
+
+				T *decompression_ptr = decompression_buffer + offset_in_compression_group;
+				ApplyFrameOfReference<T_S>(reinterpret_cast<T_S *>(decompression_ptr),
+				                           static_cast<T_S>(current_frame_of_reference), to_skip);
+				DeltaDecode<T_S>(reinterpret_cast<T_S *>(decompression_ptr), static_cast<T_S>(current_delta_offset),
+				                 to_skip);
+				current_delta_offset = decompression_ptr[to_skip - 1];
 			}
+
+			skipped += to_skip;
+			current_group_offset += to_skip;
 		}
 	}
 
@@ -757,7 +765,6 @@ void BitpackingScanPartial(ColumnSegment &segment, ColumnScanState &state, idx_t
 	bool skip_sign_extend = true;
 
 	idx_t scanned = 0;
-
 	while (scanned < scan_count) {
 		// Exhausted this metadata group, move pointers to next group and load metadata for next group.
 		if (scan_state.current_group_offset >= BITPACKING_METADATA_GROUP_SIZE) {

package/src/duckdb/src/storage/data_table.cpp

@@ -208,7 +208,7 @@ void DataTable::InitializeScanWithOffset(TableScanState &state, const vector<col
 }
 
 idx_t DataTable::MaxThreads(ClientContext &context) {
-	idx_t parallel_scan_vector_count = RowGroup::ROW_GROUP_VECTOR_COUNT;
+	idx_t parallel_scan_vector_count = Storage::ROW_GROUP_VECTOR_COUNT;
 	if (ClientConfig::GetConfig(context).verify_parallelism) {
 		parallel_scan_vector_count = 1;
 	}

package/src/duckdb/src/storage/local_storage.cpp

@@ -102,7 +102,7 @@ void LocalTableStorage::WriteNewRowGroup() {
 }
 
 void LocalTableStorage::FlushBlocks() {
-	if (!merged_storage && row_groups->GetTotalRows() > RowGroup::ROW_GROUP_SIZE) {
+	if (!merged_storage && row_groups->GetTotalRows() > Storage::ROW_GROUP_SIZE) {
 		optimistic_writer.WriteLastRowGroup(*row_groups);
 	}
 	optimistic_writer.FinalFlush();

package/src/duckdb/src/storage/metadata/metadata_manager.cpp

@@ -34,6 +34,12 @@ MetadataHandle MetadataManager::AllocateHandle() {
 	MetadataPointer pointer;
 	pointer.block_index = free_block;
 	auto &block = blocks[free_block];
+	if (block.block->BlockId() < MAXIMUM_BLOCK) {
+		// this block is a disk-backed block, yet we are planning to write to it
+		// we need to convert it into a transient block before we can write to it
+		ConvertToTransient(block);
+		D_ASSERT(block.block->BlockId() >= MAXIMUM_BLOCK);
+	}
 	D_ASSERT(!block.free_blocks.empty());
 	pointer.index = block.free_blocks.back();
 	// mark the block as used
@@ -54,6 +60,23 @@ MetadataHandle MetadataManager::Pin(MetadataPointer pointer) {
 	return handle;
 }
 
+void MetadataManager::ConvertToTransient(MetadataBlock &block) {
+	// pin the old block
+	auto old_buffer = buffer_manager.Pin(block.block);
+
+	// allocate a new transient block to replace it
+	shared_ptr<BlockHandle> new_block;
+	auto new_buffer = buffer_manager.Allocate(Storage::BLOCK_SIZE, false, &new_block);
+
+	// copy the data to the transient block
+	memcpy(new_buffer.Ptr(), old_buffer.Ptr(), Storage::BLOCK_SIZE);
+
+	block.block = std::move(new_block);
+
+	// unregister the old block
+	block_manager.UnregisterBlock(block.block_id, false);
+}
+
 block_id_t MetadataManager::AllocateNewBlock() {
 	auto new_block_id = GetNextBlockId();
 
@@ -91,11 +114,11 @@ MetaBlockPointer MetadataManager::GetDiskPointer(MetadataPointer pointer, uint32
 	return MetaBlockPointer(block_pointer, offset);
 }
 
-block_id_t MetaBlockPointer::GetBlockId() {
+block_id_t MetaBlockPointer::GetBlockId() const {
 	return block_id_t(block_pointer & ~(idx_t(0xFF) << 56ULL));
 }
 
-uint32_t MetaBlockPointer::GetBlockIndex() {
+uint32_t MetaBlockPointer::GetBlockIndex() const {
 	return block_pointer >> 56ULL;
 }
 
@@ -262,6 +285,22 @@ void MetadataManager::MarkBlocksAsModified() {
 	}
 }
 
+void MetadataManager::ClearModifiedBlocks(const vector<MetaBlockPointer> &pointers) {
+	for (auto &pointer : pointers) {
+		auto block_id = pointer.GetBlockId();
+		auto block_index = pointer.GetBlockIndex();
+		auto entry = modified_blocks.find(block_id);
+		if (entry == modified_blocks.end()) {
+			throw InternalException("ClearModifiedBlocks - Block id %llu not found in modified_blocks", block_id);
+		}
+		auto &modified_list = entry->second;
+		// verify the block has been modified
+		D_ASSERT(modified_list && (1ULL << block_index));
+		// unset the bit
+		modified_list &= ~(1ULL << block_index);
+	}
+}
+
 block_id_t MetadataManager::GetNextBlockId() {
 	return block_manager.GetFreeBlockId();
 }

package/src/duckdb/src/storage/metadata/metadata_reader.cpp

@@ -2,9 +2,14 @@
 
 namespace duckdb {
 
-MetadataReader::MetadataReader(MetadataManager &manager, MetaBlockPointer pointer, BlockReaderType type)
-    : manager(manager), type(type), next_pointer(FromDiskPointer(pointer)), has_next_block(true), index(0), offset(0),
-      next_offset(pointer.offset), capacity(0) {
+MetadataReader::MetadataReader(MetadataManager &manager, MetaBlockPointer pointer,
+                               optional_ptr<vector<MetaBlockPointer>> read_pointers_p, BlockReaderType type)
+    : manager(manager), type(type), next_pointer(FromDiskPointer(pointer)), has_next_block(true),
+      read_pointers(read_pointers_p), index(0), offset(0), next_offset(pointer.offset), capacity(0) {
+	if (read_pointers) {
+		D_ASSERT(read_pointers->empty());
+		read_pointers->push_back(pointer);
+	}
 }
 
 MetadataReader::MetadataReader(MetadataManager &manager, BlockPointer pointer)
@@ -57,6 +62,10 @@ void MetadataReader::ReadNextBlock() {
 		has_next_block = false;
 	} else {
 		next_pointer = FromDiskPointer(MetaBlockPointer(next_block, 0));
+		MetaBlockPointer next_block_pointer(next_block, 0);
+		if (read_pointers) {
+			read_pointers->push_back(next_block_pointer);
+		}
 	}
 	if (next_offset < sizeof(block_id_t)) {
 		next_offset = sizeof(block_id_t);

package/src/duckdb/src/storage/metadata/metadata_writer.cpp

@@ -3,7 +3,9 @@
 
 namespace duckdb {
 
-MetadataWriter::MetadataWriter(MetadataManager &manager) : manager(manager), capacity(0), offset(0) {
+MetadataWriter::MetadataWriter(MetadataManager &manager, optional_ptr<vector<MetaBlockPointer>> written_pointers_p)
+    : manager(manager), written_pointers(written_pointers_p), capacity(0), offset(0) {
+	D_ASSERT(!written_pointers || written_pointers->empty());
 }
 
 MetadataWriter::~MetadataWriter() {
@@ -38,7 +40,8 @@ void MetadataWriter::NextBlock() {
 
 	// write the block id of the new block to the start of the current block
 	if (capacity > 0) {
-		Store<idx_t>(manager.GetDiskPointer(new_handle.pointer).block_pointer, BasePtr());
+		auto disk_block = manager.GetDiskPointer(new_handle.pointer);
+		Store<idx_t>(disk_block.block_pointer, BasePtr());
 	}
 	// now update the block id of the block
 	block = std::move(new_handle);
@@ -46,6 +49,9 @@ void MetadataWriter::NextBlock() {
 	offset = sizeof(idx_t);
 	capacity = MetadataManager::METADATA_BLOCK_SIZE;
 	Store<idx_t>(-1, BasePtr());
+	if (written_pointers) {
+		written_pointers->push_back(manager.GetDiskPointer(current_pointer));
+	}
 }
 
 void MetadataWriter::WriteData(const_data_ptr_t buffer, idx_t write_size) {

package/src/duckdb/src/storage/single_file_block_manager.cpp

@@ -240,8 +240,7 @@ void SingleFileBlockManager::LoadFreeList() {
 		// no free list
 		return;
 	}
-
-	MetadataReader reader(GetMetadataManager(), free_pointer, BlockReaderType::REGISTER_BLOCKS);
+	MetadataReader reader(GetMetadataManager(), free_pointer, nullptr, BlockReaderType::REGISTER_BLOCKS);
 	auto free_list_count = reader.Read<uint64_t>();
 	free_list.clear();
 	for (idx_t i = 0; i < free_list_count; i++) {

package/src/duckdb/src/storage/storage_info.cpp

@@ -2,7 +2,7 @@
 
 namespace duckdb {
 
-const uint64_t VERSION_NUMBER = 59;
+const uint64_t VERSION_NUMBER = 60;
 
 struct StorageVersionInfo {
 	const char *version_name;

package/src/duckdb/src/storage/table/chunk_info.cpp

@@ -2,6 +2,7 @@
 #include "duckdb/transaction/transaction.hpp"
 #include "duckdb/common/serializer/serializer.hpp"
 #include "duckdb/common/serializer/deserializer.hpp"
+#include "duckdb/common/serializer/memory_stream.hpp"
 
 namespace duckdb {
 
@@ -29,15 +30,19 @@ static bool UseVersion(TransactionData transaction, transaction_t id) {
 	return TransactionVersionOperator::UseInsertedVersion(transaction.start_time, transaction.transaction_id, id);
 }
 
-unique_ptr<ChunkInfo> ChunkInfo::Deserialize(Deserializer &deserializer) {
-	auto type = deserializer.ReadProperty<ChunkInfoType>(100, "type");
+void ChunkInfo::Write(WriteStream &writer) const {
+	writer.Write<ChunkInfoType>(type);
+}
+
+unique_ptr<ChunkInfo> ChunkInfo::Read(ReadStream &reader) {
+	auto type = reader.Read<ChunkInfoType>();
 	switch (type) {
 	case ChunkInfoType::EMPTY_INFO:
 		return nullptr;
 	case ChunkInfoType::CONSTANT_INFO:
-		return ChunkConstantInfo::Deserialize(deserializer);
+		return ChunkConstantInfo::Read(reader);
 	case ChunkInfoType::VECTOR_INFO:
-		return ChunkVectorInfo::Deserialize(deserializer);
+		return ChunkVectorInfo::Read(reader);
 	default:
 		throw SerializationException("Could not deserialize Chunk Info Type: unrecognized type");
 	}
@@ -79,22 +84,23 @@ void ChunkConstantInfo::CommitAppend(transaction_t commit_id, idx_t start, idx_t
 	insert_id = commit_id;
 }
 
+bool ChunkConstantInfo::HasDeletes() const {
+	bool is_deleted = insert_id >= TRANSACTION_ID_START || delete_id < TRANSACTION_ID_START;
+	return is_deleted;
+}
+
 idx_t ChunkConstantInfo::GetCommittedDeletedCount(idx_t max_count) {
 	return delete_id < TRANSACTION_ID_START ? max_count : 0;
 }
 
-void ChunkConstantInfo::Serialize(Serializer &serializer) const {
-	bool is_deleted = insert_id >= TRANSACTION_ID_START || delete_id < TRANSACTION_ID_START;
-	if (!is_deleted) {
-		serializer.WriteProperty(100, "type", ChunkInfoType::EMPTY_INFO);
-		return;
-	}
-	serializer.WriteProperty(100, "type", type);
-	serializer.WriteProperty(200, "start", start);
+void ChunkConstantInfo::Write(WriteStream &writer) const {
+	D_ASSERT(HasDeletes());
+	ChunkInfo::Write(writer);
+	writer.Write<idx_t>(start);
 }
 
-unique_ptr<ChunkInfo> ChunkConstantInfo::Deserialize(Deserializer &deserializer) {
-	auto start = deserializer.ReadProperty<idx_t>(200, "start");
+unique_ptr<ChunkInfo> ChunkConstantInfo::Read(ReadStream &reader) {
+	auto start = reader.Read<idx_t>();
 	auto info = make_uniq<ChunkConstantInfo>(start);
 	info->insert_id = 0;
 	info->delete_id = 0;
@@ -218,6 +224,10 @@ void ChunkVectorInfo::CommitAppend(transaction_t commit_id, idx_t start, idx_t e
 	}
 }
 
+bool ChunkVectorInfo::HasDeletes() const {
+	return any_deleted;
+}
+
 idx_t ChunkVectorInfo::GetCommittedDeletedCount(idx_t max_count) {
 	if (!any_deleted) {
 		return 0;
@@ -231,45 +241,41 @@ idx_t ChunkVectorInfo::GetCommittedDeletedCount(idx_t max_count) {
 	return delete_count;
 }
 
-void ChunkVectorInfo::Serialize(Serializer &serializer) const {
+void ChunkVectorInfo::Write(WriteStream &writer) const {
 	SelectionVector sel(STANDARD_VECTOR_SIZE);
 	transaction_t start_time = TRANSACTION_ID_START - 1;
 	transaction_t transaction_id = DConstants::INVALID_INDEX;
 	idx_t count = GetSelVector(start_time, transaction_id, sel, STANDARD_VECTOR_SIZE);
 	if (count == STANDARD_VECTOR_SIZE) {
 		// nothing is deleted: skip writing anything
-		serializer.WriteProperty(100, "type", ChunkInfoType::EMPTY_INFO);
+		writer.Write<ChunkInfoType>(ChunkInfoType::EMPTY_INFO);
 		return;
 	}
 	if (count == 0) {
 		// everything is deleted: write a constant vector
-		serializer.WriteProperty(100, "type", ChunkInfoType::CONSTANT_INFO);
-		serializer.WriteProperty(200, "start", start);
+		writer.Write<ChunkInfoType>(ChunkInfoType::CONSTANT_INFO);
+		writer.Write<idx_t>(start);
 		return;
 	}
 	// write a boolean vector
-	serializer.WriteProperty(100, "type", ChunkInfoType::VECTOR_INFO);
-	serializer.WriteProperty(200, "start", start);
-	bool deleted_tuples[STANDARD_VECTOR_SIZE];
-	for (idx_t i = 0; i < STANDARD_VECTOR_SIZE; i++) {
-		deleted_tuples[i] = true;
-	}
+	ChunkInfo::Write(writer);
+	writer.Write<idx_t>(start);
+	ValidityMask mask(STANDARD_VECTOR_SIZE);
+	mask.Initialize(STANDARD_VECTOR_SIZE);
 	for (idx_t i = 0; i < count; i++) {
-		deleted_tuples[sel.get_index(i)] = false;
+		mask.SetInvalid(sel.get_index(i));
 	}
-	serializer.WriteProperty(201, "deleted_tuples", data_ptr_cast(deleted_tuples), sizeof(bool) * STANDARD_VECTOR_SIZE);
+	mask.Write(writer, STANDARD_VECTOR_SIZE);
 }
 
-unique_ptr<ChunkInfo> ChunkVectorInfo::Deserialize(Deserializer &deserializer) {
-	auto start = deserializer.ReadProperty<idx_t>(200, "start");
-
+unique_ptr<ChunkInfo> ChunkVectorInfo::Read(ReadStream &reader) {
+	auto start = reader.Read<idx_t>();
 	auto result = make_uniq<ChunkVectorInfo>(start);
 	result->any_deleted = true;
-	bool deleted_tuples[STANDARD_VECTOR_SIZE];
-	deserializer.ReadProperty(201, "deleted_tuples", data_ptr_cast(deleted_tuples),
-	                          sizeof(bool) * STANDARD_VECTOR_SIZE);
+	ValidityMask mask;
+	mask.Read(reader, STANDARD_VECTOR_SIZE);
 	for (idx_t i = 0; i < STANDARD_VECTOR_SIZE; i++) {
-		if (deleted_tuples[i]) {
+		if (mask.RowIsValid(i)) {
 			result->deleted[i] = 0;
 		}
 	}

package/src/duckdb/src/storage/table/column_data.cpp

@@ -87,7 +87,7 @@ void ColumnData::InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx)
 	state.last_offset = 0;
 }
 
-idx_t ColumnData::ScanVector(ColumnScanState &state, Vector &result, idx_t remaining) {
+idx_t ColumnData::ScanVector(ColumnScanState &state, Vector &result, idx_t remaining, bool has_updates) {
 	state.previous_states.clear();
 	if (state.version != version) {
 		InitializeScanWithOffset(state, state.row_index);
@@ -113,7 +113,8 @@ idx_t ColumnData::ScanVector(ColumnScanState &state, Vector &result, idx_t remai
 		idx_t scan_count = MinValue<idx_t>(remaining, state.current->start + state.current->count - state.row_index);
 		idx_t result_offset = initial_remaining - remaining;
 		if (scan_count > 0) {
-			state.current->Scan(state, scan_count, result, result_offset, scan_count == initial_remaining);
+			state.current->Scan(state, scan_count, result, result_offset,
+			                    !has_updates && scan_count == initial_remaining);
 
 			state.row_index += scan_count;
 			remaining -= scan_count;
@@ -138,10 +139,14 @@ idx_t ColumnData::ScanVector(ColumnScanState &state, Vector &result, idx_t remai
 
 template <bool SCAN_COMMITTED, bool ALLOW_UPDATES>
 idx_t ColumnData::ScanVector(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result) {
-	auto scan_count = ScanVector(state, result, STANDARD_VECTOR_SIZE);
-
-	lock_guard<mutex> update_guard(update_lock);
-	if (updates) {
+	bool has_updates;
+	{
+		lock_guard<mutex> update_guard(update_lock);
+		has_updates = updates ? true : false;
+	}
+	auto scan_count = ScanVector(state, result, STANDARD_VECTOR_SIZE, has_updates);
+	if (has_updates) {
+		lock_guard<mutex> update_guard(update_lock);
 		if (!ALLOW_UPDATES && updates->HasUncommittedUpdates(vector_index)) {
 			throw TransactionException("Cannot create index with outstanding updates");
 		}
@@ -179,7 +184,7 @@ idx_t ColumnData::ScanCommitted(idx_t vector_index, ColumnScanState &state, Vect
 void ColumnData::ScanCommittedRange(idx_t row_group_start, idx_t offset_in_row_group, idx_t count, Vector &result) {
 	ColumnScanState child_state;
 	InitializeScanWithOffset(child_state, row_group_start + offset_in_row_group);
-	auto scan_count = ScanVector(child_state, result, count);
+	auto scan_count = ScanVector(child_state, result, count, updates ? true : false);
 	if (updates) {
 		result.Flatten(scan_count);
 		updates->FetchCommittedRange(offset_in_row_group, count, result);
@@ -192,7 +197,7 @@ idx_t ColumnData::ScanCount(ColumnScanState &state, Vector &result, idx_t count)
 	}
 	// ScanCount can only be used if there are no updates
 	D_ASSERT(!updates);
-	return ScanVector(state, result, count);
+	return ScanVector(state, result, count, false);
 }
 
 void ColumnData::Select(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
@@ -339,7 +344,7 @@ idx_t ColumnData::Fetch(ColumnScanState &state, row_t row_id, Vector &result) {
 	state.row_index = start + ((row_id - start) / STANDARD_VECTOR_SIZE * STANDARD_VECTOR_SIZE);
 	state.current = data.GetSegment(state.row_index);
 	state.internal_index = state.current->start;
-	return ScanVector(state, result, STANDARD_VECTOR_SIZE);
+	return ScanVector(state, result, STANDARD_VECTOR_SIZE, false);
 }
 
 void ColumnData::FetchRow(TransactionData transaction, ColumnFetchState &state, row_t row_id, Vector &result,

package/src/duckdb/src/storage/table/list_column_data.cpp

@@ -86,7 +86,7 @@ idx_t ListColumnData::ScanCount(ColumnScanState &state, Vector &result, idx_t co
 	D_ASSERT(!updates);
 
 	Vector offset_vector(LogicalType::UBIGINT, count);
-	idx_t scan_count = ScanVector(state, offset_vector, count);
+	idx_t scan_count = ScanVector(state, offset_vector, count, false);
 	D_ASSERT(scan_count > 0);
 	validity.ScanCount(state.child_states[0], result, count);
 
@@ -132,7 +132,7 @@ void ListColumnData::Skip(ColumnScanState &state, idx_t count) {
 	// note that we only need to read the first and last entry
 	// however, let's just read all "count" entries for now
 	Vector result(LogicalType::UBIGINT, count);
-	idx_t scan_count = ScanVector(state, result, count);
+	idx_t scan_count = ScanVector(state, result, count, false);
 	if (scan_count == 0) {
 		return;
 	}