duckdb 0.8.2-dev4314.0 → 0.8.2-dev4424.0

package/src/duckdb/src/storage/metadata/metadata_reader.cpp

@@ -2,9 +2,14 @@
 
 namespace duckdb {
 
-MetadataReader::MetadataReader(MetadataManager &manager, MetaBlockPointer pointer, BlockReaderType type)
-    : manager(manager), type(type), next_pointer(FromDiskPointer(pointer)), has_next_block(true), index(0), offset(0),
-      next_offset(pointer.offset), capacity(0) {
+MetadataReader::MetadataReader(MetadataManager &manager, MetaBlockPointer pointer,
+                               optional_ptr<vector<MetaBlockPointer>> read_pointers_p, BlockReaderType type)
+    : manager(manager), type(type), next_pointer(FromDiskPointer(pointer)), has_next_block(true),
+      read_pointers(read_pointers_p), index(0), offset(0), next_offset(pointer.offset), capacity(0) {
+	if (read_pointers) {
+		D_ASSERT(read_pointers->empty());
+		read_pointers->push_back(pointer);
+	}
 }
 
 MetadataReader::MetadataReader(MetadataManager &manager, BlockPointer pointer)
@@ -57,6 +62,10 @@ void MetadataReader::ReadNextBlock() {
 		has_next_block = false;
 	} else {
 		next_pointer = FromDiskPointer(MetaBlockPointer(next_block, 0));
+		MetaBlockPointer next_block_pointer(next_block, 0);
+		if (read_pointers) {
+			read_pointers->push_back(next_block_pointer);
+		}
 	}
 	if (next_offset < sizeof(block_id_t)) {
 		next_offset = sizeof(block_id_t);

package/src/duckdb/src/storage/metadata/metadata_writer.cpp

@@ -3,7 +3,9 @@
 
 namespace duckdb {
 
-MetadataWriter::MetadataWriter(MetadataManager &manager) : manager(manager), capacity(0), offset(0) {
+MetadataWriter::MetadataWriter(MetadataManager &manager, optional_ptr<vector<MetaBlockPointer>> written_pointers_p)
+    : manager(manager), written_pointers(written_pointers_p), capacity(0), offset(0) {
+	D_ASSERT(!written_pointers || written_pointers->empty());
 }
 
 MetadataWriter::~MetadataWriter() {
@@ -38,7 +40,8 @@ void MetadataWriter::NextBlock() {
 
 	// write the block id of the new block to the start of the current block
 	if (capacity > 0) {
-		Store<idx_t>(manager.GetDiskPointer(new_handle.pointer).block_pointer, BasePtr());
+		auto disk_block = manager.GetDiskPointer(new_handle.pointer);
+		Store<idx_t>(disk_block.block_pointer, BasePtr());
 	}
 	// now update the block id of the block
 	block = std::move(new_handle);
@@ -46,6 +49,9 @@ void MetadataWriter::NextBlock() {
 	offset = sizeof(idx_t);
 	capacity = MetadataManager::METADATA_BLOCK_SIZE;
 	Store<idx_t>(-1, BasePtr());
+	if (written_pointers) {
+		written_pointers->push_back(manager.GetDiskPointer(current_pointer));
+	}
 }
 
 void MetadataWriter::WriteData(const_data_ptr_t buffer, idx_t write_size) {

package/src/duckdb/src/storage/single_file_block_manager.cpp

@@ -240,8 +240,7 @@ void SingleFileBlockManager::LoadFreeList() {
 		// no free list
 		return;
 	}
-
-	MetadataReader reader(GetMetadataManager(), free_pointer, BlockReaderType::REGISTER_BLOCKS);
+	MetadataReader reader(GetMetadataManager(), free_pointer, nullptr, BlockReaderType::REGISTER_BLOCKS);
 	auto free_list_count = reader.Read<uint64_t>();
 	free_list.clear();
 	for (idx_t i = 0; i < free_list_count; i++) {

package/src/duckdb/src/storage/storage_info.cpp

@@ -2,7 +2,7 @@
 
 namespace duckdb {
 
-const uint64_t VERSION_NUMBER = 59;
+const uint64_t VERSION_NUMBER = 60;
 
 struct StorageVersionInfo {
 	const char *version_name;

package/src/duckdb/src/storage/table/chunk_info.cpp

@@ -2,6 +2,7 @@
 #include "duckdb/transaction/transaction.hpp"
 #include "duckdb/common/serializer/serializer.hpp"
 #include "duckdb/common/serializer/deserializer.hpp"
+#include "duckdb/common/serializer/memory_stream.hpp"
 
 namespace duckdb {
 
@@ -29,15 +30,19 @@ static bool UseVersion(TransactionData transaction, transaction_t id) {
 	return TransactionVersionOperator::UseInsertedVersion(transaction.start_time, transaction.transaction_id, id);
 }
 
-unique_ptr<ChunkInfo> ChunkInfo::Deserialize(Deserializer &deserializer) {
-	auto type = deserializer.ReadProperty<ChunkInfoType>(100, "type");
+void ChunkInfo::Write(WriteStream &writer) const {
+	writer.Write<ChunkInfoType>(type);
+}
+
+unique_ptr<ChunkInfo> ChunkInfo::Read(ReadStream &reader) {
+	auto type = reader.Read<ChunkInfoType>();
 	switch (type) {
 	case ChunkInfoType::EMPTY_INFO:
 		return nullptr;
 	case ChunkInfoType::CONSTANT_INFO:
-		return ChunkConstantInfo::Deserialize(deserializer);
+		return ChunkConstantInfo::Read(reader);
 	case ChunkInfoType::VECTOR_INFO:
-		return ChunkVectorInfo::Deserialize(deserializer);
+		return ChunkVectorInfo::Read(reader);
 	default:
 		throw SerializationException("Could not deserialize Chunk Info Type: unrecognized type");
 	}
@@ -79,22 +84,23 @@ void ChunkConstantInfo::CommitAppend(transaction_t commit_id, idx_t start, idx_t
 	insert_id = commit_id;
 }
 
+bool ChunkConstantInfo::HasDeletes() const {
+	bool is_deleted = insert_id >= TRANSACTION_ID_START || delete_id < TRANSACTION_ID_START;
+	return is_deleted;
+}
+
 idx_t ChunkConstantInfo::GetCommittedDeletedCount(idx_t max_count) {
 	return delete_id < TRANSACTION_ID_START ? max_count : 0;
 }
 
-void ChunkConstantInfo::Serialize(Serializer &serializer) const {
-	bool is_deleted = insert_id >= TRANSACTION_ID_START || delete_id < TRANSACTION_ID_START;
-	if (!is_deleted) {
-		serializer.WriteProperty(100, "type", ChunkInfoType::EMPTY_INFO);
-		return;
-	}
-	serializer.WriteProperty(100, "type", type);
-	serializer.WriteProperty(200, "start", start);
+void ChunkConstantInfo::Write(WriteStream &writer) const {
+	D_ASSERT(HasDeletes());
+	ChunkInfo::Write(writer);
+	writer.Write<idx_t>(start);
 }
 
-unique_ptr<ChunkInfo> ChunkConstantInfo::Deserialize(Deserializer &deserializer) {
-	auto start = deserializer.ReadProperty<idx_t>(200, "start");
+unique_ptr<ChunkInfo> ChunkConstantInfo::Read(ReadStream &reader) {
+	auto start = reader.Read<idx_t>();
 	auto info = make_uniq<ChunkConstantInfo>(start);
 	info->insert_id = 0;
 	info->delete_id = 0;
@@ -218,6 +224,10 @@ void ChunkVectorInfo::CommitAppend(transaction_t commit_id, idx_t start, idx_t e
 	}
 }
 
+bool ChunkVectorInfo::HasDeletes() const {
+	return any_deleted;
+}
+
 idx_t ChunkVectorInfo::GetCommittedDeletedCount(idx_t max_count) {
 	if (!any_deleted) {
 		return 0;
@@ -231,45 +241,41 @@ idx_t ChunkVectorInfo::GetCommittedDeletedCount(idx_t max_count) {
 	return delete_count;
 }
 
-void ChunkVectorInfo::Serialize(Serializer &serializer) const {
+void ChunkVectorInfo::Write(WriteStream &writer) const {
 	SelectionVector sel(STANDARD_VECTOR_SIZE);
 	transaction_t start_time = TRANSACTION_ID_START - 1;
 	transaction_t transaction_id = DConstants::INVALID_INDEX;
 	idx_t count = GetSelVector(start_time, transaction_id, sel, STANDARD_VECTOR_SIZE);
 	if (count == STANDARD_VECTOR_SIZE) {
 		// nothing is deleted: skip writing anything
-		serializer.WriteProperty(100, "type", ChunkInfoType::EMPTY_INFO);
+		writer.Write<ChunkInfoType>(ChunkInfoType::EMPTY_INFO);
 		return;
 	}
 	if (count == 0) {
 		// everything is deleted: write a constant vector
-		serializer.WriteProperty(100, "type", ChunkInfoType::CONSTANT_INFO);
-		serializer.WriteProperty(200, "start", start);
+		writer.Write<ChunkInfoType>(ChunkInfoType::CONSTANT_INFO);
+		writer.Write<idx_t>(start);
 		return;
 	}
 	// write a boolean vector
-	serializer.WriteProperty(100, "type", ChunkInfoType::VECTOR_INFO);
-	serializer.WriteProperty(200, "start", start);
-	bool deleted_tuples[STANDARD_VECTOR_SIZE];
-	for (idx_t i = 0; i < STANDARD_VECTOR_SIZE; i++) {
-		deleted_tuples[i] = true;
-	}
+	ChunkInfo::Write(writer);
+	writer.Write<idx_t>(start);
+	ValidityMask mask(STANDARD_VECTOR_SIZE);
+	mask.Initialize(STANDARD_VECTOR_SIZE);
 	for (idx_t i = 0; i < count; i++) {
-		deleted_tuples[sel.get_index(i)] = false;
+		mask.SetInvalid(sel.get_index(i));
 	}
-	serializer.WriteProperty(201, "deleted_tuples", data_ptr_cast(deleted_tuples), sizeof(bool) * STANDARD_VECTOR_SIZE);
+	mask.Write(writer, STANDARD_VECTOR_SIZE);
 }
 
-unique_ptr<ChunkInfo> ChunkVectorInfo::Deserialize(Deserializer &deserializer) {
-	auto start = deserializer.ReadProperty<idx_t>(200, "start");
-
+unique_ptr<ChunkInfo> ChunkVectorInfo::Read(ReadStream &reader) {
+	auto start = reader.Read<idx_t>();
 	auto result = make_uniq<ChunkVectorInfo>(start);
 	result->any_deleted = true;
-	bool deleted_tuples[STANDARD_VECTOR_SIZE];
-	deserializer.ReadProperty(201, "deleted_tuples", data_ptr_cast(deleted_tuples),
-	                          sizeof(bool) * STANDARD_VECTOR_SIZE);
+	ValidityMask mask;
+	mask.Read(reader, STANDARD_VECTOR_SIZE);
 	for (idx_t i = 0; i < STANDARD_VECTOR_SIZE; i++) {
-		if (deleted_tuples[i]) {
+		if (mask.RowIsValid(i)) {
 			result->deleted[i] = 0;
 		}
 	}

package/src/duckdb/src/storage/table/column_data.cpp

@@ -87,7 +87,7 @@ void ColumnData::InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx)
 	state.last_offset = 0;
 }
 
-idx_t ColumnData::ScanVector(ColumnScanState &state, Vector &result, idx_t remaining) {
+idx_t ColumnData::ScanVector(ColumnScanState &state, Vector &result, idx_t remaining, bool has_updates) {
 	state.previous_states.clear();
 	if (state.version != version) {
 		InitializeScanWithOffset(state, state.row_index);
@@ -113,7 +113,8 @@ idx_t ColumnData::ScanVector(ColumnScanState &state, Vector &result, idx_t remai
 		idx_t scan_count = MinValue<idx_t>(remaining, state.current->start + state.current->count - state.row_index);
 		idx_t result_offset = initial_remaining - remaining;
 		if (scan_count > 0) {
-			state.current->Scan(state, scan_count, result, result_offset, scan_count == initial_remaining);
+			state.current->Scan(state, scan_count, result, result_offset,
+			                    !has_updates && scan_count == initial_remaining);
 
 			state.row_index += scan_count;
 			remaining -= scan_count;
@@ -138,10 +139,14 @@ idx_t ColumnData::ScanVector(ColumnScanState &state, Vector &result, idx_t remai
 
 template <bool SCAN_COMMITTED, bool ALLOW_UPDATES>
 idx_t ColumnData::ScanVector(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result) {
-	auto scan_count = ScanVector(state, result, STANDARD_VECTOR_SIZE);
-
-	lock_guard<mutex> update_guard(update_lock);
-	if (updates) {
+	bool has_updates;
+	{
+		lock_guard<mutex> update_guard(update_lock);
+		has_updates = updates ? true : false;
+	}
+	auto scan_count = ScanVector(state, result, STANDARD_VECTOR_SIZE, has_updates);
+	if (has_updates) {
+		lock_guard<mutex> update_guard(update_lock);
 		if (!ALLOW_UPDATES && updates->HasUncommittedUpdates(vector_index)) {
 			throw TransactionException("Cannot create index with outstanding updates");
 		}
@@ -179,7 +184,7 @@ idx_t ColumnData::ScanCommitted(idx_t vector_index, ColumnScanState &state, Vect
 void ColumnData::ScanCommittedRange(idx_t row_group_start, idx_t offset_in_row_group, idx_t count, Vector &result) {
 	ColumnScanState child_state;
 	InitializeScanWithOffset(child_state, row_group_start + offset_in_row_group);
-	auto scan_count = ScanVector(child_state, result, count);
+	auto scan_count = ScanVector(child_state, result, count, updates ? true : false);
 	if (updates) {
 		result.Flatten(scan_count);
 		updates->FetchCommittedRange(offset_in_row_group, count, result);
@@ -192,7 +197,7 @@ idx_t ColumnData::ScanCount(ColumnScanState &state, Vector &result, idx_t count)
 	}
 	// ScanCount can only be used if there are no updates
 	D_ASSERT(!updates);
-	return ScanVector(state, result, count);
+	return ScanVector(state, result, count, false);
 }
 
 void ColumnData::Select(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
@@ -339,7 +344,7 @@ idx_t ColumnData::Fetch(ColumnScanState &state, row_t row_id, Vector &result) {
 	state.row_index = start + ((row_id - start) / STANDARD_VECTOR_SIZE * STANDARD_VECTOR_SIZE);
 	state.current = data.GetSegment(state.row_index);
 	state.internal_index = state.current->start;
-	return ScanVector(state, result, STANDARD_VECTOR_SIZE);
+	return ScanVector(state, result, STANDARD_VECTOR_SIZE, false);
 }
 
 void ColumnData::FetchRow(TransactionData transaction, ColumnFetchState &state, row_t row_id, Vector &result,

package/src/duckdb/src/storage/table/list_column_data.cpp

@@ -86,7 +86,7 @@ idx_t ListColumnData::ScanCount(ColumnScanState &state, Vector &result, idx_t co
 	D_ASSERT(!updates);
 
 	Vector offset_vector(LogicalType::UBIGINT, count);
-	idx_t scan_count = ScanVector(state, offset_vector, count);
+	idx_t scan_count = ScanVector(state, offset_vector, count, false);
 	D_ASSERT(scan_count > 0);
 	validity.ScanCount(state.child_states[0], result, count);
 
@@ -132,7 +132,7 @@ void ListColumnData::Skip(ColumnScanState &state, idx_t count) {
 	// note that we only need to read the first and last entry
 	// however, let's just read all "count" entries for now
 	Vector result(LogicalType::UBIGINT, count);
-	idx_t scan_count = ScanVector(state, result, count);
+	idx_t scan_count = ScanVector(state, result, count, false);
 	if (scan_count == 0) {
 		return;
 	}