duckdb 0.8.2-dev4314.0 → 0.8.2-dev4424.0

package/src/duckdb/src/storage/table/row_version_manager.cpp

@@ -0,0 +1,228 @@
+#include "duckdb/storage/table/row_version_manager.hpp"
+#include "duckdb/transaction/transaction_data.hpp"
+#include "duckdb/storage/metadata/metadata_manager.hpp"
+#include "duckdb/storage/metadata/metadata_reader.hpp"
+#include "duckdb/storage/metadata/metadata_writer.hpp"
+#include "duckdb/common/pair.hpp"
+
+namespace duckdb {
+
+RowVersionManager::RowVersionManager(idx_t start) : start(start), has_changes(false) {
+}
+
+void RowVersionManager::SetStart(idx_t new_start) {
+	lock_guard<mutex> l(version_lock);
+	this->start = new_start;
+	idx_t current_start = start;
+	for (idx_t i = 0; i < Storage::ROW_GROUP_VECTOR_COUNT; i++) {
+		if (vector_info[i]) {
+			vector_info[i]->start = current_start;
+		}
+		current_start += STANDARD_VECTOR_SIZE;
+	}
+}
+
+idx_t RowVersionManager::GetCommittedDeletedCount(idx_t count) {
+	lock_guard<mutex> l(version_lock);
+	idx_t deleted_count = 0;
+	for (idx_t r = 0, i = 0; r < count; r += STANDARD_VECTOR_SIZE, i++) {
+		if (!vector_info[i]) {
+			continue;
+		}
+		idx_t max_count = MinValue<idx_t>(STANDARD_VECTOR_SIZE, count - r);
+		if (max_count == 0) {
+			break;
+		}
+		deleted_count += vector_info[i]->GetCommittedDeletedCount(max_count);
+	}
+	return deleted_count;
+}
+
+optional_ptr<ChunkInfo> RowVersionManager::GetChunkInfo(idx_t vector_idx) {
+	return vector_info[vector_idx].get();
+}
+
+idx_t RowVersionManager::GetSelVector(TransactionData transaction, idx_t vector_idx, SelectionVector &sel_vector,
+                                      idx_t max_count) {
+	lock_guard<mutex> l(version_lock);
+	auto chunk_info = GetChunkInfo(vector_idx);
+	if (!chunk_info) {
+		return max_count;
+	}
+	return chunk_info->GetSelVector(transaction, sel_vector, max_count);
+}
+
+idx_t RowVersionManager::GetCommittedSelVector(transaction_t start_time, transaction_t transaction_id, idx_t vector_idx,
+                                               SelectionVector &sel_vector, idx_t max_count) {
+	lock_guard<mutex> l(version_lock);
+	auto info = GetChunkInfo(vector_idx);
+	if (!info) {
+		return max_count;
+	}
+	return info->GetCommittedSelVector(start_time, transaction_id, sel_vector, max_count);
+}
+
+bool RowVersionManager::Fetch(TransactionData transaction, idx_t row) {
+	lock_guard<mutex> lock(version_lock);
+	idx_t vector_index = row / STANDARD_VECTOR_SIZE;
+	auto info = GetChunkInfo(vector_index);
+	if (!info) {
+		return true;
+	}
+	return info->Fetch(transaction, row - vector_index * STANDARD_VECTOR_SIZE);
+}
+
+void RowVersionManager::AppendVersionInfo(TransactionData transaction, idx_t count, idx_t row_group_start,
+                                          idx_t row_group_end) {
+	lock_guard<mutex> lock(version_lock);
+	has_changes = true;
+	idx_t start_vector_idx = row_group_start / STANDARD_VECTOR_SIZE;
+	idx_t end_vector_idx = (row_group_end - 1) / STANDARD_VECTOR_SIZE;
+	for (idx_t vector_idx = start_vector_idx; vector_idx <= end_vector_idx; vector_idx++) {
+		idx_t vector_start =
+		    vector_idx == start_vector_idx ? row_group_start - start_vector_idx * STANDARD_VECTOR_SIZE : 0;
+		idx_t vector_end =
+		    vector_idx == end_vector_idx ? row_group_end - end_vector_idx * STANDARD_VECTOR_SIZE : STANDARD_VECTOR_SIZE;
+		if (vector_start == 0 && vector_end == STANDARD_VECTOR_SIZE) {
+			// entire vector is encapsulated by append: append a single constant
+			auto constant_info = make_uniq<ChunkConstantInfo>(start + vector_idx * STANDARD_VECTOR_SIZE);
+			constant_info->insert_id = transaction.transaction_id;
+			constant_info->delete_id = NOT_DELETED_ID;
+			vector_info[vector_idx] = std::move(constant_info);
+		} else {
+			// part of a vector is encapsulated: append to that part
+			optional_ptr<ChunkVectorInfo> new_info;
+			if (!vector_info[vector_idx]) {
+				// first time appending to this vector: create new info
+				auto insert_info = make_uniq<ChunkVectorInfo>(start + vector_idx * STANDARD_VECTOR_SIZE);
+				new_info = insert_info.get();
+				vector_info[vector_idx] = std::move(insert_info);
+			} else if (vector_info[vector_idx]->type == ChunkInfoType::VECTOR_INFO) {
+				// use existing vector
+				new_info = &vector_info[vector_idx]->Cast<ChunkVectorInfo>();
+			} else {
+				throw InternalException("Error in RowVersionManager::AppendVersionInfo - expected either a "
+				                        "ChunkVectorInfo or no version info");
+			}
+			new_info->Append(vector_start, vector_end, transaction.transaction_id);
+		}
+	}
+}
+
+void RowVersionManager::CommitAppend(transaction_t commit_id, idx_t row_group_start, idx_t count) {
+	idx_t row_group_end = row_group_start + count;
+
+	lock_guard<mutex> lock(version_lock);
+	idx_t start_vector_idx = row_group_start / STANDARD_VECTOR_SIZE;
+	idx_t end_vector_idx = (row_group_end - 1) / STANDARD_VECTOR_SIZE;
+	for (idx_t vector_idx = start_vector_idx; vector_idx <= end_vector_idx; vector_idx++) {
+		idx_t vstart = vector_idx == start_vector_idx ? row_group_start - start_vector_idx * STANDARD_VECTOR_SIZE : 0;
+		idx_t vend =
+		    vector_idx == end_vector_idx ? row_group_end - end_vector_idx * STANDARD_VECTOR_SIZE : STANDARD_VECTOR_SIZE;
+
+		auto info = vector_info[vector_idx].get();
+		info->CommitAppend(commit_id, vstart, vend);
+	}
+}
+
+void RowVersionManager::RevertAppend(idx_t start_row) {
+	lock_guard<mutex> lock(version_lock);
+	idx_t start_vector_idx = (start_row + (STANDARD_VECTOR_SIZE - 1)) / STANDARD_VECTOR_SIZE;
+	for (idx_t vector_idx = start_vector_idx; vector_idx < Storage::ROW_GROUP_VECTOR_COUNT; vector_idx++) {
+		vector_info[vector_idx].reset();
+	}
+}
+
+ChunkVectorInfo &RowVersionManager::GetVectorInfo(idx_t vector_idx) {
+	if (!vector_info[vector_idx]) {
+		// no info yet: create it
+		vector_info[vector_idx] = make_uniq<ChunkVectorInfo>(start + vector_idx * STANDARD_VECTOR_SIZE);
+	} else if (vector_info[vector_idx]->type == ChunkInfoType::CONSTANT_INFO) {
+		auto &constant = vector_info[vector_idx]->Cast<ChunkConstantInfo>();
+		// info exists but it's a constant info: convert to a vector info
+		auto new_info = make_uniq<ChunkVectorInfo>(start + vector_idx * STANDARD_VECTOR_SIZE);
+		new_info->insert_id = constant.insert_id;
+		for (idx_t i = 0; i < STANDARD_VECTOR_SIZE; i++) {
+			new_info->inserted[i] = constant.insert_id;
+		}
+		vector_info[vector_idx] = std::move(new_info);
+	}
+	D_ASSERT(vector_info[vector_idx]->type == ChunkInfoType::VECTOR_INFO);
+	return vector_info[vector_idx]->Cast<ChunkVectorInfo>();
+}
+
+idx_t RowVersionManager::DeleteRows(idx_t vector_idx, transaction_t transaction_id, row_t rows[], idx_t count) {
+	lock_guard<mutex> lock(version_lock);
+	has_changes = true;
+	return GetVectorInfo(vector_idx).Delete(transaction_id, rows, count);
+}
+
+void RowVersionManager::CommitDelete(idx_t vector_idx, transaction_t commit_id, row_t rows[], idx_t count) {
+	lock_guard<mutex> lock(version_lock);
+	has_changes = true;
+	GetVectorInfo(vector_idx).CommitDelete(commit_id, rows, count);
+}
+
+vector<MetaBlockPointer> RowVersionManager::Checkpoint(MetadataManager &manager) {
+	if (!has_changes && !storage_pointers.empty()) {
+		// the row version manager already exists on disk and no changes were made
+		// we can write the current pointer as-is
+		// ensure the blocks we are pointing to are not marked as free
+		manager.ClearModifiedBlocks(storage_pointers);
+		// return the root pointer
+		return storage_pointers;
+	}
+	// first count how many ChunkInfo's we need to deserialize
+	vector<pair<idx_t, reference<ChunkInfo>>> to_serialize;
+	for (idx_t vector_idx = 0; vector_idx < Storage::ROW_GROUP_VECTOR_COUNT; vector_idx++) {
+		auto chunk_info = vector_info[vector_idx].get();
+		if (!chunk_info) {
+			continue;
+		}
+		if (!chunk_info->HasDeletes()) {
+			continue;
+		}
+		to_serialize.emplace_back(vector_idx, *chunk_info);
+	}
+	if (to_serialize.empty()) {
+		return vector<MetaBlockPointer>();
+	}
+
+	storage_pointers.clear();
+
+	MetadataWriter writer(manager, &storage_pointers);
+	// now serialize the actual version information
+	writer.Write<idx_t>(to_serialize.size());
+	for (auto &entry : to_serialize) {
+		auto &vector_idx = entry.first;
+		auto &chunk_info = entry.second.get();
+		writer.Write<idx_t>(vector_idx);
+		chunk_info.Write(writer);
+	}
+	writer.Flush();
+
+	has_changes = false;
+	return storage_pointers;
+}
+
+shared_ptr<RowVersionManager> RowVersionManager::Deserialize(MetaBlockPointer delete_pointer, MetadataManager &manager,
+                                                             idx_t start) {
+	if (!delete_pointer.IsValid()) {
+		return nullptr;
+	}
+	auto version_info = make_shared<RowVersionManager>(start);
+	MetadataReader source(manager, delete_pointer, &version_info->storage_pointers);
+	auto chunk_count = source.Read<idx_t>();
+	D_ASSERT(chunk_count > 0);
+	for (idx_t i = 0; i < chunk_count; i++) {
+		idx_t vector_index = source.Read<idx_t>();
+		if (vector_index >= Storage::ROW_GROUP_VECTOR_COUNT) {
+			throw Exception("In DeserializeDeletes, vector_index is out of range for the row group. Corrupted file?");
+		}
+		version_info->vector_info[vector_index] = ChunkInfo::Read(source);
+	}
+	version_info->has_changes = false;
+	return version_info;
+}
+
+} // namespace duckdb

package/src/duckdb/src/storage/table/update_segment.cpp

@@ -328,7 +328,7 @@ void UpdateSegment::FetchCommittedRange(idx_t start_row, idx_t count, Vector &re
 	idx_t start_vector = start_row / STANDARD_VECTOR_SIZE;
 	idx_t end_vector = (end_row - 1) / STANDARD_VECTOR_SIZE;
 	D_ASSERT(start_vector <= end_vector);
-	D_ASSERT(end_vector < RowGroup::ROW_GROUP_VECTOR_COUNT);
+	D_ASSERT(end_vector < Storage::ROW_GROUP_VECTOR_COUNT);
 
 	for (idx_t vector_idx = start_vector; vector_idx <= end_vector; vector_idx++) {
 		if (!root->info[vector_idx]) {
@@ -1089,7 +1089,7 @@ void UpdateSegment::Update(TransactionData transaction, idx_t column_index, Vect
 	idx_t vector_offset = column_data.start + vector_index * STANDARD_VECTOR_SIZE;
 
 	D_ASSERT(idx_t(first_id) >= column_data.start);
-	D_ASSERT(vector_index < RowGroup::ROW_GROUP_VECTOR_COUNT);
+	D_ASSERT(vector_index < Storage::ROW_GROUP_VECTOR_COUNT);
 
 	// first check the version chain
 	UpdateInfo *node = nullptr;

package/src/duckdb/src/transaction/cleanup_state.cpp

@@ -8,6 +8,7 @@
 #include "duckdb/catalog/dependency_manager.hpp"
 #include "duckdb/storage/table/chunk_info.hpp"
 #include "duckdb/storage/table/update_segment.hpp"
+#include "duckdb/storage/table/row_version_manager.hpp"
 
 namespace duckdb {
 
@@ -69,7 +70,7 @@ void CleanupState::CleanupDelete(DeleteInfo &info) {
 
 	count = 0;
 	for (idx_t i = 0; i < info.count; i++) {
-		row_numbers[count++] = info.vinfo->start + info.rows[i];
+		row_numbers[count++] = info.base_row + info.rows[i];
 	}
 	Flush();
 }

package/src/duckdb/src/transaction/commit_state.cpp

@@ -16,13 +16,14 @@
 #include "duckdb/transaction/update_info.hpp"
 #include "duckdb/catalog/catalog_entry/scalar_macro_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_entry/view_catalog_entry.hpp"
+#include "duckdb/storage/table/row_version_manager.hpp"
 #include "duckdb/common/serializer/binary_deserializer.hpp"
 #include "duckdb/common/serializer/memory_stream.hpp"
 
 namespace duckdb {
 
-CommitState::CommitState(ClientContext &context, transaction_t commit_id, optional_ptr<WriteAheadLog> log)
-    : log(log), commit_id(commit_id), current_table_info(nullptr), context(context) {
+CommitState::CommitState(transaction_t commit_id, optional_ptr<WriteAheadLog> log)
+    : log(log), commit_id(commit_id), current_table_info(nullptr) {
 }
 
 void CommitState::SwitchTable(DataTableInfo *table_info, UndoFlags new_op) {
@@ -280,7 +281,7 @@ void CommitState::CommitEntry(UndoFlags type, data_ptr_t data) {
 			WriteDelete(*info);
 		}
 		// mark the tuples as committed
-		info->vinfo->CommitDelete(commit_id, info->rows, info->count);
+		info->version_info->CommitDelete(info->vector_idx, commit_id, info->rows, info->count);
 		break;
 	}
 	case UndoFlags::UPDATE_TUPLE: {
@@ -321,7 +322,7 @@ void CommitState::RevertCommit(UndoFlags type, data_ptr_t data) {
 		auto info = reinterpret_cast<DeleteInfo *>(data);
 		info->table->info->cardinality += info->count;
 		// revert the commit by writing the (uncommitted) transaction_id back into the version info
-		info->vinfo->CommitDelete(transaction_id, info->rows, info->count);
+		info->version_info->CommitDelete(info->vector_idx, transaction_id, info->rows, info->count);
 		break;
 	}
 	case UndoFlags::UPDATE_TUPLE: {

package/src/duckdb/src/transaction/duck_transaction.cpp

@@ -70,10 +70,12 @@ void DuckTransaction::PushCatalogEntry(CatalogEntry &entry, data_ptr_t extra_dat
 	}
 }
 
-void DuckTransaction::PushDelete(DataTable &table, ChunkVectorInfo *vinfo, row_t rows[], idx_t count, idx_t base_row) {
+void DuckTransaction::PushDelete(DataTable &table, RowVersionManager &info, idx_t vector_idx, row_t rows[], idx_t count,
+                                 idx_t base_row) {
 	auto delete_info = reinterpret_cast<DeleteInfo *>(
 	    undo_buffer.CreateEntry(UndoFlags::DELETE_TUPLE, sizeof(DeleteInfo) + sizeof(row_t) * count));
-	delete_info->vinfo = vinfo;
+	delete_info->version_info = &info;
+	delete_info->vector_idx = vector_idx;
 	delete_info->table = &table;
 	delete_info->count = count;
 	delete_info->base_row = base_row;

package/src/duckdb/src/transaction/rollback_state.cpp

@@ -9,6 +9,7 @@
 #include "duckdb/catalog/catalog_set.hpp"
 #include "duckdb/storage/data_table.hpp"
 #include "duckdb/storage/table/update_segment.hpp"
+#include "duckdb/storage/table/row_version_manager.hpp"
 
 namespace duckdb {
 
@@ -30,7 +31,7 @@ void RollbackState::RollbackEntry(UndoFlags type, data_ptr_t data) {
 	case UndoFlags::DELETE_TUPLE: {
 		auto info = reinterpret_cast<DeleteInfo *>(data);
 		// reset the deleted flag on rollback
-		info->vinfo->CommitDelete(NOT_DELETED_ID, info->rows, info->count);
+		info->version_info->CommitDelete(info->vector_idx, NOT_DELETED_ID, info->rows, info->count);
 		break;
 	}
 	case UndoFlags::UPDATE_TUPLE: {

package/src/duckdb/src/transaction/undo_buffer.cpp

@@ -11,12 +11,10 @@
 #include "duckdb/transaction/rollback_state.hpp"
 #include "duckdb/common/pair.hpp"
 
-#include <unordered_map>
-
 namespace duckdb {
 constexpr uint32_t UNDO_ENTRY_HEADER_SIZE = sizeof(UndoFlags) + sizeof(uint32_t);
 
-UndoBuffer::UndoBuffer(ClientContext &context_p) : context(context_p), allocator(BufferAllocator::Get(context_p)) {
+UndoBuffer::UndoBuffer(ClientContext &context_p) : allocator(BufferAllocator::Get(context_p)) {
 }
 
 data_ptr_t UndoBuffer::CreateEntry(UndoFlags type, idx_t len) {
@@ -138,7 +136,7 @@ void UndoBuffer::Cleanup() {
 
 void UndoBuffer::Commit(UndoBuffer::IteratorState &iterator_state, optional_ptr<WriteAheadLog> log,
                         transaction_t commit_id) {
-	CommitState state(context, commit_id, log);
+	CommitState state(commit_id, log);
 	if (log) {
 		// commit WITH write ahead log
 		IterateEntries(iterator_state, [&](UndoFlags type, data_ptr_t data) { state.CommitEntry<true>(type, data); });
@@ -149,7 +147,7 @@ void UndoBuffer::Commit(UndoBuffer::IteratorState &iterator_state, optional_ptr<
 }
 
 void UndoBuffer::RevertCommit(UndoBuffer::IteratorState &end_state, transaction_t transaction_id) {
-	CommitState state(context, transaction_id, nullptr);
+	CommitState state(transaction_id, nullptr);
 	UndoBuffer::IteratorState start_state;
 	IterateEntries(start_state, end_state, [&](UndoFlags type, data_ptr_t data) { state.RevertCommit(type, data); });
 }

package/src/duckdb/ub_src_storage_table.cpp

@@ -18,6 +18,8 @@
 
 #include "src/storage/table/row_group_collection.cpp"
 
+#include "src/storage/table/row_version_manager.cpp"
+
 #include "src/storage/table/scan_state.cpp"
 
 #include "src/storage/table/standard_column_data.cpp"

package/test/prepare.test.ts

@@ -652,7 +652,16 @@ describe('prepare', function() {
             });
             it("should aggregate kurtosis(num)", function (done) {
                 db.all("SELECT kurtosis(num) as kurtosis FROM foo", function (err: null | Error, res: TableData) {
-                    assert.equal(res[0].kurtosis, -1.1999999999999997);
+                    // The `num` column of table `foo` contains each integer from 0 to 999,999 exactly once.
+                    // This is a uniform distribution. The excess kurtosis for a uniform distribution is exactly -1.2.
+                    // See https://en.wikipedia.org/wiki/Kurtosis#Other_well-known_distributions
+                    const expected = -1.2;
+                    
+                    // The calculated value can differ from the exact answer by small amounts on different platforms due
+                    // to floating-point errors. This tolerance was determined experimentally.
+                    const tolerance = Number.EPSILON * 10;
+
+                    assert.ok(Math.abs(res[0].kurtosis - expected) < tolerance);
                     done(err);
                 });
             });

package/test/test_all_types.test.ts

@@ -90,7 +90,7 @@ const correct_answer_map: Record<string, any[]> = {
   date_array: [
     [],
     [
-      new Date(1970, 0, 1),
+      new Date(Date.UTC(1970, 0, 1)),
       null,
       new Date("0001-01-01T00:00:00.000Z"),
       new Date("9999-12-31T00:00:00.000Z"),
@@ -100,7 +100,7 @@ const correct_answer_map: Record<string, any[]> = {
   timestamp_array: [
     [],
     [
-      new Date(1970, 0, 1),
+      new Date(Date.UTC(1970, 0, 1)),
       null,
       new Date("0001-01-01T00:00:00.000Z"),
       new Date("9999-12-31T23:59:59.999Z"),
@@ -111,7 +111,7 @@ const correct_answer_map: Record<string, any[]> = {
   timestamptz_array: [
     [],
     [
-      new Date(1970, 0, 1),
+      new Date(Date.UTC(1970, 0, 1)),
       null,
       new Date("0001-01-01T00:00:00.000Z"),
       new Date("9999-12-31T23:59:59.999Z"),
@@ -171,7 +171,7 @@ const correct_answer_map: Record<string, any[]> = {
   ],
 
   timestamp: [
-    new Date("1990-01-01T00:00"),
+    new Date(Date.UTC(1990, 0, 1)),
     new Date("9999-12-31T23:59:59.000Z"),
     null,
   ],