duckdb 0.6.2-dev971.0 → 0.7.1-dev2.0

package/src/duckdb/src/storage/table/update_segment.cpp

@@ -5,7 +5,7 @@
 #include "duckdb/storage/statistics/string_statistics.hpp"
 #include "duckdb/storage/statistics/validity_statistics.hpp"
 #include "duckdb/storage/table/column_data.hpp"
-#include "duckdb/transaction/transaction.hpp"
+#include "duckdb/transaction/duck_transaction.hpp"
 #include "duckdb/transaction/update_info.hpp"
 
 namespace duckdb {
@@ -616,7 +616,7 @@ struct UpdateSelectElement {
 
 template <>
 string_t UpdateSelectElement::Operation(UpdateSegment *segment, string_t element) {
-	return element.IsInlined() ? element : segment->GetStringHeap().AddString(element);
+	return element.IsInlined() ? element : segment->GetStringHeap().AddBlob(element);
 }
 
 template <class T>
@@ -942,7 +942,7 @@ idx_t UpdateStringStatistics(UpdateSegment *segment, SegmentStatistics &stats, V
 		for (idx_t i = 0; i < count; i++) {
 			((StringStatistics &)*stats.statistics).Update(update_data[i]);
 			if (!update_data[i].IsInlined()) {
-				update_data[i] = segment->GetStringHeap().AddString(update_data[i]);
+				update_data[i] = segment->GetStringHeap().AddBlob(update_data[i]);
 			}
 		}
 		sel.Initialize(nullptr);
@@ -955,7 +955,7 @@ idx_t UpdateStringStatistics(UpdateSegment *segment, SegmentStatistics &stats, V
 				sel.set_index(not_null_count++, i);
 				((StringStatistics &)*stats.statistics).Update(update_data[i]);
 				if (!update_data[i].IsInlined()) {
-					update_data[i] = segment->GetStringHeap().AddString(update_data[i]);
+					update_data[i] = segment->GetStringHeap().AddBlob(update_data[i]);
 				}
 			}
 		}
@@ -1118,7 +1118,8 @@ void UpdateSegment::Update(TransactionData transaction, idx_t column_index, Vect
 		if (!node) {
 			// no updates made yet by this transaction: initially the update info to empty
 			if (transaction.transaction) {
-				node = transaction.transaction->CreateUpdateInfo(type_size, count);
+				auto &dtransaction = (DuckTransaction &)*transaction.transaction;
+				node = dtransaction.CreateUpdateInfo(type_size, count);
 			} else {
 				node = CreateEmptyUpdateInfo(transaction, type_size, count, update_info_data);
 			}
@@ -1179,7 +1180,7 @@ void UpdateSegment::Update(TransactionData transaction, idx_t column_index, Vect
 		transaction_node->Verify();
 		result->info->Verify();
 
-		root->info[vector_index] = move(result);
+		root->info[vector_index] = std::move(result);
 	}
 }
 

package/src/duckdb/src/storage/table_index_list.cpp

@@ -1,11 +1,13 @@
 #include "duckdb/storage/table/table_index_list.hpp"
 #include "duckdb/storage/data_table.hpp"
+#include "duckdb/common/types/conflict_manager.hpp"
+#include "duckdb/execution/index/art/art.hpp"
 
 namespace duckdb {
 void TableIndexList::AddIndex(unique_ptr<Index> index) {
 	D_ASSERT(index);
 	lock_guard<mutex> lock(indexes_lock);
-	indexes.push_back(move(index));
+	indexes.push_back(std::move(index));
 }
 void TableIndexList::RemoveIndex(Index *index) {
 	D_ASSERT(index);
@@ -32,7 +34,7 @@ idx_t TableIndexList::Count() {
 
 void TableIndexList::Move(TableIndexList &other) {
 	D_ASSERT(indexes.empty());
-	indexes = move(other.indexes);
+	indexes = std::move(other.indexes);
 }
 
 Index *TableIndexList::FindForeignKeyIndex(const vector<PhysicalIndex> &fk_keys, ForeignKeyType fk_type) {
@@ -46,20 +48,20 @@ Index *TableIndexList::FindForeignKeyIndex(const vector<PhysicalIndex> &fk_keys,
 	return result;
 }
 
-void TableIndexList::VerifyForeignKey(const vector<PhysicalIndex> &fk_keys, bool is_append, DataChunk &chunk,
-                                      vector<string> &err_msgs) {
-	auto fk_type = is_append ? ForeignKeyType::FK_TYPE_PRIMARY_KEY_TABLE : ForeignKeyType::FK_TYPE_FOREIGN_KEY_TABLE;
+void TableIndexList::VerifyForeignKey(const vector<PhysicalIndex> &fk_keys, DataChunk &chunk,
+                                      ConflictManager &conflict_manager) {
+	auto fk_type = conflict_manager.LookupType() == VerifyExistenceType::APPEND_FK
+	                   ? ForeignKeyType::FK_TYPE_PRIMARY_KEY_TABLE
+	                   : ForeignKeyType::FK_TYPE_FOREIGN_KEY_TABLE;
 
 	// check whether or not the chunk can be inserted or deleted into the referenced table' storage
 	auto index = FindForeignKeyIndex(fk_keys, fk_type);
 	if (!index) {
 		throw InternalException("Internal Foreign Key error: could not find index to verify...");
 	}
-	if (is_append) {
-		index->VerifyAppendForeignKey(chunk, err_msgs.data());
-	} else {
-		index->VerifyDeleteForeignKey(chunk, err_msgs.data());
-	}
+	conflict_manager.SetIndexCount(1);
+
+	index->LookupValues(chunk, conflict_manager);
 }
 
 vector<column_t> TableIndexList::GetRequiredColumns() {

package/src/duckdb/src/storage/wal_replay.cpp

@@ -191,7 +191,7 @@ void ReplayState::ReplayCreateTable() {
 
 	// bind the constraints to the table again
 	auto binder = Binder::CreateBinder(context);
-	auto bound_info = binder->BindCreateTableInfo(move(info));
+	auto bound_info = binder->BindCreateTableInfo(std::move(info));
 
 	catalog.CreateTable(context, bound_info.get());
 }
@@ -402,7 +402,7 @@ void ReplayState::ReplayInsert() {
 	}
 
 	// append to the current table
-	current_table->storage->LocalAppend(*current_table, context, chunk);
+	current_table->GetStorage().LocalAppend(*current_table, context, chunk);
 }
 
 void ReplayState::ReplayDelete() {
@@ -423,7 +423,7 @@ void ReplayState::ReplayDelete() {
 	// delete the tuples from the current table
 	for (idx_t i = 0; i < chunk.size(); i++) {
 		row_ids[0] = source_ids[i];
-		current_table->storage->Delete(*current_table, context, row_identifiers, 1);
+		current_table->GetStorage().Delete(*current_table, context, row_identifiers, 1);
 	}
 }
 
@@ -443,16 +443,16 @@ void ReplayState::ReplayUpdate() {
 		throw InternalException("Corrupt WAL: update without table");
 	}
 
-	if (column_path[0] >= current_table->columns.PhysicalColumnCount()) {
+	if (column_path[0] >= current_table->GetColumns().PhysicalColumnCount()) {
 		throw InternalException("Corrupt WAL: column index for update out of bounds");
 	}
 
 	// remove the row id vector from the chunk
-	auto row_ids = move(chunk.data.back());
+	auto row_ids = std::move(chunk.data.back());
 	chunk.data.pop_back();
 
 	// now perform the update
-	current_table->storage->UpdateColumn(*current_table, context, row_ids, column_path, chunk);
+	current_table->GetStorage().UpdateColumn(*current_table, context, row_ids, column_path, chunk);
 }
 
 void ReplayState::ReplayCheckpoint() {

package/src/duckdb/src/transaction/commit_state.cpp

@@ -1,7 +1,9 @@
 #include "duckdb/transaction/commit_state.hpp"
 
+#include "duckdb/catalog/catalog_entry/duck_table_entry.hpp"
 #include "duckdb/catalog/catalog_entry/type_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_set.hpp"
+#include "duckdb/catalog/duck_catalog.hpp"
 #include "duckdb/common/serializer/buffered_deserializer.hpp"
 #include "duckdb/parser/parsed_data/alter_table_info.hpp"
 #include "duckdb/storage/data_table.hpp"
@@ -39,7 +41,8 @@ void CommitState::WriteCatalogEntry(CatalogEntry *entry, data_ptr_t dataptr) {
 	switch (parent->type) {
 	case CatalogType::TABLE_ENTRY:
 		if (entry->type == CatalogType::TABLE_ENTRY) {
-			auto table_entry = (TableCatalogEntry *)entry;
+			auto table_entry = (DuckTableEntry *)entry;
+			D_ASSERT(table_entry->IsDuckTable());
 			// ALTER TABLE statement, read the extra data after the entry
 			auto extra_data_size = Load<idx_t>(dataptr);
 			auto extra_data = (data_ptr_t)(dataptr + sizeof(idx_t));
@@ -91,7 +94,8 @@ void CommitState::WriteCatalogEntry(CatalogEntry *entry, data_ptr_t dataptr) {
 	case CatalogType::DELETED_ENTRY:
 		switch (entry->type) {
 		case CatalogType::TABLE_ENTRY: {
-			auto table_entry = (TableCatalogEntry *)entry;
+			auto table_entry = (DuckTableEntry *)entry;
+			D_ASSERT(table_entry->IsDuckTable());
 			table_entry->CommitDrop();
 			log->WriteDropTable(table_entry);
 			break;
@@ -220,9 +224,11 @@ void CommitState::CommitEntry(UndoFlags type, data_ptr_t data) {
 
 		auto &catalog = catalog_entry->catalog;
 		D_ASSERT(catalog);
+		D_ASSERT(catalog->IsDuckCatalog());
 
 		// Grab a write lock on the catalog
-		lock_guard<mutex> write_lock(catalog->write_lock);
+		auto &duck_catalog = (DuckCatalog &)*catalog;
+		lock_guard<mutex> write_lock(duck_catalog.GetWriteLock());
 		catalog_entry->set->UpdateTimestamp(catalog_entry->parent, commit_id);
 		if (catalog_entry->name != catalog_entry->parent->name) {
 			catalog_entry->set->UpdateTimestamp(catalog_entry, commit_id);

package/src/duckdb/src/transaction/duck_transaction.cpp

@@ -0,0 +1,155 @@
+#include "duckdb/transaction/duck_transaction.hpp"
+
+#include "duckdb/main/client_context.hpp"
+#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
+#include "duckdb/common/exception.hpp"
+#include "duckdb/parser/column_definition.hpp"
+#include "duckdb/storage/data_table.hpp"
+#include "duckdb/storage/write_ahead_log.hpp"
+#include "duckdb/storage/storage_manager.hpp"
+
+#include "duckdb/transaction/append_info.hpp"
+#include "duckdb/transaction/delete_info.hpp"
+#include "duckdb/transaction/update_info.hpp"
+#include "duckdb/transaction/local_storage.hpp"
+#include "duckdb/main/config.hpp"
+#include "duckdb/storage/table/column_data.hpp"
+#include "duckdb/main/client_data.hpp"
+#include "duckdb/main/attached_database.hpp"
+
+namespace duckdb {
+
+TransactionData::TransactionData(DuckTransaction &transaction_p) // NOLINT
+    : transaction(&transaction_p), transaction_id(transaction_p.transaction_id), start_time(transaction_p.start_time) {
+}
+TransactionData::TransactionData(transaction_t transaction_id_p, transaction_t start_time_p)
+    : transaction(nullptr), transaction_id(transaction_id_p), start_time(start_time_p) {
+}
+
+DuckTransaction::DuckTransaction(TransactionManager &manager, ClientContext &context_p, transaction_t start_time,
+                                 transaction_t transaction_id)
+    : Transaction(manager, context_p), start_time(start_time), transaction_id(transaction_id), commit_id(0),
+      highest_active_query(0), undo_buffer(context_p), storage(make_unique<LocalStorage>(context_p, *this)) {
+}
+
+DuckTransaction::~DuckTransaction() {
+}
+
+DuckTransaction &DuckTransaction::Get(ClientContext &context, AttachedDatabase &db) {
+	return DuckTransaction::Get(context, db.GetCatalog());
+}
+
+DuckTransaction &DuckTransaction::Get(ClientContext &context, Catalog &catalog) {
+	auto &transaction = Transaction::Get(context, catalog);
+	if (!transaction.IsDuckTransaction()) {
+		throw InternalException("DuckTransaction::Get called on non-DuckDB transaction");
+	}
+	return (DuckTransaction &)transaction;
+}
+
+LocalStorage &DuckTransaction::GetLocalStorage() {
+	return *storage;
+}
+
+void DuckTransaction::PushCatalogEntry(CatalogEntry *entry, data_ptr_t extra_data, idx_t extra_data_size) {
+	idx_t alloc_size = sizeof(CatalogEntry *);
+	if (extra_data_size > 0) {
+		alloc_size += extra_data_size + sizeof(idx_t);
+	}
+	auto baseptr = undo_buffer.CreateEntry(UndoFlags::CATALOG_ENTRY, alloc_size);
+	// store the pointer to the catalog entry
+	Store<CatalogEntry *>(entry, baseptr);
+	if (extra_data_size > 0) {
+		// copy the extra data behind the catalog entry pointer (if any)
+		baseptr += sizeof(CatalogEntry *);
+		// first store the extra data size
+		Store<idx_t>(extra_data_size, baseptr);
+		baseptr += sizeof(idx_t);
+		// then copy over the actual data
+		memcpy(baseptr, extra_data, extra_data_size);
+	}
+}
+
+void DuckTransaction::PushDelete(DataTable *table, ChunkVectorInfo *vinfo, row_t rows[], idx_t count, idx_t base_row) {
+	auto delete_info =
+	    (DeleteInfo *)undo_buffer.CreateEntry(UndoFlags::DELETE_TUPLE, sizeof(DeleteInfo) + sizeof(row_t) * count);
+	delete_info->vinfo = vinfo;
+	delete_info->table = table;
+	delete_info->count = count;
+	delete_info->base_row = base_row;
+	memcpy(delete_info->rows, rows, sizeof(row_t) * count);
+}
+
+void DuckTransaction::PushAppend(DataTable *table, idx_t start_row, idx_t row_count) {
+	auto append_info = (AppendInfo *)undo_buffer.CreateEntry(UndoFlags::INSERT_TUPLE, sizeof(AppendInfo));
+	append_info->table = table;
+	append_info->start_row = start_row;
+	append_info->count = row_count;
+}
+
+UpdateInfo *DuckTransaction::CreateUpdateInfo(idx_t type_size, idx_t entries) {
+	auto update_info = (UpdateInfo *)undo_buffer.CreateEntry(
+	    UndoFlags::UPDATE_TUPLE, sizeof(UpdateInfo) + (sizeof(sel_t) + type_size) * STANDARD_VECTOR_SIZE);
+	update_info->max = STANDARD_VECTOR_SIZE;
+	update_info->tuples = (sel_t *)(((data_ptr_t)update_info) + sizeof(UpdateInfo));
+	update_info->tuple_data = ((data_ptr_t)update_info) + sizeof(UpdateInfo) + sizeof(sel_t) * update_info->max;
+	update_info->version_number = transaction_id;
+	return update_info;
+}
+
+bool DuckTransaction::ChangesMade() {
+	return undo_buffer.ChangesMade() || storage->ChangesMade();
+}
+
+bool DuckTransaction::AutomaticCheckpoint(AttachedDatabase &db) {
+	auto &storage_manager = db.GetStorageManager();
+	return storage_manager.AutomaticCheckpoint(storage->EstimatedSize() + undo_buffer.EstimatedSize());
+}
+
+string DuckTransaction::Commit(AttachedDatabase &db, transaction_t commit_id, bool checkpoint) noexcept {
+	// "checkpoint" parameter indicates if the caller will checkpoint. If checkpoint ==
+	//    true: Then this function will NOT write to the WAL or flush/persist.
+	//          This method only makes commit in memory, expecting caller to checkpoint/flush.
+	//    false: Then this function WILL write to the WAL and Flush/Persist it.
+	this->commit_id = commit_id;
+
+	UndoBuffer::IteratorState iterator_state;
+	LocalStorage::CommitState commit_state;
+	unique_ptr<StorageCommitState> storage_commit_state;
+	WriteAheadLog *log;
+	if (!db.IsSystem()) {
+		auto &storage_manager = db.GetStorageManager();
+		log = storage_manager.GetWriteAheadLog();
+		storage_commit_state = storage_manager.GenStorageCommitState(*this, checkpoint);
+	} else {
+		log = nullptr;
+	}
+	try {
+		storage->Commit(commit_state, *this);
+		undo_buffer.Commit(iterator_state, log, commit_id);
+		if (log) {
+			// commit any sequences that were used to the WAL
+			for (auto &entry : sequence_usage) {
+				log->WriteSequenceValue(entry.first, entry.second);
+			}
+		}
+		if (storage_commit_state) {
+			storage_commit_state->FlushCommit();
+		}
+		return string();
+	} catch (std::exception &ex) {
+		undo_buffer.RevertCommit(iterator_state, transaction_id);
+		return ex.what();
+	}
+}
+
+void DuckTransaction::Rollback() noexcept {
+	storage->Rollback();
+	undo_buffer.Rollback();
+}
+
+void DuckTransaction::Cleanup() {
+	undo_buffer.Cleanup();
+}
+
+} // namespace duckdb