duckdb 0.6.2-dev781.0 → 0.6.2-dev891.0

package/src/duckdb/src/transaction/meta_transaction.cpp

@@ -0,0 +1,106 @@
+#include "duckdb/transaction/meta_transaction.hpp"
+#include "duckdb/main/client_context.hpp"
+#include "duckdb/main/attached_database.hpp"
+#include "duckdb/transaction/transaction_manager.hpp"
+
+namespace duckdb {
+
+MetaTransaction::MetaTransaction(ClientContext &context_p, timestamp_t start_timestamp_p, idx_t catalog_version_p)
+    : context(context_p), start_timestamp(start_timestamp_p), catalog_version(catalog_version_p), read_only(true),
+      active_query(MAXIMUM_QUERY_ID), modified_database(nullptr) {
+}
+
+MetaTransaction &MetaTransaction::Get(ClientContext &context) {
+	return context.transaction.ActiveTransaction();
+}
+
+ValidChecker &ValidChecker::Get(MetaTransaction &transaction) {
+	return transaction.transaction_validity;
+}
+
+Transaction &Transaction::Get(ClientContext &context, AttachedDatabase &db) {
+	auto &meta_transaction = MetaTransaction::Get(context);
+	return meta_transaction.GetTransaction(&db);
+}
+
+Transaction &MetaTransaction::GetTransaction(AttachedDatabase *db) {
+	auto entry = transactions.find(db);
+	if (entry == transactions.end()) {
+		auto new_transaction = db->GetTransactionManager().StartTransaction(context);
+		new_transaction->active_query = active_query;
+		all_transactions.push_back(db);
+		transactions[db] = new_transaction;
+		return *new_transaction;
+	} else {
+		D_ASSERT(entry->second->active_query == active_query);
+		return *entry->second;
+	}
+}
+
+Transaction &Transaction::Get(ClientContext &context, Catalog &catalog) {
+	return Transaction::Get(context, catalog.GetAttached());
+}
+
+string MetaTransaction::Commit() {
+	string error;
+	// commit transactions in reverse order
+	for (idx_t i = all_transactions.size(); i > 0; i--) {
+		auto db = all_transactions[i - 1];
+		auto entry = transactions.find(db);
+		if (entry == transactions.end()) {
+			throw InternalException("Could not find transaction corresponding to database in MetaTransaction");
+		}
+		auto &transaction_manager = db->GetTransactionManager();
+		auto transaction = entry->second;
+		if (error.empty()) {
+			// commit
+			error = transaction_manager.CommitTransaction(context, transaction);
+		} else {
+			// we have encountered an error previously - roll back subsequent entries
+			transaction_manager.RollbackTransaction(transaction);
+		}
+	}
+	return error;
+}
+
+void MetaTransaction::Rollback() {
+	// rollback transactions in reverse order
+	for (idx_t i = all_transactions.size(); i > 0; i--) {
+		auto db = all_transactions[i - 1];
+		auto &transaction_manager = db->GetTransactionManager();
+		auto entry = transactions.find(db);
+		D_ASSERT(entry != transactions.end());
+		auto transaction = entry->second;
+		transaction_manager.RollbackTransaction(transaction);
+	}
+}
+
+idx_t MetaTransaction::GetActiveQuery() {
+	return active_query;
+}
+
+void MetaTransaction::SetActiveQuery(transaction_t query_number) {
+	active_query = query_number;
+	for (auto &entry : transactions) {
+		entry.second->active_query = query_number;
+	}
+}
+
+void MetaTransaction::ModifyDatabase(AttachedDatabase *db) {
+	if (db->IsSystem() || db->IsTemporary()) {
+		// we can always modify the system and temp databases
+		return;
+	}
+	if (!modified_database) {
+		modified_database = db;
+		return;
+	}
+	if (db != modified_database) {
+		throw TransactionException(
+		    "Attempting to write to database \"%s\" in a transaction that has already modified database \"%s\" - a "
+		    "single transaction can only write to a single attached database.",
+		    db->GetName(), modified_database->GetName());
+	}
+}
+
+} // namespace duckdb

package/src/duckdb/src/transaction/transaction.cpp

@@ -15,6 +15,7 @@
 #include "duckdb/main/config.hpp"
 #include "duckdb/storage/table/column_data.hpp"
 #include "duckdb/main/client_data.hpp"
+#include "duckdb/main/attached_database.hpp"
 
 #include <cstring>
 
@@ -27,21 +28,16 @@ TransactionData::TransactionData(transaction_t transaction_id_p, transaction_t s
     : transaction(nullptr), transaction_id(transaction_id_p), start_time(start_time_p) {
 }
 
-Transaction::Transaction(ClientContext &context_p, transaction_t start_time, transaction_t transaction_id,
-                         timestamp_t start_timestamp, idx_t catalog_version)
-    : context(context_p.shared_from_this()), start_time(start_time), transaction_id(transaction_id), commit_id(0),
-      highest_active_query(0), active_query(MAXIMUM_QUERY_ID), start_timestamp(start_timestamp),
-      catalog_version(catalog_version), temporary_objects(context_p.client_data->temporary_objects),
-      undo_buffer(context.lock()), storage(make_unique<LocalStorage>(context_p, *this)) {
+Transaction::Transaction(TransactionManager &manager_p, ClientContext &context_p, transaction_t start_time,
+                         transaction_t transaction_id)
+    : manager(manager_p), context(context_p.shared_from_this()), start_time(start_time), transaction_id(transaction_id),
+      commit_id(0), active_query(MAXIMUM_QUERY_ID), highest_active_query(0), undo_buffer(context_p),
+      storage(make_unique<LocalStorage>(context_p, *this)) {
 }
 
 Transaction::~Transaction() {
 }
 
-Transaction &Transaction::GetTransaction(ClientContext &context) {
-	return context.ActiveTransaction();
-}
-
 LocalStorage &Transaction::GetLocalStorage() {
 	return *storage;
 }
@@ -96,23 +92,29 @@ bool Transaction::ChangesMade() {
 	return undo_buffer.ChangesMade() || storage->ChangesMade();
 }
 
-bool Transaction::AutomaticCheckpoint(DatabaseInstance &db) {
-	auto &storage_manager = StorageManager::GetStorageManager(db);
+bool Transaction::AutomaticCheckpoint(AttachedDatabase &db) {
+	auto &storage_manager = db.GetStorageManager();
 	return storage_manager.AutomaticCheckpoint(storage->EstimatedSize() + undo_buffer.EstimatedSize());
 }
 
-string Transaction::Commit(DatabaseInstance &db, transaction_t commit_id, bool checkpoint) noexcept {
+string Transaction::Commit(AttachedDatabase &db, transaction_t commit_id, bool checkpoint) noexcept {
 	// "checkpoint" parameter indicates if the caller will checkpoint. If checkpoint ==
 	//    true: Then this function will NOT write to the WAL or flush/persist.
 	//          This method only makes commit in memory, expecting caller to checkpoint/flush.
 	//    false: Then this function WILL write to the WAL and Flush/Persist it.
 	this->commit_id = commit_id;
-	auto &storage_manager = StorageManager::GetStorageManager(db);
-	auto log = storage_manager.GetWriteAheadLog();
 
 	UndoBuffer::IteratorState iterator_state;
 	LocalStorage::CommitState commit_state;
-	auto storage_commit_state = storage_manager.GenStorageCommitState(*this, checkpoint);
+	unique_ptr<StorageCommitState> storage_commit_state;
+	WriteAheadLog *log;
+	if (!db.IsSystem()) {
+		auto &storage_manager = db.GetStorageManager();
+		log = storage_manager.GetWriteAheadLog();
+		storage_commit_state = storage_manager.GenStorageCommitState(*this, checkpoint);
+	} else {
+		log = nullptr;
+	}
 	try {
 		storage->Commit(commit_state, *this);
 		undo_buffer.Commit(iterator_state, log, commit_id);
@@ -122,7 +124,9 @@ string Transaction::Commit(DatabaseInstance &db, transaction_t commit_id, bool c
 				log->WriteSequenceValue(entry.first, entry.second);
 			}
 		}
-		storage_commit_state->FlushCommit();
+		if (storage_commit_state) {
+			storage_commit_state->FlushCommit();
+		}
 		return string();
 	} catch (std::exception &ex) {
 		undo_buffer.RevertCommit(iterator_state, transaction_id);
@@ -139,8 +143,4 @@ void Transaction::Cleanup() {
 	undo_buffer.Cleanup();
 }
 
-ValidChecker &ValidChecker::Get(Transaction &transaction) {
-	return transaction.transaction_validity;
-}
-
 } // namespace duckdb

package/src/duckdb/src/transaction/transaction_context.cpp

@@ -1,11 +1,17 @@
 #include "duckdb/transaction/transaction_context.hpp"
 
 #include "duckdb/common/exception.hpp"
-#include "duckdb/transaction/transaction.hpp"
+#include "duckdb/transaction/meta_transaction.hpp"
 #include "duckdb/transaction/transaction_manager.hpp"
+#include "duckdb/main/config.hpp"
+#include "duckdb/main/database_manager.hpp"
 
 namespace duckdb {
 
+TransactionContext::TransactionContext(ClientContext &context)
+    : context(context), auto_commit(true), current_transaction(nullptr) {
+}
+
 TransactionContext::~TransactionContext() {
 	if (current_transaction) {
 		try {
@@ -19,17 +25,27 @@ void TransactionContext::BeginTransaction() {
 	if (current_transaction) {
 		throw TransactionException("cannot start a transaction within a transaction");
 	}
-	current_transaction = transaction_manager.StartTransaction(context);
+	auto start_timestamp = Timestamp::GetCurrentTimestamp();
+	auto catalog_version = Catalog::GetSystemCatalog(context).GetCatalogVersion();
+	current_transaction = make_unique<MetaTransaction>(context, start_timestamp, catalog_version);
+
+	auto &config = DBConfig::GetConfig(context);
+	if (config.options.immediate_transaction_mode) {
+		// if immediate transaction mode is enabled then start all transactions immediately
+		auto databases = DatabaseManager::Get(context).GetDatabases(context);
+		for (auto db : databases) {
+			current_transaction->GetTransaction(db);
+		}
+	}
 }
 
 void TransactionContext::Commit() {
 	if (!current_transaction) {
 		throw TransactionException("failed to commit: no transaction active");
 	}
-	auto transaction = current_transaction;
-	SetAutoCommit(true);
-	current_transaction = nullptr;
-	string error = transaction_manager.CommitTransaction(context, transaction);
+	auto transaction = move(current_transaction);
+	ClearTransaction();
+	string error = transaction->Commit();
 	if (!error.empty()) {
 		throw TransactionException("Failed to commit: %s", error);
 	}
@@ -46,9 +62,9 @@ void TransactionContext::Rollback() {
 	if (!current_transaction) {
 		throw TransactionException("failed to rollback: no transaction active");
 	}
-	auto transaction = current_transaction;
+	auto transaction = move(current_transaction);
 	ClearTransaction();
-	transaction_manager.RollbackTransaction(transaction);
+	transaction->Rollback();
 }
 
 void TransactionContext::ClearTransaction() {
@@ -56,4 +72,24 @@ void TransactionContext::ClearTransaction() {
 	current_transaction = nullptr;
 }
 
+idx_t TransactionContext::GetActiveQuery() {
+	if (!current_transaction) {
+		throw InternalException("GetActiveQuery called without active transaction");
+	}
+	return current_transaction->GetActiveQuery();
+}
+
+void TransactionContext::ResetActiveQuery() {
+	if (current_transaction) {
+		SetActiveQuery(MAXIMUM_QUERY_ID);
+	}
+}
+
+void TransactionContext::SetActiveQuery(transaction_t query_number) {
+	if (!current_transaction) {
+		throw InternalException("SetActiveQuery called without active transaction");
+	}
+	current_transaction->SetActiveQuery(query_number);
+}
+
 } // namespace duckdb

package/src/duckdb/src/transaction/transaction_manager.cpp

@@ -10,6 +10,8 @@
 #include "duckdb/transaction/transaction.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/main/connection_manager.hpp"
+#include "duckdb/main/attached_database.hpp"
+#include "duckdb/main/database_manager.hpp"
 
 namespace duckdb {
 
@@ -38,16 +40,14 @@ struct CheckpointLock {
 	}
 };
 
-TransactionManager::TransactionManager(DatabaseInstance &db) : db(db), thread_is_checkpointing(false) {
-	// start timestamp starts at zero
-	current_start_timestamp = 0;
+TransactionManager::TransactionManager(AttachedDatabase &db) : db(db), thread_is_checkpointing(false) {
+	// start timestamp starts at two
+	current_start_timestamp = 2;
 	// transaction ID starts very high:
 	// it should be much higher than the current start timestamp
 	// if transaction_id < start_timestamp for any set of active transactions
 	// uncommited data could be read by
 	current_transaction_id = TRANSACTION_ID_START;
-	// the current active query id
-	current_query_number = 1;
 	lowest_active_id = TRANSACTION_ID_START;
 	lowest_active_start = MAX_TRANSACTION_ID;
 }
@@ -66,16 +66,13 @@ Transaction *TransactionManager::StartTransaction(ClientContext &context) {
 	// obtain the start time and transaction ID of this transaction
 	transaction_t start_time = current_start_timestamp++;
 	transaction_t transaction_id = current_transaction_id++;
-	timestamp_t start_timestamp = Timestamp::GetCurrentTimestamp();
 	if (active_transactions.empty()) {
 		lowest_active_start = start_time;
 		lowest_active_id = transaction_id;
 	}
 
 	// create the actual transaction
-	auto &catalog = Catalog::GetCatalog(db);
-	auto transaction =
-	    make_unique<Transaction>(context, start_time, transaction_id, start_timestamp, catalog.GetCatalogVersion());
+	auto transaction = make_unique<Transaction>(*this, context, start_time, transaction_id);
 	auto transaction_ptr = transaction.get();
 
 	// store it in the set of active transactions
@@ -106,19 +103,19 @@ void TransactionManager::LockClients(vector<ClientLockWrapper> &client_locks, Cl
 }
 
 void TransactionManager::Checkpoint(ClientContext &context, bool force) {
-	auto &storage_manager = StorageManager::GetStorageManager(db);
+	auto &storage_manager = db.GetStorageManager();
 	if (storage_manager.InMemory()) {
 		return;
 	}
 
 	// first check if no other thread is checkpointing right now
-	auto lock = make_unique<lock_guard<mutex>>(transaction_lock);
+	auto lock = unique_lock<mutex>(transaction_lock);
 	if (thread_is_checkpointing) {
 		throw TransactionException("Cannot CHECKPOINT: another thread is checkpointing right now");
 	}
 	CheckpointLock checkpoint_lock(*this);
 	checkpoint_lock.Lock();
-	lock.reset();
+	lock.unlock();
 
 	// lock all the clients AND the connection manager now
 	// this ensures no new queries can be started, and no new connections to the database can be made
@@ -126,8 +123,8 @@ void TransactionManager::Checkpoint(ClientContext &context, bool force) {
 	vector<ClientLockWrapper> client_locks;
 	LockClients(client_locks, context);
 
-	lock = make_unique<lock_guard<mutex>>(transaction_lock);
-	auto current = &Transaction::GetTransaction(context);
+	auto current = &Transaction::Get(context, db);
+	lock.lock();
 	if (current->ChangesMade()) {
 		throw TransactionException("Cannot CHECKPOINT: the current transaction has transaction local changes");
 	}
@@ -155,12 +152,14 @@ void TransactionManager::Checkpoint(ClientContext &context, bool force) {
 			D_ASSERT(CanCheckpoint(nullptr));
 		}
 	}
-	auto &storage = StorageManager::GetStorageManager(context);
-	storage.CreateCheckpoint();
+	storage_manager.CreateCheckpoint();
 }
 
 bool TransactionManager::CanCheckpoint(Transaction *current) {
-	auto &storage_manager = StorageManager::GetStorageManager(db);
+	if (db.IsSystem()) {
+		return false;
+	}
+	auto &storage_manager = db.GetStorageManager();
 	if (storage_manager.InMemory()) {
 		return false;
 	}
@@ -223,7 +222,7 @@ string TransactionManager::CommitTransaction(ClientContext &context, Transaction
 	// checkpoint
 	if (checkpoint) {
 		// checkpoint the database to disk
-		auto &storage_manager = StorageManager::GetStorageManager(db);
+		auto &storage_manager = db.GetStorageManager();
 		storage_manager.CreateCheckpoint(false, true);
 	}
 	return error;
@@ -264,6 +263,7 @@ void TransactionManager::RemoveTransaction(Transaction *transaction) noexcept {
 	transaction_t lowest_stored_query = lowest_start_time;
 	D_ASSERT(t_index != active_transactions.size());
 	auto current_transaction = move(active_transactions[t_index]);
+	auto current_query = DatabaseManager::Get(db).ActiveQueryNumber();
 	if (transaction->commit_id != 0) {
 		// the transaction was committed, add it to the list of recently
 		// committed transactions
@@ -271,7 +271,7 @@ void TransactionManager::RemoveTransaction(Transaction *transaction) noexcept {
 	} else {
 		// the transaction was aborted, but we might still need its information
 		// add it to the set of transactions awaiting GC
-		current_transaction->highest_active_query = current_query_number;
+		current_transaction->highest_active_query = current_query;
 		old_transactions.push_back(move(current_transaction));
 	}
 	// remove the transaction from the set of currently active transactions
@@ -296,7 +296,7 @@ void TransactionManager::RemoveTransaction(Transaction *transaction) noexcept {
 			// when all the currently active scans have finished running...)
 			recently_committed_transactions[i]->Cleanup();
 			// store the current highest active query
-			recently_committed_transactions[i]->highest_active_query = current_query_number;
+			recently_committed_transactions[i]->highest_active_query = current_query;
 			// move it to the list of transactions awaiting GC
 			old_transactions.push_back(move(recently_committed_transactions[i]));
 		} else {

package/src/duckdb/src/transaction/undo_buffer.cpp

@@ -16,9 +16,7 @@
 namespace duckdb {
 constexpr uint32_t UNDO_ENTRY_HEADER_SIZE = sizeof(UndoFlags) + sizeof(uint32_t);
 
-UndoBuffer::UndoBuffer(const shared_ptr<ClientContext> &context_p)
-    : context(*context_p), allocator(BufferAllocator::Get(*context_p)) {
-	D_ASSERT(context_p);
+UndoBuffer::UndoBuffer(ClientContext &context_p) : context(context_p), allocator(BufferAllocator::Get(context_p)) {
 }
 
 data_ptr_t UndoBuffer::CreateEntry(UndoFlags type, idx_t len) {

package/src/duckdb/third_party/libpg_query/include/nodes/nodes.hpp

@@ -421,6 +421,8 @@ typedef enum PGNodeTag {
 	T_PGPragmaStmt,
 	T_PGExportStmt,
 	T_PGImportStmt,
+	T_PGAttachStmt,
+	T_PGUseStmt,
 
 	/*
 	 * TAGS FOR PARSE TREE NODES (parsenodes.h)

package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp

@@ -1326,6 +1326,7 @@ typedef enum PGObjectType {
  */
 typedef struct PGCreateSchemaStmt {
 	PGNodeTag type;
+	char *catalogname;                    /* the name of the catalog in which to create the schema */
 	char *schemaname;                     /* the name of the schema to create */
 	PGList *schemaElts;                   /* schema components (list of parsenodes) */
 	PGOnCreateConflict onconflict;        /* what to do on create conflict */
@@ -1895,6 +1896,7 @@ typedef struct PGCreateTableAsStmt {
 typedef struct PGCheckPointStmt {
 	PGNodeTag type;
 	bool force;
+	char *name;
 } PGCheckPointStmt;
 
 /* ----------------------
@@ -1963,6 +1965,7 @@ typedef struct PGCallStmt {
 
 typedef struct PGExportStmt {
 	PGNodeTag type;
+	char *database;       /* database name */
 	char *filename;       /* filename */
 	PGList *options;      /* PGList of PGDefElem nodes */
 } PGExportStmt;
@@ -2046,13 +2049,36 @@ typedef struct PGCreateTypeStmt
 {
 	PGNodeTag		type;
 	PGNewTypeKind	kind;
-	PGList	   *typeName;		/* qualified name (list of Value strings) */
+	PGRangeVar	   *typeName;	/* qualified name (list of Value strings) */
 	PGList	   *vals;			/* enum values (list of Value strings) */
 	PGTypeName *ofType;			/* original type of alias name */
     PGNode *query;
 } PGCreateTypeStmt;
 
+/* ----------------------
+ *		Attach Statement
+ * ----------------------
+ */
+
+typedef struct PGAttachStmt
+{
+	PGNodeTag		type;
+	char *path;			/* The file path of the to-be-attached database */
+	char *name;			/* The name of the attached database */
+	PGList *options;      /* PGList of PGDefElem nodes */
+    PGNode *query;
+} PGAttachStmt;
+
 
+/* ----------------------
+ *		Use Statement
+ * ----------------------
+ */
+
+typedef struct PGUseStmt {
+	PGNodeTag type;
+	PGRangeVar *name;    /* variable to be set */
+} PGUseStmt;