duckdb 0.6.2-dev971.0 → 0.7.1-dev2.0

package/src/duckdb/src/include/duckdb/transaction/transaction_manager.hpp

@@ -27,61 +27,32 @@ class Transaction;
 //! The Transaction Manager is responsible for creating and managing
 //! transactions
 class TransactionManager {
-	friend struct CheckpointLock;
-
 public:
 	explicit TransactionManager(AttachedDatabase &db);
-	~TransactionManager();
+	virtual ~TransactionManager();
 
 	//! Start a new transaction
-	Transaction *StartTransaction(ClientContext &context);
-	//! Commit the given transaction
-	string CommitTransaction(ClientContext &context, Transaction *transaction);
+	virtual Transaction *StartTransaction(ClientContext &context) = 0;
+	//! Commit the given transaction. Returns a non-empty error message on failure.
+	virtual string CommitTransaction(ClientContext &context, Transaction *transaction) = 0;
 	//! Rollback the given transaction
-	void RollbackTransaction(Transaction *transaction);
-
-	transaction_t LowestActiveId() {
-		return lowest_active_id;
-	}
-	transaction_t LowestActiveStart() {
-		return lowest_active_start;
-	}
+	virtual void RollbackTransaction(Transaction *transaction) = 0;
 
-	void Checkpoint(ClientContext &context, bool force = false);
+	virtual void Checkpoint(ClientContext &context, bool force = false) = 0;
 
 	static TransactionManager &Get(AttachedDatabase &db);
 
-	void SetBaseCommitId(transaction_t base) {
-		D_ASSERT(base >= TRANSACTION_ID_START);
-		current_transaction_id = base;
+	virtual bool IsDuckTransactionManager() {
+		return false;
 	}
 
-private:
-	bool CanCheckpoint(Transaction *current = nullptr);
-	//! Remove the given transaction from the list of active transactions
-	void RemoveTransaction(Transaction *transaction) noexcept;
-	void LockClients(vector<ClientLockWrapper> &client_locks, ClientContext &context);
+	AttachedDatabase &GetDB() {
+		return db;
+	}
 
+protected:
 	//! The attached database
 	AttachedDatabase &db;
-	//! The current start timestamp used by transactions
-	transaction_t current_start_timestamp;
-	//! The current transaction ID used by transactions
-	transaction_t current_transaction_id;
-	//! The lowest active transaction id
-	atomic<transaction_t> lowest_active_id;
-	//! The lowest active transaction timestamp
-	atomic<transaction_t> lowest_active_start;
-	//! Set of currently running transactions
-	vector<unique_ptr<Transaction>> active_transactions;
-	//! Set of recently committed transactions
-	vector<unique_ptr<Transaction>> recently_committed_transactions;
-	//! Transactions awaiting GC
-	vector<unique_ptr<Transaction>> old_transactions;
-	//! The lock used for transaction operations
-	mutex transaction_lock;
-
-	bool thread_is_checkpointing;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb.h

@@ -119,10 +119,10 @@ typedef enum DUCKDB_TYPE {
 	DUCKDB_TYPE_MAP,
 	// duckdb_hugeint
 	DUCKDB_TYPE_UUID,
-	// const char*
-	DUCKDB_TYPE_JSON,
 	// union type, only useful as logical type
 	DUCKDB_TYPE_UNION,
+	// duckdb_bit
+	DUCKDB_TYPE_BIT,
 } duckdb_type;
 
 //! Days are stored as days since 1970-01-01
@@ -192,6 +192,11 @@ typedef struct {
 	idx_t size;
 } duckdb_blob;
 
+typedef struct {
+	uint64_t offset;
+	uint64_t length;
+} duckdb_list_entry;
+
 typedef struct {
 #if DUCKDB_API_VERSION < DUCKDB_API_0_3_2
 	void *data;
@@ -1561,6 +1566,24 @@ Returns the size of the child vector of the list
 */
 DUCKDB_API idx_t duckdb_list_vector_get_size(duckdb_vector vector);
 
+/*!
+Sets the total size of the underlying child-vector of a list vector.
+
+* vector: The list vector.
+* size: The size of the child list.
+* returns: The duckdb state. Returns DuckDBError if the vector is nullptr.
+*/
+DUCKDB_API duckdb_state duckdb_list_vector_set_size(duckdb_vector vector, idx_t size);
+
+/*!
+Sets the total capacity of the underlying child-vector of a list.
+
+* vector: The list vector.
+* required_capacity: the total capacity to reserve.
+* return: The duckdb state. Returns DuckDBError if the vector is nullptr.
+*/
+DUCKDB_API duckdb_state duckdb_list_vector_reserve(duckdb_vector vector, idx_t required_capacity);
+
 /*!
 Retrieves the child vector of a struct vector.
 
@@ -2289,6 +2312,13 @@ on the task state.
 */
 DUCKDB_API void duckdb_destroy_task_state(duckdb_task_state state);
 
+/*!
+Returns true if execution of the current query is finished.
+
+* con: The connection on which to check
+*/
+DUCKDB_API bool duckdb_execution_is_finished(duckdb_connection con);
+
 #ifdef __cplusplus
 }
 #endif

package/src/duckdb/src/include/duckdb.hpp

@@ -12,3 +12,4 @@
 #include "duckdb/main/database.hpp"
 #include "duckdb/main/query_result.hpp"
 #include "duckdb/main/loadable_extension.hpp"
+#include "duckdb/main/appender.hpp"

package/src/duckdb/src/main/appender.cpp

@@ -1,5 +1,6 @@
 #include "duckdb/main/appender.hpp"
 
+#include "duckdb/catalog/catalog_entry/duck_table_entry.hpp"
 #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
 #include "duckdb/common/exception.hpp"
 #include "duckdb/main/connection.hpp"
@@ -19,8 +20,8 @@ BaseAppender::BaseAppender(Allocator &allocator, AppenderType type_p)
 }
 
 BaseAppender::BaseAppender(Allocator &allocator_p, vector<LogicalType> types_p, AppenderType type_p)
-    : allocator(allocator_p), types(move(types_p)), collection(make_unique<ColumnDataCollection>(allocator, types)),
-      column(0), appender_type(type_p) {
+    : allocator(allocator_p), types(std::move(types_p)),
+      collection(make_unique<ColumnDataCollection>(allocator, types)), column(0), appender_type(type_p) {
 	InitializeChunk();
 }
 
@@ -357,7 +358,7 @@ void Appender::FlushInternal(ColumnDataCollection &collection) {
 }
 
 void InternalAppender::FlushInternal(ColumnDataCollection &collection) {
-	table.storage->LocalAppend(table, context, collection);
+	table.GetStorage().LocalAppend(table, context, collection);
 }
 
 void BaseAppender::Close() {

package/src/duckdb/src/main/attached_database.cpp

@@ -1,7 +1,9 @@
 #include "duckdb/main/attached_database.hpp"
 #include "duckdb/storage/storage_manager.hpp"
-#include "duckdb/transaction/transaction_manager.hpp"
+#include "duckdb/transaction/duck_transaction_manager.hpp"
 #include "duckdb/common/file_system.hpp"
+#include "duckdb/catalog/duck_catalog.hpp"
+#include "duckdb/storage/storage_extension.hpp"
 
 namespace duckdb {
 
@@ -13,19 +15,38 @@ AttachedDatabase::AttachedDatabase(DatabaseInstance &db, AttachedDatabaseType ty
 	if (type == AttachedDatabaseType::TEMP_DATABASE) {
 		storage = make_unique<SingleFileStorageManager>(*this, ":memory:", false);
 	}
-	catalog = make_unique<Catalog>(*this);
-	transaction_manager = make_unique<TransactionManager>(*this);
+	catalog = make_unique<DuckCatalog>(*this);
+	transaction_manager = make_unique<DuckTransactionManager>(*this);
 	internal = true;
 }
 
 AttachedDatabase::AttachedDatabase(DatabaseInstance &db, Catalog &catalog_p, string name_p, string file_path_p,
                                    AccessMode access_mode)
-    : CatalogEntry(CatalogType::DATABASE_ENTRY, &catalog_p, move(name_p)), db(db),
+    : CatalogEntry(CatalogType::DATABASE_ENTRY, &catalog_p, std::move(name_p)), db(db),
       type(access_mode == AccessMode::READ_ONLY ? AttachedDatabaseType::READ_ONLY_DATABASE
                                                 : AttachedDatabaseType::READ_WRITE_DATABASE) {
-	storage = make_unique<SingleFileStorageManager>(*this, move(file_path_p), access_mode == AccessMode::READ_ONLY);
-	catalog = make_unique<Catalog>(*this);
-	transaction_manager = make_unique<TransactionManager>(*this);
+	storage =
+	    make_unique<SingleFileStorageManager>(*this, std::move(file_path_p), access_mode == AccessMode::READ_ONLY);
+	catalog = make_unique<DuckCatalog>(*this);
+	transaction_manager = make_unique<DuckTransactionManager>(*this);
+	internal = true;
+}
+
+AttachedDatabase::AttachedDatabase(DatabaseInstance &db, Catalog &catalog_p, StorageExtension &storage_extension,
+                                   string name_p, AttachInfo &info, AccessMode access_mode)
+    : CatalogEntry(CatalogType::DATABASE_ENTRY, &catalog_p, std::move(name_p)), db(db),
+      type(access_mode == AccessMode::READ_ONLY ? AttachedDatabaseType::READ_ONLY_DATABASE
+                                                : AttachedDatabaseType::READ_WRITE_DATABASE) {
+	catalog = storage_extension.attach(storage_extension.storage_info.get(), *this, name, info, access_mode);
+	if (!catalog) {
+		throw InternalException("AttachedDatabase - attach function did not return a catalog");
+	}
+	transaction_manager =
+	    storage_extension.create_transaction_manager(storage_extension.storage_info.get(), *this, *catalog);
+	if (!transaction_manager) {
+		throw InternalException(
+		    "AttachedDatabase - create_transaction_manager function did not return a transaction manager");
+	}
 	internal = true;
 }
 
@@ -33,7 +54,7 @@ AttachedDatabase::~AttachedDatabase() {
 	if (Exception::UncaughtException()) {
 		return;
 	}
-	if (IsSystem()) {
+	if (!storage) {
 		return;
 	}
 
@@ -75,12 +96,14 @@ void AttachedDatabase::Initialize() {
 		catalog->Initialize(true);
 	} else {
 		catalog->Initialize(false);
+	}
+	if (storage) {
 		storage->Initialize();
 	}
 }
 
 StorageManager &AttachedDatabase::GetStorageManager() {
-	if (IsSystem()) {
+	if (!storage) {
 		throw InternalException("Internal system catalog does not have storage");
 	}
 	return *storage;

package/src/duckdb/src/main/capi/arrow-c.cpp

@@ -104,7 +104,7 @@ duckdb_state duckdb_execute_prepared_arrow(duckdb_prepared_statement prepared_st
 
 	auto result = wrapper->statement->Execute(wrapper->values, false);
 	D_ASSERT(result->type == QueryResultType::MATERIALIZED_RESULT);
-	arrow_wrapper->result = duckdb::unique_ptr_cast<QueryResult, MaterializedQueryResult>(move(result));
+	arrow_wrapper->result = duckdb::unique_ptr_cast<QueryResult, MaterializedQueryResult>(std::move(result));
 	*out_result = (duckdb_arrow)arrow_wrapper;
 	return !arrow_wrapper->result->HasError() ? DuckDBSuccess : DuckDBError;
 }

package/src/duckdb/src/main/capi/data_chunk-c.cpp

@@ -1,6 +1,7 @@
-#include "duckdb/main/capi/capi_internal.hpp"
 #include "duckdb/common/types/data_chunk.hpp"
 #include "duckdb/common/types/string_type.hpp"
+#include "duckdb/main/capi/capi_internal.hpp"
+
 #include <string.h>
 
 duckdb_data_chunk duckdb_create_data_chunk(duckdb_logical_type *ctypes, idx_t column_count) {
@@ -128,6 +129,24 @@ idx_t duckdb_list_vector_get_size(duckdb_vector vector) {
 	return duckdb::ListVector::GetListSize(*v);
 }
 
+duckdb_state duckdb_list_vector_set_size(duckdb_vector vector, idx_t size) {
+	if (!vector) {
+		return duckdb_state::DuckDBError;
+	}
+	auto v = (duckdb::Vector *)vector;
+	duckdb::ListVector::SetListSize(*v, size);
+	return duckdb_state::DuckDBSuccess;
+}
+
+duckdb_state duckdb_list_vector_reserve(duckdb_vector vector, idx_t required_capacity) {
+	if (!vector) {
+		return duckdb_state::DuckDBError;
+	}
+	auto v = (duckdb::Vector *)vector;
+	duckdb::ListVector::Reserve(*v, required_capacity);
+	return duckdb_state::DuckDBSuccess;
+}
+
 duckdb_vector duckdb_struct_vector_get_child(duckdb_vector vector, idx_t index) {
 	if (!vector) {
 		return nullptr;

package/src/duckdb/src/main/capi/duckdb-c.cpp

@@ -65,7 +65,7 @@ void duckdb_disconnect(duckdb_connection *connection) {
 duckdb_state duckdb_query(duckdb_connection connection, const char *query, duckdb_result *out) {
 	Connection *conn = (Connection *)connection;
 	auto result = conn->Query(query);
-	return duckdb_translate_result(move(result), out);
+	return duckdb_translate_result(std::move(result), out);
 }
 
 const char *duckdb_library_version() {

package/src/duckdb/src/main/capi/helper-c.cpp

@@ -36,8 +36,6 @@ LogicalTypeId ConvertCTypeToCPP(duckdb_type c_type) {
 		return LogicalTypeId::TIME;
 	case DUCKDB_TYPE_VARCHAR:
 		return LogicalTypeId::VARCHAR;
-	case DUCKDB_TYPE_JSON:
-		return LogicalTypeId::JSON;
 	case DUCKDB_TYPE_BLOB:
 		return LogicalTypeId::BLOB;
 	case DUCKDB_TYPE_INTERVAL:
@@ -98,10 +96,10 @@ duckdb_type ConvertCPPTypeToC(const LogicalType &sql_type) {
 		return DUCKDB_TYPE_TIME;
 	case LogicalTypeId::VARCHAR:
 		return DUCKDB_TYPE_VARCHAR;
-	case LogicalTypeId::JSON:
-		return DUCKDB_TYPE_JSON;
 	case LogicalTypeId::BLOB:
 		return DUCKDB_TYPE_BLOB;
+	case LogicalTypeId::BIT:
+		return DUCKDB_TYPE_BIT;
 	case LogicalTypeId::INTERVAL:
 		return DUCKDB_TYPE_INTERVAL;
 	case LogicalTypeId::DECIMAL:

package/src/duckdb/src/main/capi/pending-c.cpp

@@ -92,5 +92,5 @@ duckdb_state duckdb_execute_pending(duckdb_pending_result pending_result, duckdb
 	}
 	auto result = wrapper->statement->Execute();
 	wrapper->statement.reset();
-	return duckdb_translate_result(move(result), out_result);
+	return duckdb_translate_result(std::move(result), out_result);
 }

package/src/duckdb/src/main/capi/prepared-c.cpp

@@ -42,7 +42,7 @@ duckdb_state duckdb_prepare_extracted_statement(duckdb_connection connection,
 		return DuckDBError;
 	}
 	auto wrapper = new PreparedStatementWrapper();
-	wrapper->statement = conn->Prepare(move(source_wrapper->statements[index]));
+	wrapper->statement = conn->Prepare(std::move(source_wrapper->statements[index]));
 
 	*out_prepared_statement = (duckdb_prepared_statement)wrapper;
 	return wrapper->statement->HasError() ? DuckDBError : DuckDBSuccess;
@@ -233,7 +233,7 @@ duckdb_state duckdb_execute_prepared(duckdb_prepared_statement prepared_statemen
 		return DuckDBError;
 	}
 	auto result = wrapper->statement->Execute(wrapper->values, false);
-	return duckdb_translate_result(move(result), out_result);
+	return duckdb_translate_result(std::move(result), out_result);
 }
 
 template <class T>

package/src/duckdb/src/main/capi/replacement_scan-c.cpp

@@ -28,8 +28,8 @@ struct CAPIReplacementScanInfo {
 	string error;
 };
 
-unique_ptr<TableFunctionRef> duckdb_capi_replacement_callback(ClientContext &context, const string &table_name,
-                                                              ReplacementScanData *data) {
+unique_ptr<TableRef> duckdb_capi_replacement_callback(ClientContext &context, const string &table_name,
+                                                      ReplacementScanData *data) {
 	auto &scan_data = (CAPIReplacementScanData &)*data;
 
 	CAPIReplacementScanInfo info(&scan_data);
@@ -44,10 +44,10 @@ unique_ptr<TableFunctionRef> duckdb_capi_replacement_callback(ClientContext &con
 	auto table_function = make_unique<TableFunctionRef>();
 	vector<unique_ptr<ParsedExpression>> children;
 	for (auto &param : info.parameters) {
-		children.push_back(make_unique<ConstantExpression>(move(param)));
+		children.push_back(make_unique<ConstantExpression>(std::move(param)));
 	}
-	table_function->function = make_unique<FunctionExpression>(info.function_name, move(children));
-	return table_function;
+	table_function->function = make_unique<FunctionExpression>(info.function_name, std::move(children));
+	return std::move(table_function);
 }
 
 } // namespace duckdb
@@ -65,7 +65,7 @@ void duckdb_add_replacement_scan(duckdb_database db, duckdb_replacement_callback
 
 	auto &config = duckdb::DBConfig::GetConfig(*wrapper->database->instance);
 	config.replacement_scans.push_back(
-	    duckdb::ReplacementScan(duckdb::duckdb_capi_replacement_callback, move(scan_info)));
+	    duckdb::ReplacementScan(duckdb::duckdb_capi_replacement_callback, std::move(scan_info)));
 }
 
 void duckdb_replacement_scan_set_function_name(duckdb_replacement_scan_info info_p, const char *function_name) {

package/src/duckdb/src/main/capi/result-c.cpp

@@ -260,7 +260,7 @@ duckdb_state duckdb_translate_result(unique_ptr<QueryResult> result_p, duckdb_re
 
 	// initialize the result_data object
 	auto result_data = new DuckDBResultData();
-	result_data->result = move(result_p);
+	result_data->result = std::move(result_p);
 	result_data->result_set_type = CAPIResultSetType::CAPI_RESULT_TYPE_NONE;
 	out->internal_data = result_data;
 

package/src/duckdb/src/main/capi/table_function-c.cpp

@@ -120,7 +120,7 @@ unique_ptr<FunctionData> CTableFunctionBind(ClientContext &context, TableFunctio
 	}
 
 	result->info = info;
-	return move(result);
+	return std::move(result);
 }
 
 unique_ptr<GlobalTableFunctionState> CTableFunctionInit(ClientContext &context, TableFunctionInitInput &data_p) {
@@ -132,7 +132,7 @@ unique_ptr<GlobalTableFunctionState> CTableFunctionInit(ClientContext &context,
 	if (!init_info.success) {
 		throw Exception(init_info.error);
 	}
-	return move(result);
+	return std::move(result);
 }
 
 unique_ptr<LocalTableFunctionState> CTableFunctionLocalInit(ExecutionContext &context, TableFunctionInitInput &data_p,
@@ -140,7 +140,7 @@ unique_ptr<LocalTableFunctionState> CTableFunctionLocalInit(ExecutionContext &co
 	auto &bind_data = (CTableBindData &)*data_p.bind_data;
 	auto result = make_unique<CTableLocalInitData>();
 	if (!bind_data.info->local_init) {
-		return move(result);
+		return std::move(result);
 	}
 
 	CTableInternalInitInfo init_info(bind_data, result->init_data, data_p.column_ids, data_p.filters);
@@ -148,7 +148,7 @@ unique_ptr<LocalTableFunctionState> CTableFunctionLocalInit(ExecutionContext &co
 	if (!init_info.success) {
 		throw Exception(init_info.error);
 	}
-	return move(result);
+	return std::move(result);
 }
 
 unique_ptr<NodeStatistics> CTableFunctionCardinality(ClientContext &context, const FunctionData *bind_data_p) {

package/src/duckdb/src/main/capi/threading-c.cpp

@@ -78,3 +78,11 @@ void duckdb_destroy_task_state(duckdb_task_state state_p) {
 	auto state = (CAPITaskState *)state_p;
 	delete state;
 }
+
+bool duckdb_execution_is_finished(duckdb_connection con) {
+	if (!con) {
+		return false;
+	}
+	duckdb::Connection *conn = (duckdb::Connection *)con;
+	return conn->context->ExecutionIsFinished();
+}