duckdb 0.8.1-dev152.0 → 0.8.1-dev180.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.8.1-dev152.0",
+  "version": "0.8.1-dev180.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/src/common/adbc/adbc.cpp

@@ -5,10 +5,13 @@
 #include "duckdb/common/string_util.hpp"
 
 #include "duckdb.h"
-#include "duckdb/main/connection.hpp"
 #include "duckdb/common/arrow/arrow_wrapper.hpp"
 #include "duckdb/common/arrow/arrow.hpp"
 
+#ifndef DUCKDB_AMALGAMATION
+#include "duckdb/main/connection.hpp"
+#endif
+
 #include <string.h>
 #include <stdlib.h>
 
@@ -37,6 +40,10 @@ duckdb_adbc::AdbcStatusCode duckdb_adbc_init(size_t count, struct duckdb_adbc::A
 	driver->StatementSetOption = duckdb_adbc::StatementSetOption;
 	driver->StatementSetSqlQuery = duckdb_adbc::StatementSetSqlQuery;
 	driver->ConnectionGetObjects = duckdb_adbc::ConnectionGetObjects;
+	driver->ConnectionCommit = duckdb_adbc::ConnectionCommit;
+	driver->ConnectionRollback = duckdb_adbc::ConnectionRollback;
+	driver->ConnectionReadPartition = duckdb_adbc::ConnectionReadPartition;
+	driver->StatementExecutePartitions = duckdb_adbc::StatementExecutePartitions;
 	return ADBC_STATUS_OK;
 }
 
@@ -162,10 +169,104 @@ AdbcStatusCode ConnectionNew(struct AdbcConnection *connection, struct AdbcError
 	return ADBC_STATUS_OK;
 }
 
+AdbcStatusCode ExecuteQuery(duckdb::Connection *conn, const char *query, struct AdbcError *error) {
+	auto res = conn->Query(query);
+	if (res->HasError()) {
+		auto error_message = "Failed to execute query \"" + std::string(query) + "\": " + res->GetError();
+		SetError(error, error_message);
+		return ADBC_STATUS_INTERNAL;
+	}
+	return ADBC_STATUS_OK;
+}
+
 AdbcStatusCode ConnectionSetOption(struct AdbcConnection *connection, const char *key, const char *value,
                                    struct AdbcError *error) {
-	// there are no connection-level options that need to be set before connecting
-	return ADBC_STATUS_OK;
+	if (!connection) {
+		SetError(error, "Connection is not set");
+		return ADBC_STATUS_INVALID_ARGUMENT;
+	}
+	auto conn = (duckdb::Connection *)connection->private_data;
+	if (strcmp(key, ADBC_CONNECTION_OPTION_AUTOCOMMIT) == 0) {
+		if (strcmp(value, ADBC_OPTION_VALUE_ENABLED) == 0) {
+			if (conn->HasActiveTransaction()) {
+				AdbcStatusCode status = ExecuteQuery(conn, "COMMIT", error);
+				if (status != ADBC_STATUS_OK) {
+					return status;
+				}
+			} else {
+				// no-op
+			}
+		} else if (strcmp(value, ADBC_OPTION_VALUE_DISABLED) == 0) {
+			if (conn->HasActiveTransaction()) {
+				// no-op
+			} else {
+				// begin
+				AdbcStatusCode status = ExecuteQuery(conn, "START TRANSACTION", error);
+				if (status != ADBC_STATUS_OK) {
+					return status;
+				}
+			}
+		} else {
+			auto error_message = "Invalid connection option value " + std::string(key) + "=" + std::string(value);
+			SetError(error, error_message);
+			return ADBC_STATUS_INVALID_ARGUMENT;
+		}
+		return ADBC_STATUS_OK;
+	}
+	auto error_message =
+	    "Unknown connection option " + std::string(key) + "=" + (value ? std::string(value) : "(NULL)");
+	SetError(error, error_message);
+	return ADBC_STATUS_NOT_IMPLEMENTED;
+}
+
+AdbcStatusCode ConnectionReadPartition(struct AdbcConnection *connection, const uint8_t *serialized_partition,
+                                       size_t serialized_length, struct ArrowArrayStream *out,
+                                       struct AdbcError *error) {
+	SetError(error, "Read Partitions are not supported in DuckDB");
+	return ADBC_STATUS_NOT_IMPLEMENTED;
+}
+
+AdbcStatusCode StatementExecutePartitions(struct AdbcStatement *statement, struct ArrowSchema *schema,
+                                          struct AdbcPartitions *partitions, int64_t *rows_affected,
+                                          struct AdbcError *error) {
+	SetError(error, "Execute Partitions are not supported in DuckDB");
+	return ADBC_STATUS_NOT_IMPLEMENTED;
+}
+
+AdbcStatusCode ConnectionCommit(struct AdbcConnection *connection, struct AdbcError *error) {
+	if (!connection) {
+		SetError(error, "Connection is not set");
+		return ADBC_STATUS_INVALID_ARGUMENT;
+	}
+	auto conn = (duckdb::Connection *)connection->private_data;
+	if (!conn->HasActiveTransaction()) {
+		SetError(error, "No active transaction, cannot commit");
+		return ADBC_STATUS_INVALID_STATE;
+	}
+
+	AdbcStatusCode status = ExecuteQuery(conn, "COMMIT", error);
+	if (status != ADBC_STATUS_OK) {
+		return status;
+	}
+	return ExecuteQuery(conn, "START TRANSACTION", error);
+}
+
+AdbcStatusCode ConnectionRollback(struct AdbcConnection *connection, struct AdbcError *error) {
+	if (!connection) {
+		SetError(error, "Connection is not set");
+		return ADBC_STATUS_INVALID_ARGUMENT;
+	}
+	auto conn = (duckdb::Connection *)connection->private_data;
+	if (!conn->HasActiveTransaction()) {
+		SetError(error, "No active transaction, cannot rollback");
+		return ADBC_STATUS_INVALID_STATE;
+	}
+
+	AdbcStatusCode status = ExecuteQuery(conn, "ROLLBACK", error);
+	if (status != ADBC_STATUS_OK) {
+		return status;
+	}
+	return ExecuteQuery(conn, "START TRANSACTION", error);
 }
 
 AdbcStatusCode ConnectionInit(struct AdbcConnection *connection, struct AdbcDatabase *database,
@@ -219,11 +320,11 @@ void release(struct ArrowArrayStream *stream) {
 	if (!stream || !stream->release) {
 		return;
 	}
-	stream->release = nullptr;
 	if (stream->private_data) {
 		duckdb_destroy_arrow((duckdb_arrow *)&stream->private_data);
 		stream->private_data = nullptr;
 	}
+	stream->release = nullptr;
 }
 
 const char *get_last_error(struct ArrowArrayStream *stream) {
@@ -270,16 +371,21 @@ AdbcStatusCode Ingest(duckdb_connection connection, const char *table_name, stru
 	if (status != ADBC_STATUS_OK) {
 		return status;
 	}
+	auto cconn = (duckdb::Connection *)connection;
 
+	auto has_table = cconn->TableInfo(table_name);
+	auto arrow_scan = cconn->TableFunction("arrow_scan", {duckdb::Value::POINTER((uintptr_t)input),
+	                                                      duckdb::Value::POINTER((uintptr_t)stream_produce),
+	                                                      duckdb::Value::POINTER((uintptr_t)get_schema)});
 	try {
-		// TODO evil cast, do we need a way to do this from the C api?
-		auto cconn = (duckdb::Connection *)connection;
-		cconn
-		    ->TableFunction("arrow_scan",
-		                    {duckdb::Value::POINTER((uintptr_t)input),
-		                     duckdb::Value::POINTER((uintptr_t)stream_produce),
-		                     duckdb::Value::POINTER((uintptr_t)get_schema)}) // TODO make this a parameter somewhere
-		    ->Create(table_name); // TODO this should probably be a temp table
+		if (!has_table) {
+			// We create the table based on an Arrow Scanner
+			arrow_scan->Create(table_name);
+		} else {
+			arrow_scan->CreateView("temp_adbc_view", true, true);
+			auto query = "insert into " + std::string(table_name) + " select * from temp_adbc_view";
+			auto result = cconn->Query(query);
+		}
 		// After creating a table, the arrow array stream is released. Hence we must set it as released to avoid
 		// double-releasing it
 		input->release = nullptr;

package/src/duckdb/src/common/adbc/driver_manager.cpp

@@ -133,10 +133,6 @@ static AdbcStatusCode ReleaseDriver(struct AdbcDriver *driver, struct AdbcError
 
 // Default stubs
 
-AdbcStatusCode ConnectionCommit(struct AdbcConnection *, struct AdbcError *error) {
-	return ADBC_STATUS_NOT_IMPLEMENTED;
-}
-
 AdbcStatusCode ConnectionGetInfo(struct AdbcConnection *connection, uint32_t *info_codes, size_t info_codes_length,
                                  struct ArrowArrayStream *out, struct AdbcError *error) {
 	return ADBC_STATUS_NOT_IMPLEMENTED;
@@ -147,27 +143,11 @@ AdbcStatusCode ConnectionGetTableSchema(struct AdbcConnection *, const char *, c
 	return ADBC_STATUS_NOT_IMPLEMENTED;
 }
 
-AdbcStatusCode ConnectionReadPartition(struct AdbcConnection *connection, const uint8_t *serialized_partition,
-                                       size_t serialized_length, struct ArrowArrayStream *out,
-                                       struct AdbcError *error) {
-	return ADBC_STATUS_NOT_IMPLEMENTED;
-}
-
-AdbcStatusCode ConnectionRollback(struct AdbcConnection *, struct AdbcError *error) {
-	return ADBC_STATUS_NOT_IMPLEMENTED;
-}
-
 AdbcStatusCode StatementBind(struct AdbcStatement *, struct ArrowArray *, struct ArrowSchema *,
                              struct AdbcError *error) {
 	return ADBC_STATUS_NOT_IMPLEMENTED;
 }
 
-AdbcStatusCode StatementExecutePartitions(struct AdbcStatement *statement, struct ArrowSchema *schema,
-                                          struct AdbcPartitions *partitions, int64_t *rows_affected,
-                                          struct AdbcError *error) {
-	return ADBC_STATUS_NOT_IMPLEMENTED;
-}
-
 AdbcStatusCode StatementGetParameterSchema(struct AdbcStatement *statement, struct ArrowSchema *schema,
                                            struct AdbcError *error) {
 	return ADBC_STATUS_NOT_IMPLEMENTED;

package/src/duckdb/src/function/table/arrow.cpp

@@ -206,8 +206,8 @@ unique_ptr<FunctionData> ArrowTableFunction::ArrowScanBind(ClientContext &contex
 			throw InvalidInputException("arrow_scan: released schema passed");
 		}
 		if (schema.dictionary) {
-			res->arrow_convert_data[col_idx] =
-			    make_uniq<ArrowConvertData>(GetArrowLogicalType(schema, res->arrow_convert_data, col_idx));
+			auto logical_type = GetArrowLogicalType(schema, res->arrow_convert_data, col_idx);
+			res->arrow_convert_data[col_idx] = make_uniq<ArrowConvertData>(std::move(logical_type));
 			return_types.emplace_back(GetArrowLogicalType(*schema.dictionary, res->arrow_convert_data, col_idx));
 		} else {
 			return_types.emplace_back(GetArrowLogicalType(schema, res->arrow_convert_data, col_idx));

package/src/duckdb/src/function/table/read_csv.cpp

@@ -378,8 +378,8 @@ private:
 	//! Current File Number
 	idx_t file_number = 0;
 	idx_t max_tuple_end = 0;
-	//! the vector stores positions where threads ended the last line they read in the CSV File, and the set stores
-	//! positions where they started reading the first line.
+	//! The vector stores positions where threads ended the last line they read in the CSV File, and the set stores
+	//! Positions where they started reading the first line.
 	vector<vector<idx_t>> tuple_end;
 	vector<set<idx_t>> tuple_start;
 	//! Tuple end to batch
@@ -559,15 +559,13 @@ bool ParallelCSVGlobalState::Next(ClientContext &context, const ReadCSVData &bin
 }
 void ParallelCSVGlobalState::UpdateVerification(VerificationPositions positions, idx_t file_number_p, idx_t batch_idx) {
 	lock_guard<mutex> parallel_lock(main_mutex);
-	if (positions.beginning_of_first_line < positions.end_of_last_line) {
-		if (positions.end_of_last_line > max_tuple_end) {
-			max_tuple_end = positions.end_of_last_line;
-		}
-		tuple_end_to_batch[file_number_p][positions.end_of_last_line] = batch_idx;
-		batch_to_tuple_end[file_number_p][batch_idx] = tuple_end[file_number_p].size();
-		tuple_start[file_number_p].insert(positions.beginning_of_first_line);
-		tuple_end[file_number_p].push_back(positions.end_of_last_line);
+	if (positions.end_of_last_line > max_tuple_end) {
+		max_tuple_end = positions.end_of_last_line;
 	}
+	tuple_end_to_batch[file_number_p][positions.end_of_last_line] = batch_idx;
+	batch_to_tuple_end[file_number_p][batch_idx] = tuple_end[file_number_p].size();
+	tuple_start[file_number_p].insert(positions.beginning_of_first_line);
+	tuple_end[file_number_p].push_back(positions.end_of_last_line);
 }
 
 void ParallelCSVGlobalState::UpdateLinesRead(CSVBufferRead &buffer_read, idx_t file_idx) {
@@ -690,17 +688,14 @@ static void ParallelReadCSVFunction(ClientContext &context, TableFunctionInput &
 		}
 		if (csv_local_state.csv_reader->finished) {
 			auto verification_updates = csv_local_state.csv_reader->GetVerificationPositions();
-			if (verification_updates.beginning_of_first_line != verification_updates.end_of_last_line) {
-				csv_global_state.UpdateVerification(verification_updates,
-				                                    csv_local_state.csv_reader->buffer->buffer->GetFileNumber(),
-				                                    csv_local_state.csv_reader->buffer->local_batch_index);
-			}
+			csv_global_state.UpdateVerification(verification_updates,
+			                                    csv_local_state.csv_reader->buffer->buffer->GetFileNumber(),
+			                                    csv_local_state.csv_reader->buffer->local_batch_index);
 			csv_global_state.UpdateLinesRead(*csv_local_state.csv_reader->buffer, csv_local_state.csv_reader->file_idx);
 			auto has_next = csv_global_state.Next(context, bind_data, csv_local_state.csv_reader);
 			if (csv_local_state.csv_reader) {
 				csv_local_state.csv_reader->linenr = 0;
 			}
-
 			if (!has_next) {
 				csv_global_state.DecrementThread();
 				break;

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.8.1-dev152"
+#define DUCKDB_VERSION "0.8.1-dev180"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "d636ce2d59"
+#define DUCKDB_SOURCE_ID "e9f6ba553a"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/function/function_serialization.hpp

@@ -25,8 +25,10 @@ public:
 		bool serialize = function.serialize;
 		writer.WriteField(serialize);
 		if (serialize) {
-			D_ASSERT(function.deserialize);
 			function.serialize(writer, bind_info, function);
+			// First check if serialize throws a NotImplementedException, in which case it doesn't require a deserialize
+			// function
+			D_ASSERT(function.deserialize);
 		}
 	}