duckdb 0.5.2-dev878.0 → 0.6.0

package/src/parquet-amalgamation.hpp

@@ -7559,7 +7559,7 @@ public:
 	static unique_ptr<ColumnReader> CreateReader(ParquetReader &reader, const LogicalType &type_p,
 	                                             const SchemaElement &schema_p, idx_t schema_idx_p, idx_t max_define,
 	                                             idx_t max_repeat);
-	virtual void InitializeRead(const std::vector<ColumnChunk> &columns, TProtocol &protocol_p);
+	virtual void InitializeRead(idx_t row_group_index, const std::vector<ColumnChunk> &columns, TProtocol &protocol_p);
 	virtual idx_t Read(uint64_t num_values, parquet_filter_t &filter, uint8_t *define_out, uint8_t *repeat_out,
 	                   Vector &result_out);
 
@@ -7579,7 +7579,7 @@ public:
 	// register the range this reader will touch for prefetching
 	virtual void RegisterPrefetch(ThriftFileTransport &transport, bool allow_merge);
 
-	virtual unique_ptr<BaseStatistics> Stats(const std::vector<ColumnChunk> &columns);
+	virtual unique_ptr<BaseStatistics> Stats(idx_t row_group_idx_p, const std::vector<ColumnChunk> &columns);
 
 protected:
 	// readers that use the default Read() need to implement those
@@ -7742,6 +7742,7 @@ struct ParquetOptions {
 
 	bool binary_as_string = false;
 	bool filename = false;
+	bool file_row_number = false;
 	bool hive_partitioning = false;
 
 public:

package/src/statement.cpp

@@ -4,6 +4,9 @@
 
 #include <algorithm>
 #include <cassert>
+#include <iostream>
+#include <string>
+#include <regex>
 
 namespace node_duckdb {
 
@@ -15,8 +18,8 @@ Napi::Object Statement::Init(Napi::Env env, Napi::Object exports) {
 	Napi::Function t =
 	    DefineClass(env, "Statement",
 	                {InstanceMethod("run", &Statement::Run), InstanceMethod("all", &Statement::All),
-	                 InstanceMethod("each", &Statement::Each), InstanceMethod("finalize", &Statement::Finish),
-	                 InstanceMethod("stream", &Statement::Stream)});
+	                 InstanceMethod("arrowIPCAll", &Statement::ArrowIPCAll), InstanceMethod("each", &Statement::Each),
+	                 InstanceMethod("finalize", &Statement::Finish), InstanceMethod("stream", &Statement::Stream)});
 
 	constructor = Napi::Persistent(t);
 	constructor.SuppressDestruct();
@@ -225,8 +228,7 @@ static Napi::Value convert_col_val(Napi::Env &env, duckdb::Value dval, duckdb::L
 		value = object_value;
 	} break;
 	default:
-		Napi::Error::New(env, "Data type is not supported " + dval.type().ToString()).ThrowAsJavaScriptException();
-		return env.Null();
+		value = Napi::String::New(env, dval.ToString());
 	}
 
 	return value;
@@ -255,7 +257,7 @@ static Napi::Value convert_chunk(Napi::Env &env, std::vector<std::string> names,
 	return scope.Escape(result);
 }
 
-enum RunType { RUN, EACH, ALL };
+enum RunType { RUN, EACH, ALL, ARROW_ALL };
 
 struct StatementParam {
 	std::vector<duckdb::Value> params;
@@ -275,7 +277,8 @@ struct RunPreparedTask : public Task {
 			return;
 		}
 
-		result = statement.statement->Execute(params->params, run_type != RunType::ALL);
+		result =
+		    statement.statement->Execute(params->params, run_type != RunType::ALL && run_type != RunType::ARROW_ALL);
 	}
 
 	void Callback() override {
@@ -350,6 +353,62 @@ struct RunPreparedTask : public Task {
 
 			cb.MakeCallback(statement.Value(), {env.Null(), result_arr});
 		} break;
+		case RunType::ARROW_ALL: {
+			auto materialized_result = (duckdb::MaterializedQueryResult *)result.get();
+			// +1 is for null bytes at end of stream
+			Napi::Array result_arr(Napi::Array::New(env, materialized_result->RowCount() + 1));
+
+			auto deleter = [](Napi::Env, void *finalizeData, void *hint) {
+				delete static_cast<std::shared_ptr<duckdb::QueryResult> *>(hint);
+			};
+
+			std::shared_ptr<duckdb::QueryResult> result_ptr = move(result);
+
+			duckdb::idx_t out_idx = 1;
+			while (true) {
+				auto chunk = result_ptr->Fetch();
+
+				if (!chunk || chunk->size() == 0) {
+					break;
+				}
+
+				D_ASSERT(chunk->ColumnCount() == 2);
+				D_ASSERT(chunk->data[0].GetType() == duckdb::LogicalType::BLOB);
+				D_ASSERT(chunk->data[1].GetType() == duckdb::LogicalType::BOOLEAN);
+
+				for (duckdb::idx_t row_idx = 0; row_idx < chunk->size(); row_idx++) {
+					duckdb::string_t blob = ((duckdb::string_t *)(chunk->data[0].GetData()))[row_idx];
+					bool is_header = chunk->data[1].GetData()[row_idx];
+
+					// Create shared pointer to give (shared) ownership to ArrayBuffer, not that for these materialized
+					// query results, the string data is owned by the QueryResult
+					auto result_ref_ptr = new std::shared_ptr<duckdb::QueryResult>(result_ptr);
+
+					auto array_buffer = Napi::ArrayBuffer::New(env, (void *)blob.GetDataUnsafe(), blob.GetSize(),
+					                                           deleter, result_ref_ptr);
+
+					auto typed_array = Napi::Uint8Array::New(env, blob.GetSize(), array_buffer, 0);
+
+					// TODO we should handle this in duckdb probably
+					if (is_header) {
+						result_arr.Set((uint32_t)0, typed_array);
+					} else {
+						D_ASSERT(out_idx < materialized_result->RowCount());
+						result_arr.Set(out_idx++, typed_array);
+					}
+				}
+			}
+
+			// TODO we should handle this in duckdb probably
+			auto null_arr = Napi::Uint8Array::New(env, 4);
+			memset(null_arr.Data(), '\0', 4);
+			result_arr.Set(out_idx++, null_arr);
+
+			// Confirm all rows are set
+			D_ASSERT(out_idx == materialized_result->RowCount() + 1);
+
+			cb.MakeCallback(statement.Value(), {env.Null(), result_arr});
+		} break;
 		}
 	}
 	std::unique_ptr<duckdb::QueryResult> result;
@@ -424,6 +483,12 @@ Napi::Value Statement::All(const Napi::CallbackInfo &info) {
 	return info.This();
 }
 
+Napi::Value Statement::ArrowIPCAll(const Napi::CallbackInfo &info) {
+	connection_ref->database_ref->Schedule(
+	    info.Env(), duckdb::make_unique<RunPreparedTask>(*this, HandleArgs(info), RunType::ARROW_ALL));
+	return info.This();
+}
+
 Napi::Value Statement::Run(const Napi::CallbackInfo &info) {
 	connection_ref->database_ref->Schedule(info.Env(),
 	                                       duckdb::make_unique<RunPreparedTask>(*this, HandleArgs(info), RunType::RUN));
@@ -472,7 +537,9 @@ Napi::FunctionReference QueryResult::constructor;
 Napi::Object QueryResult::Init(Napi::Env env, Napi::Object exports) {
 	Napi::HandleScope scope(env);
 
-	Napi::Function t = DefineClass(env, "QueryResult", {InstanceMethod("nextChunk", &QueryResult::NextChunk)});
+	Napi::Function t = DefineClass(env, "QueryResult",
+	                               {InstanceMethod("nextChunk", &QueryResult::NextChunk),
+	                                InstanceMethod("nextIpcBuffer", &QueryResult::NextIpcBuffer)});
 
 	constructor = Napi::Persistent(t);
 	constructor.SuppressDestruct();
@@ -522,6 +589,48 @@ struct GetChunkTask : public Task {
 	std::unique_ptr<duckdb::DataChunk> chunk;
 };
 
+struct GetNextArrowIpcTask : public Task {
+	GetNextArrowIpcTask(QueryResult &query_result, Napi::Promise::Deferred deferred)
+	    : Task(query_result), deferred(deferred) {
+	}
+
+	void DoWork() override {
+		auto &query_result = Get<QueryResult>();
+		chunk = query_result.result->Fetch();
+	}
+
+	void DoCallback() override {
+		auto &query_result = Get<QueryResult>();
+		Napi::Env env = query_result.Env();
+		Napi::HandleScope scope(env);
+
+		if (chunk == nullptr || chunk->size() == 0) {
+			deferred.Resolve(env.Null());
+			return;
+		}
+
+		// Arrow IPC streams should be a single column of a single blob
+		D_ASSERT(chunk->size() == 1 && chunk->ColumnCount() == 2);
+		D_ASSERT(chunk->data[0].GetType() == duckdb::LogicalType::BLOB);
+
+		duckdb::string_t blob = *(duckdb::string_t *)(chunk->data[0].GetData());
+
+		// Transfer ownership and Construct ArrayBuffer
+		auto data_chunk_ptr = new std::unique_ptr<duckdb::DataChunk>();
+		*data_chunk_ptr = std::move(chunk);
+		auto deleter = [](Napi::Env, void *finalizeData, void *hint) {
+			delete static_cast<std::unique_ptr<duckdb::DataChunk> *>(hint);
+		};
+		auto array_buffer =
+		    Napi::ArrayBuffer::New(env, (void *)blob.GetDataUnsafe(), blob.GetSize(), deleter, data_chunk_ptr);
+
+		deferred.Resolve(array_buffer);
+	}
+
+	Napi::Promise::Deferred deferred;
+	std::unique_ptr<duckdb::DataChunk> chunk;
+};
+
 Napi::Value QueryResult::NextChunk(const Napi::CallbackInfo &info) {
 	auto env = info.Env();
 	auto deferred = Napi::Promise::Deferred::New(env);
@@ -530,4 +639,12 @@ Napi::Value QueryResult::NextChunk(const Napi::CallbackInfo &info) {
 	return deferred.Promise();
 }
 
+// Should only be called on an arrow ipc query
+Napi::Value QueryResult::NextIpcBuffer(const Napi::CallbackInfo &info) {
+	auto env = info.Env();
+	auto deferred = Napi::Promise::Deferred::New(env);
+	database_ref->Schedule(env, duckdb::make_unique<GetNextArrowIpcTask>(*this, deferred));
+	return deferred.Promise();
+}
+
 } // namespace node_duckdb

package/test/arrow.test.js

@@ -0,0 +1,61 @@
+var sqlite3 = require('..');
+var assert = require('assert');
+var fs = require('fs');
+
+describe('exec', function() {
+    var db;
+    before(function(done) {
+        db = new sqlite3.Database(':memory:', done);
+    });
+
+    // Note: arrow IPC api requires the arrow extension to be loaded. The tests for this functionality reside in:
+    //       https://github.com/duckdblabs/arrow
+    describe(`Arrow IPC API fails neatly when extension not loaded`, () => {
+        let db;
+        let conn;
+        before((done) => {
+            db = new sqlite3.Database(':memory:', {"allow_unsigned_extensions": "true"}, () => {
+                done();
+            });
+        });
+
+        it(`Basic examples`, async () => {
+            const range_size = 130000;
+            const query = `SELECT * FROM range(0,${range_size}) tbl(i)`;
+
+            db.arrowIPCStream(query).then(
+                () => Promise.reject(new Error('Expected method to reject.')),
+                err => {
+                    assert(err.message.includes("Catalog Error: Function with name to_arrow_ipc is not on the catalog, but it exists in the arrow extension. To Install and Load the extension, run: INSTALL arrow; LOAD arrow;"))
+                }
+            );
+
+            db.arrowIPCAll(`SELECT * FROM ipc_table`, function (err, result) {
+                if (err) {
+                    assert(err.message.includes("Catalog Error: Function with name to_arrow_ipc is not on the catalog, but it exists in the arrow extension. To Install and Load the extension, run: INSTALL arrow; LOAD arrow;"))
+                } else {
+                    assert.fail("Expected error");
+                }
+            });
+
+            assert.throws(() => db.register_buffer("ipc_table", [1,'a',1], true), TypeError, "Incorrect parameters");
+        });
+
+        it('Register buffer should be disabled currently', function(done) {
+            try {
+                db.register_buffer();
+                assert(0);
+            } catch (error) {
+                assert(error.message.includes('Register buffer currently not implemented'))
+            }
+
+            try {
+                db.unregister_buffer();
+                assert(0);
+            } catch (error) {
+                assert(error.message.includes('Register buffer currently not implemented'))
+            }
+            done()
+        });
+    });
+});

package/test/data_type_support.test.js

@@ -188,4 +188,10 @@ describe("data type support", function () {
       done();
     });
   });
+  it("converts unsupported data types to strings", function(done) {
+      db.all("SELECT CAST('11:10:10' AS TIME) as time", function(err, rows) {
+          assert.equal(rows[0].time, '11:10:10');
+          done();
+      });
+  });
 });

package/test/open_close.test.js

@@ -150,13 +150,13 @@ describe('open/close', function() {
         });
     });
 
-    describe('closing with unfinalized statements', function(done) {
+    describe('closing with unfinalized statements', function() {
         var completed = false;
         var completedSecond = false;
         var closed = false;
 
         var db;
-        before(function() {
+        before(function(done) {
             db = new sqlite3.Database(':memory:', done);
         });
 

package/test/support/createdb.js

@@ -24,17 +24,19 @@ function createdb(callback) {
         if (callback) callback();
     } else {
         console.log("Creating test database... This may take several minutes.");
-        var db = new sqlite3.Database(db_path);
-        db.serialize(function() {
-            db.run("CREATE TABLE foo (id INT, txt TEXT)");
-            db.run("BEGIN TRANSACTION");
-            var stmt = db.prepare("INSERT INTO foo VALUES(?, ?)");
-            for (var i = 0; i < count; i++) {
-                stmt.run(i, randomString());
-            }
-            stmt.finalize();
-            db.run("COMMIT TRANSACTION", function () {
-                db.close(callback);
+        var db = new sqlite3.Database(db_path, () => {
+            db.serialize(function() {
+                db.run("CREATE TABLE foo (id INT, txt TEXT)");
+                db.run("BEGIN TRANSACTION");
+                var stmt = db.prepare("INSERT INTO foo VALUES(?, ?)");
+                for (var i = 0; i < count; i++) {
+                    stmt.run(i, randomString());
+                }
+
+                stmt.finalize();
+                db.run("COMMIT TRANSACTION", function () {
+                    db.close(callback);
+                });
             });
         });
     }

package/test/typescript_decls.test.ts

@@ -5,7 +5,7 @@ import fs from "fs";
 describe("TypeScript declarataions", function () {
   var db: duckdb.Database;
   before(function (done) {
-    db = new duckdb.Database(":memory:", done);
+    db = new duckdb.Database(":memory:", duckdb.OPEN_READWRITE, done);
   });
 
   it("typescript: Database constructor no callback", (done) => {
@@ -13,17 +13,39 @@ describe("TypeScript declarataions", function () {
     done();
   });
 
+  it("Database.create -- read only flag", (done) => {
+    const roDb = new duckdb.Database(
+      ":memory:",
+      duckdb.OPEN_READONLY,
+      (err: duckdb.DuckDbError | null, res: any) => {
+        assert.equal(err?.code, "DUCKDB_NODEJS_ERROR");
+        assert.equal(err?.errno, -1);
+        const errMessage: string = err?.message ?? "";
+        assert(
+          errMessage.includes(
+            "Cannot launch in-memory database in read-only mode"
+          )
+        );
+        done();
+      }
+    );
+  });
+
   it("typescript: Database constructor path error", (done) => {
-    const tdb0 = new duckdb.Database("./bogusPath.db", (err, res) => {
-      // Issue: I'm a little surprised that specifying an invalid file path
-      // doesn't seem to immediately signal an error here, but it doesn't.
-      tdb0.all(
-        "PRAGMA show_tables",
-        (err: duckdb.DuckDbError | null, res: any) => {
-          done();
-        }
-      );
-    });
+    const tdb0 = new duckdb.Database(
+      "./bogusPath.db",
+      duckdb.OPEN_READWRITE,
+      (err, res) => {
+        // Issue: I'm a little surprised that specifying an invalid file path
+        // doesn't seem to immediately signal an error here, but it doesn't.
+        tdb0.all(
+          "PRAGMA show_tables",
+          (err: duckdb.DuckDbError | null, res: any) => {
+            done();
+          }
+        );
+      }
+    );
   });
 
   it("typescript: query with error", (done) => {
@@ -95,7 +117,7 @@ describe("TypeScript declarataions", function () {
   });
 
   it("typescript: ternary int udf", function (done) {
-    db.register(
+    db.register_udf(
       "udf",
       "integer",
       (x: number, y: number, z: number) => x + y + z
@@ -107,7 +129,7 @@ describe("TypeScript declarataions", function () {
         assert.equal(rows[0].v, 42);
       }
     );
-    db.unregister("udf", done);
+    db.unregister_udf("udf", done);
   });
   it("typescript: retrieve 100,000 rows with Statement#each", function (done) {
     var total = 100000;
@@ -136,7 +158,7 @@ describe("TypeScript declarataions", function () {
 describe("typescript: simple prepared statement", function () {
   var db: duckdb.Database;
   before(function (done) {
-    db = new duckdb.Database(":memory:", done);
+    db = new duckdb.Database(":memory:", duckdb.OPEN_READWRITE, done);
   });
 
   it("should prepare, run and finalize the statement", function (done) {
@@ -151,7 +173,7 @@ describe("typescript: simple prepared statement", function () {
 describe("typescript: prepared statements", function () {
   var db: duckdb.Database;
   before(function (done) {
-    db = new duckdb.Database(":memory:", done);
+    db = new duckdb.Database(":memory:", duckdb.OPEN_READWRITE, done);
   });
 
   var inserted = 0;
@@ -193,7 +215,7 @@ describe("typescript: stream and QueryResult", function () {
   let db: duckdb.Database;
   let conn: duckdb.Connection;
   before((done) => {
-    db = new duckdb.Database(":memory:", () => {
+    db = new duckdb.Database(":memory:", duckdb.OPEN_READWRITE, () => {
       conn = new duckdb.Connection(db, done);
     });
   });