duckdb 0.5.2-dev2.0 → 0.5.2-dev2001.0

package/src/parquet-amalgamation.hpp

@@ -7541,6 +7541,7 @@ class ParquetReader;
 using duckdb_apache::thrift::protocol::TProtocol;
 
 using duckdb_parquet::format::ColumnChunk;
+using duckdb_parquet::format::CompressionCodec;
 using duckdb_parquet::format::FieldRepetitionType;
 using duckdb_parquet::format::PageHeader;
 using duckdb_parquet::format::SchemaElement;
@@ -7558,7 +7559,7 @@ public:
 	static unique_ptr<ColumnReader> CreateReader(ParquetReader &reader, const LogicalType &type_p,
 	                                             const SchemaElement &schema_p, idx_t schema_idx_p, idx_t max_define,
 	                                             idx_t max_repeat);
-	virtual void InitializeRead(const std::vector<ColumnChunk> &columns, TProtocol &protocol_p);
+	virtual void InitializeRead(idx_t row_group_index, const std::vector<ColumnChunk> &columns, TProtocol &protocol_p);
 	virtual idx_t Read(uint64_t num_values, parquet_filter_t &filter, uint8_t *define_out, uint8_t *repeat_out,
 	                   Vector &result_out);
 
@@ -7578,7 +7579,7 @@ public:
 	// register the range this reader will touch for prefetching
 	virtual void RegisterPrefetch(ThriftFileTransport &transport, bool allow_merge);
 
-	virtual unique_ptr<BaseStatistics> Stats(const std::vector<ColumnChunk> &columns);
+	virtual unique_ptr<BaseStatistics> Stats(idx_t row_group_idx_p, const std::vector<ColumnChunk> &columns);
 
 protected:
 	// readers that use the default Read() need to implement those
@@ -7617,9 +7618,10 @@ protected:
 
 private:
 	void PrepareRead(parquet_filter_t &filter);
-	void PreparePage(idx_t compressed_page_size, idx_t uncompressed_page_size);
+	void PreparePage(PageHeader &page_hdr);
 	void PrepareDataPage(PageHeader &page_hdr);
 	void PreparePageV2(PageHeader &page_hdr);
+	void DecompressInternal(CompressionCodec::type codec, const char *src, idx_t src_size, char *dst, idx_t dst_size);
 
 	const duckdb_parquet::format::ColumnChunk *chunk = nullptr;
 
@@ -7740,6 +7742,7 @@ struct ParquetOptions {
 
 	bool binary_as_string = false;
 	bool filename = false;
+	bool file_row_number = false;
 	bool hive_partitioning = false;
 
 public:

package/src/statement.cpp

@@ -4,6 +4,9 @@
 
 #include <algorithm>
 #include <cassert>
+#include <iostream>
+#include <string>
+#include <regex>
 
 namespace node_duckdb {
 
@@ -15,8 +18,8 @@ Napi::Object Statement::Init(Napi::Env env, Napi::Object exports) {
 	Napi::Function t =
 	    DefineClass(env, "Statement",
 	                {InstanceMethod("run", &Statement::Run), InstanceMethod("all", &Statement::All),
-	                 InstanceMethod("each", &Statement::Each), InstanceMethod("finalize", &Statement::Finish),
-	                 InstanceMethod("stream", &Statement::Stream)});
+	                 InstanceMethod("arrowIPCAll", &Statement::ArrowIPCAll), InstanceMethod("each", &Statement::Each),
+	                 InstanceMethod("finalize", &Statement::Finish), InstanceMethod("stream", &Statement::Stream)});
 
 	constructor = Napi::Persistent(t);
 	constructor.SuppressDestruct();
@@ -225,8 +228,7 @@ static Napi::Value convert_col_val(Napi::Env &env, duckdb::Value dval, duckdb::L
 		value = object_value;
 	} break;
 	default:
-		Napi::Error::New(env, "Data type is not supported " + dval.type().ToString()).ThrowAsJavaScriptException();
-		return env.Null();
+		value = Napi::String::New(env, dval.ToString());
 	}
 
 	return value;
@@ -255,7 +257,7 @@ static Napi::Value convert_chunk(Napi::Env &env, std::vector<std::string> names,
 	return scope.Escape(result);
 }
 
-enum RunType { RUN, EACH, ALL };
+enum RunType { RUN, EACH, ALL, ARROW_ALL };
 
 struct StatementParam {
 	std::vector<duckdb::Value> params;
@@ -275,7 +277,8 @@ struct RunPreparedTask : public Task {
 			return;
 		}
 
-		result = statement.statement->Execute(params->params, run_type != RunType::ALL);
+		result =
+		    statement.statement->Execute(params->params, run_type != RunType::ALL && run_type != RunType::ARROW_ALL);
 	}
 
 	void Callback() override {
@@ -350,6 +353,62 @@ struct RunPreparedTask : public Task {
 
 			cb.MakeCallback(statement.Value(), {env.Null(), result_arr});
 		} break;
+		case RunType::ARROW_ALL: {
+			auto materialized_result = (duckdb::MaterializedQueryResult *)result.get();
+			// +1 is for null bytes at end of stream
+			Napi::Array result_arr(Napi::Array::New(env, materialized_result->RowCount() + 1));
+
+			auto deleter = [](Napi::Env, void *finalizeData, void *hint) {
+				delete static_cast<std::shared_ptr<duckdb::QueryResult> *>(hint);
+			};
+
+			std::shared_ptr<duckdb::QueryResult> result_ptr = move(result);
+
+			duckdb::idx_t out_idx = 1;
+			while (true) {
+				auto chunk = result_ptr->Fetch();
+
+				if (!chunk || chunk->size() == 0) {
+					break;
+				}
+
+				D_ASSERT(chunk->ColumnCount() == 2);
+				D_ASSERT(chunk->data[0].GetType() == duckdb::LogicalType::BLOB);
+				D_ASSERT(chunk->data[1].GetType() == duckdb::LogicalType::BOOLEAN);
+
+				for (duckdb::idx_t row_idx = 0; row_idx < chunk->size(); row_idx++) {
+					duckdb::string_t blob = ((duckdb::string_t *)(chunk->data[0].GetData()))[row_idx];
+					bool is_header = chunk->data[1].GetData()[row_idx];
+
+					// Create shared pointer to give (shared) ownership to ArrayBuffer, not that for these materialized
+					// query results, the string data is owned by the QueryResult
+					auto result_ref_ptr = new std::shared_ptr<duckdb::QueryResult>(result_ptr);
+
+					auto array_buffer = Napi::ArrayBuffer::New(env, (void *)blob.GetDataUnsafe(), blob.GetSize(),
+					                                           deleter, result_ref_ptr);
+
+					auto typed_array = Napi::Uint8Array::New(env, blob.GetSize(), array_buffer, 0);
+
+					// TODO we should handle this in duckdb probably
+					if (is_header) {
+						result_arr.Set((uint32_t)0, typed_array);
+					} else {
+						D_ASSERT(out_idx < materialized_result->RowCount());
+						result_arr.Set(out_idx++, typed_array);
+					}
+				}
+			}
+
+			// TODO we should handle this in duckdb probably
+			auto null_arr = Napi::Uint8Array::New(env, 4);
+			memset(null_arr.Data(), '\0', 4);
+			result_arr.Set(out_idx++, null_arr);
+
+			// Confirm all rows are set
+			D_ASSERT(out_idx == materialized_result->RowCount() + 1);
+
+			cb.MakeCallback(statement.Value(), {env.Null(), result_arr});
+		} break;
 		}
 	}
 	std::unique_ptr<duckdb::QueryResult> result;
@@ -424,6 +483,12 @@ Napi::Value Statement::All(const Napi::CallbackInfo &info) {
 	return info.This();
 }
 
+Napi::Value Statement::ArrowIPCAll(const Napi::CallbackInfo &info) {
+	connection_ref->database_ref->Schedule(
+	    info.Env(), duckdb::make_unique<RunPreparedTask>(*this, HandleArgs(info), RunType::ARROW_ALL));
+	return info.This();
+}
+
 Napi::Value Statement::Run(const Napi::CallbackInfo &info) {
 	connection_ref->database_ref->Schedule(info.Env(),
 	                                       duckdb::make_unique<RunPreparedTask>(*this, HandleArgs(info), RunType::RUN));
@@ -472,7 +537,9 @@ Napi::FunctionReference QueryResult::constructor;
 Napi::Object QueryResult::Init(Napi::Env env, Napi::Object exports) {
 	Napi::HandleScope scope(env);
 
-	Napi::Function t = DefineClass(env, "QueryResult", {InstanceMethod("nextChunk", &QueryResult::NextChunk)});
+	Napi::Function t = DefineClass(env, "QueryResult",
+	                               {InstanceMethod("nextChunk", &QueryResult::NextChunk),
+	                                InstanceMethod("nextIpcBuffer", &QueryResult::NextIpcBuffer)});
 
 	constructor = Napi::Persistent(t);
 	constructor.SuppressDestruct();
@@ -522,6 +589,48 @@ struct GetChunkTask : public Task {
 	std::unique_ptr<duckdb::DataChunk> chunk;
 };
 
+struct GetNextArrowIpcTask : public Task {
+	GetNextArrowIpcTask(QueryResult &query_result, Napi::Promise::Deferred deferred)
+	    : Task(query_result), deferred(deferred) {
+	}
+
+	void DoWork() override {
+		auto &query_result = Get<QueryResult>();
+		chunk = query_result.result->Fetch();
+	}
+
+	void DoCallback() override {
+		auto &query_result = Get<QueryResult>();
+		Napi::Env env = query_result.Env();
+		Napi::HandleScope scope(env);
+
+		if (chunk == nullptr || chunk->size() == 0) {
+			deferred.Resolve(env.Null());
+			return;
+		}
+
+		// Arrow IPC streams should be a single column of a single blob
+		D_ASSERT(chunk->size() == 1 && chunk->ColumnCount() == 2);
+		D_ASSERT(chunk->data[0].GetType() == duckdb::LogicalType::BLOB);
+
+		duckdb::string_t blob = *(duckdb::string_t *)(chunk->data[0].GetData());
+
+		// Transfer ownership and Construct ArrayBuffer
+		auto data_chunk_ptr = new std::unique_ptr<duckdb::DataChunk>();
+		*data_chunk_ptr = std::move(chunk);
+		auto deleter = [](Napi::Env, void *finalizeData, void *hint) {
+			delete static_cast<std::unique_ptr<duckdb::DataChunk> *>(hint);
+		};
+		auto array_buffer =
+		    Napi::ArrayBuffer::New(env, (void *)blob.GetDataUnsafe(), blob.GetSize(), deleter, data_chunk_ptr);
+
+		deferred.Resolve(array_buffer);
+	}
+
+	Napi::Promise::Deferred deferred;
+	std::unique_ptr<duckdb::DataChunk> chunk;
+};
+
 Napi::Value QueryResult::NextChunk(const Napi::CallbackInfo &info) {
 	auto env = info.Env();
 	auto deferred = Napi::Promise::Deferred::New(env);
@@ -530,4 +639,12 @@ Napi::Value QueryResult::NextChunk(const Napi::CallbackInfo &info) {
 	return deferred.Promise();
 }
 
+// Should only be called on an arrow ipc query
+Napi::Value QueryResult::NextIpcBuffer(const Napi::CallbackInfo &info) {
+	auto env = info.Env();
+	auto deferred = Napi::Promise::Deferred::New(env);
+	database_ref->Schedule(env, duckdb::make_unique<GetNextArrowIpcTask>(*this, deferred));
+	return deferred.Promise();
+}
+
 } // namespace node_duckdb

package/test/data_type_support.test.js

@@ -188,4 +188,10 @@ describe("data type support", function () {
       done();
     });
   });
+  it("converts unsupported data types to strings", function(done) {
+      db.all("SELECT CAST('11:10:10' AS TIME) as time", function(err, rows) {
+          assert.equal(rows[0].time, '11:10:10');
+          done();
+      });
+  });
 });

package/test/open_close.test.js

@@ -150,13 +150,13 @@ describe('open/close', function() {
         });
     });
 
-    describe('closing with unfinalized statements', function(done) {
+    describe('closing with unfinalized statements', function() {
         var completed = false;
         var completedSecond = false;
         var closed = false;
 
         var db;
-        before(function() {
+        before(function(done) {
             db = new sqlite3.Database(':memory:', done);
         });
 

package/test/support/createdb.js

@@ -24,17 +24,19 @@ function createdb(callback) {
         if (callback) callback();
     } else {
         console.log("Creating test database... This may take several minutes.");
-        var db = new sqlite3.Database(db_path);
-        db.serialize(function() {
-            db.run("CREATE TABLE foo (id INT, txt TEXT)");
-            db.run("BEGIN TRANSACTION");
-            var stmt = db.prepare("INSERT INTO foo VALUES(?, ?)");
-            for (var i = 0; i < count; i++) {
-                stmt.run(i, randomString());
-            }
-            stmt.finalize();
-            db.run("COMMIT TRANSACTION", function () {
-                db.close(callback);
+        var db = new sqlite3.Database(db_path, () => {
+            db.serialize(function() {
+                db.run("CREATE TABLE foo (id INT, txt TEXT)");
+                db.run("BEGIN TRANSACTION");
+                var stmt = db.prepare("INSERT INTO foo VALUES(?, ?)");
+                for (var i = 0; i < count; i++) {
+                    stmt.run(i, randomString());
+                }
+
+                stmt.finalize();
+                db.run("COMMIT TRANSACTION", function () {
+                    db.close(callback);
+                });
             });
         });
     }

package/test/typescript_decls.test.ts

@@ -0,0 +1,209 @@
+import * as duckdb from "..";
+import assert from "assert";
+import fs from "fs";
+
+describe("TypeScript declarataions", function () {
+  var db: duckdb.Database;
+  before(function (done) {
+    db = new duckdb.Database(":memory:", done);
+  });
+
+  it("typescript: Database constructor no callback", (done) => {
+    const tdb0 = new duckdb.Database(":memory:"); // no callback argument
+    done();
+  });
+
+  it("typescript: Database constructor path error", (done) => {
+    const tdb0 = new duckdb.Database("./bogusPath.db", (err, res) => {
+      // Issue: I'm a little surprised that specifying an invalid file path
+      // doesn't seem to immediately signal an error here, but it doesn't.
+      tdb0.all(
+        "PRAGMA show_tables",
+        (err: duckdb.DuckDbError | null, res: any) => {
+          done();
+        }
+      );
+    });
+  });
+
+  it("typescript: query with error", (done) => {
+    // query with an error:
+    db.all(
+      "SELECT * FROM sometable",
+      (err: duckdb.DuckDbError | null, res: any) => {
+        assert.equal(err?.code, "DUCKDB_NODEJS_ERROR");
+        assert.equal(err?.errno, -1);
+        done();
+      }
+    );
+  });
+
+  it("typescript: Database#exec", function (done) {
+    var sql = fs.readFileSync("test/support/script.sql", "utf8");
+    db.exec(sql, function (err: duckdb.DuckDbError | null) {
+      if (err) throw err;
+      done();
+    });
+  });
+
+  it("typescript: retrieve database structure", function (done) {
+    db.all(
+      "SELECT type, name FROM sqlite_master ORDER BY type, name",
+      function (err: duckdb.DuckDbError | null, rows: duckdb.TableData) {
+        if (err) throw err;
+        assert.deepEqual(rows, [
+          { type: "table", name: "grid_key" },
+          { type: "table", name: "grid_utfgrid" },
+          { type: "table", name: "images" },
+          { type: "table", name: "keymap" },
+          { type: "table", name: "map" },
+          { type: "table", name: "metadata" },
+          { type: "view", name: "grid_data" },
+          { type: "view", name: "grids" },
+          { type: "view", name: "tiles" },
+        ]);
+        done();
+      }
+    );
+  });
+
+  it("typescript: database#all with no callback", (done) => {
+    db.all("select 42 as x");
+    done();
+  });
+
+  it("typescript: database#connect", (done) => {
+    const conn = db.connect();
+    assert(conn instanceof duckdb.Connection);
+    done();
+  });
+
+  it("typescript: ensure empty results work ok", (done) => {
+    db.all(
+      "create table test_table as select 42 as x",
+      (err: duckdb.DuckDbError | null, res: duckdb.TableData) => {
+        db.all(
+          "drop table test_table",
+          (err: duckdb.DuckDbError | null, res: duckdb.TableData) => {
+            console.log("drop table results: ", err, res);
+            assert.deepEqual(res, []);
+            done();
+          }
+        );
+      }
+    );
+  });
+
+  it("typescript: ternary int udf", function (done) {
+    db.register_udf(
+      "udf",
+      "integer",
+      (x: number, y: number, z: number) => x + y + z
+    );
+    db.all(
+      "select udf(21, 20, 1) v",
+      function (err: duckdb.DuckDbError | null, rows: duckdb.TableData) {
+        if (err) throw err;
+        assert.equal(rows[0].v, 42);
+      }
+    );
+    db.unregister_udf("udf", done);
+  });
+  it("typescript: retrieve 100,000 rows with Statement#each", function (done) {
+    var total = 100000;
+    var retrieved = 0;
+
+    db.each(
+      "SELECT * FROM range(0, ?)",
+      total,
+      function (err: duckdb.DuckDbError | null, row: any) {
+        if (err) throw err;
+        retrieved++;
+
+        if (retrieved === total) {
+          assert.equal(
+            retrieved,
+            total,
+            "Only retrieved " + retrieved + " out of " + total + " rows."
+          );
+          done();
+        }
+      }
+    );
+  });
+});
+
+describe("typescript: simple prepared statement", function () {
+  var db: duckdb.Database;
+  before(function (done) {
+    db = new duckdb.Database(":memory:", done);
+  });
+
+  it("should prepare, run and finalize the statement", function (done) {
+    db.prepare("CREATE TABLE foo (bar text)").run().finalize(done);
+  });
+
+  after(function (done) {
+    db.close(done);
+  });
+});
+
+describe("typescript: prepared statements", function () {
+  var db: duckdb.Database;
+  before(function (done) {
+    db = new duckdb.Database(":memory:", done);
+  });
+
+  var inserted = 0;
+  var retrieved = 0;
+
+  // We insert and retrieve that many rows.
+  var count = 1000;
+
+  it("typescript: should create the table", function (done) {
+    db.prepare("CREATE TABLE foo (txt text, num int, flt double, blb blob)")
+      .run()
+      .finalize(done);
+  });
+
+  it("typescript: should insert " + count + " rows", function (done) {
+    for (var i = 0; i < count; i++) {
+      db.prepare("INSERT INTO foo VALUES(?, ?, ?, ?)")
+        .run(
+          "String " + i,
+          i,
+          i * Math.PI,
+          null,
+          function (err: duckdb.DuckDbError | null) {
+            if (err) throw err;
+            inserted++;
+          }
+        )
+        .finalize(function (err) {
+          if (err) throw err;
+          if (inserted == count) done();
+        });
+    }
+  });
+});
+
+describe("typescript: stream and QueryResult", function () {
+  const total = 1000;
+
+  let db: duckdb.Database;
+  let conn: duckdb.Connection;
+  before((done) => {
+    db = new duckdb.Database(":memory:", () => {
+      conn = new duckdb.Connection(db, done);
+    });
+  });
+
+  it("streams results", async () => {
+    let retrieved = 0;
+    const stream = conn.stream("SELECT * FROM range(0, ?)", total);
+    for await (const row of stream) {
+      retrieved++;
+    }
+    assert.equal(total, retrieved);
+  });
+});