duckdb 0.6.2-dev1060.0 → 0.6.2-dev1070.0

package/lib/duckdb.d.ts

@@ -61,6 +61,15 @@ export class IpcResultStreamIterator implements AsyncIterator<Uint8Array>, Async
   toArray(): Promise<ArrowArray>;
 }
 
+export interface ReplacementScanResult {
+  function: string;
+  parameters: Array<unknown>;
+}
+
+export type ReplacementScanCallback = (
+  table: string
+) => ReplacementScanResult | null;
+
 export class Database {
   constructor(path: string, accessMode?: number | Record<string,string>, callback?: Callback<any>);
   constructor(path: string, callback?: Callback<any>);
@@ -99,6 +108,10 @@ export class Database {
   register_buffer(name: string, array: ArrowIterable, force: boolean, callback?: Callback<void>): void;
 
   unregister_buffer(name: string, callback?: Callback<void>): void;
+
+  registerReplacementScan(
+    replacementScan: ReplacementScanCallback
+  ): Promise<void>;
 }
 
 export class Statement {

package/lib/duckdb.js

@@ -609,6 +609,15 @@ Database.prototype.unregister_udf = function () {
     return this;
 }
 
+/**
+ * Register a table replace scan function
+ * @method
+ * @arg fun Replacement scan function
+ * @return {this}
+ */
+
+Database.prototype.registerReplacementScan;
+
 /**
  * Not implemented
  */

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.6.2-dev1060.0",
+  "version": "0.6.2-dev1070.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {
@@ -26,6 +26,7 @@
     "test": "test"
   },
   "devDependencies": {
+    "@types/chai": "^4.3.4",
     "@types/mocha": "^10.0.0",
     "@types/node": "^18.11.0",
     "apache-arrow": "^9.0.0",

package/src/database.cpp

@@ -1,7 +1,13 @@
-#include "duckdb_node.hpp"
+#include "duckdb/parser/expression/constant_expression.hpp"
+#include "duckdb/parser/expression/function_expression.hpp"
+#include "duckdb/parser/tableref/table_function_ref.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
+#include "duckdb_node.hpp"
 #include "napi.h"
 
+#include <iostream>
+#include <thread>
+
 namespace node_duckdb {
 
 Napi::FunctionReference Database::constructor;
@@ -13,7 +19,8 @@ Napi::Object Database::Init(Napi::Env env, Napi::Object exports) {
 	    env, "Database",
 	    {InstanceMethod("close_internal", &Database::Close), InstanceMethod("wait", &Database::Wait),
 	     InstanceMethod("serialize", &Database::Serialize), InstanceMethod("parallelize", &Database::Parallelize),
-	     InstanceMethod("connect", &Database::Connect), InstanceMethod("interrupt", &Database::Interrupt)});
+	     InstanceMethod("connect", &Database::Connect), InstanceMethod("interrupt", &Database::Interrupt),
+	     InstanceMethod("registerReplacementScan", &Database::RegisterReplacementScan)});
 
 	constructor = Napi::Persistent(t);
 	constructor.SuppressDestruct();
@@ -267,4 +274,107 @@ Napi::Value Database::Connect(const Napi::CallbackInfo &info) {
 	return Connection::constructor.New({Value()});
 }
 
+struct JSRSArgs {
+	std::string table = "";
+	std::string function = "";
+	std::vector<duckdb::Value> parameters;
+	bool done = false;
+	duckdb::PreservedError error;
+};
+
+struct NodeReplacementScanData : duckdb::ReplacementScanData {
+	NodeReplacementScanData(duckdb_node_rs_function_t rs) : rs(std::move(rs)) {};
+	duckdb_node_rs_function_t rs;
+};
+
+void DuckDBNodeRSLauncher(Napi::Env env, Napi::Function jsrs, std::nullptr_t *, JSRSArgs *jsargs) {
+	try {
+		Napi::EscapableHandleScope scope(env);
+		auto arg = Napi::String::New(env, jsargs->table);
+		auto result = jsrs({arg});
+		if (result && result.IsObject()) {
+			auto obj = result.As<Napi::Object>();
+			jsargs->function = obj.Get("function").ToString().Utf8Value();
+			auto parameters = obj.Get("parameters");
+			if (parameters.IsArray()) {
+				auto paramArray = parameters.As<Napi::Array>();
+				for (uint32_t i = 0; i < paramArray.Length(); i++) {
+					jsargs->parameters.push_back(Utils::BindParameter(paramArray.Get(i)));
+				}
+			} else {
+				throw duckdb::Exception("Expected parameter array");
+			}
+		} else if (!result.IsNull()) {
+			throw duckdb::Exception("Invalid scan replacement result");
+		}
+	} catch (const duckdb::Exception &e) {
+		jsargs->error = duckdb::PreservedError(e);
+	} catch (const std::exception &e) {
+		jsargs->error = duckdb::PreservedError(e);
+	}
+	jsargs->done = true;
+}
+
+static duckdb::unique_ptr<duckdb::TableFunctionRef>
+ScanReplacement(duckdb::ClientContext &context, const std::string &table_name, duckdb::ReplacementScanData *data) {
+	JSRSArgs jsargs;
+	jsargs.table = table_name;
+	((NodeReplacementScanData *)data)->rs.BlockingCall(&jsargs);
+	while (!jsargs.done) {
+		std::this_thread::yield();
+	}
+	if (jsargs.error) {
+		jsargs.error.Throw();
+	}
+	if (jsargs.function != "") {
+		auto table_function = duckdb::make_unique<duckdb::TableFunctionRef>();
+		std::vector<std::unique_ptr<duckdb::ParsedExpression>> children;
+		for (auto &param : jsargs.parameters) {
+			children.push_back(duckdb::make_unique<duckdb::ConstantExpression>(std::move(param)));
+		}
+		table_function->function =
+		    duckdb::make_unique<duckdb::FunctionExpression>(jsargs.function, std::move(children));
+		return table_function;
+	}
+	return nullptr;
+}
+
+struct RegisterRsTask : public Task {
+	RegisterRsTask(Database &database, duckdb_node_rs_function_t rs, Napi::Promise::Deferred deferred)
+	    : Task(database), rs(std::move(rs)), deferred(deferred) {
+	}
+
+	void DoWork() override {
+		auto &database = Get<Database>();
+		if (database.database) {
+			database.database->instance->config.replacement_scans.emplace_back(
+			    ScanReplacement, duckdb::make_unique<NodeReplacementScanData>(rs));
+		}
+	}
+
+	void DoCallback() override {
+		deferred.Resolve(deferred.Env().Undefined());
+	}
+
+	duckdb_node_rs_function_t rs;
+	Napi::Promise::Deferred deferred;
+};
+
+Napi::Value Database::RegisterReplacementScan(const Napi::CallbackInfo &info) {
+	auto env = info.Env();
+	auto deferred = Napi::Promise::Deferred::New(info.Env());
+	if (info.Length() < 1) {
+		Napi::TypeError::New(env, "Replacement scan callback expected").ThrowAsJavaScriptException();
+		return env.Null();
+	}
+	Napi::Function rs_callback = info[0].As<Napi::Function>();
+	auto rs = duckdb_node_rs_function_t::New(env, rs_callback, "duckdb_node_rs_" + (replacement_scan_count++), 0, 1,
+	                                         nullptr, [](Napi::Env, void *, std::nullptr_t *ctx) {});
+	rs.Unref(env);
+
+	Schedule(info.Env(), duckdb::make_unique<RegisterRsTask>(*this, rs, deferred));
+
+	return deferred.Promise();
+}
+
 } // namespace node_duckdb

package/src/duckdb/extension/icu/icu-dateadd.cpp

@@ -58,7 +58,20 @@ timestamp_t ICUCalendarAdd::Operation(timestamp_t timestamp, interval_t interval
 	calendar->setTime(udate, status);
 
 	// Add interval fields from lowest to highest
-	calendar->add(UCAL_MILLISECOND, interval.micros / Interval::MICROS_PER_MSEC, status);
+
+	// Break units apart to avoid overflow
+	auto remaining = interval.micros / Interval::MICROS_PER_MSEC;
+	calendar->add(UCAL_MILLISECOND, remaining % Interval::MSECS_PER_SEC, status);
+
+	remaining /= Interval::MSECS_PER_SEC;
+	calendar->add(UCAL_SECOND, remaining % Interval::SECS_PER_MINUTE, status);
+
+	remaining /= Interval::SECS_PER_MINUTE;
+	calendar->add(UCAL_MINUTE, remaining % Interval::MINS_PER_HOUR, status);
+
+	remaining /= Interval::MINS_PER_HOUR;
+	calendar->add(UCAL_HOUR, remaining, status);
+
 	calendar->add(UCAL_DATE, interval.days, status);
 	calendar->add(UCAL_MONTH, interval.months, status);
 

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.6.2-dev1060"
+#define DUCKDB_VERSION "0.6.2-dev1070"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "b3f6a8f16d"
+#define DUCKDB_SOURCE_ID "e0cff6a3dd"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb_node.hpp

@@ -56,6 +56,11 @@ struct Task {
 
 class Connection;
 
+struct JSRSArgs;
+void DuckDBNodeRSLauncher(Napi::Env env, Napi::Function jsrs, std::nullptr_t *, JSRSArgs *data);
+
+typedef Napi::TypedThreadSafeFunction<std::nullptr_t, JSRSArgs, DuckDBNodeRSLauncher> duckdb_node_rs_function_t;
+
 class Database : public Napi::ObjectWrap<Database> {
 public:
 	explicit Database(const Napi::CallbackInfo &info);
@@ -83,6 +88,7 @@ public:
 	Napi::Value Parallelize(const Napi::CallbackInfo &info);
 	Napi::Value Interrupt(const Napi::CallbackInfo &info);
 	Napi::Value Close(const Napi::CallbackInfo &info);
+	Napi::Value RegisterReplacementScan(const Napi::CallbackInfo &info);
 
 public:
 	constexpr static int DUCKDB_NODEJS_ERROR = -1;
@@ -97,6 +103,7 @@ private:
 	static Napi::FunctionReference constructor;
 	Napi::Env env;
 	int64_t bytes_allocated = 0;
+	int replacement_scan_count = 0;
 };
 
 struct JSArgs;
@@ -199,6 +206,7 @@ public:
 		auto obj = T::constructor.New(args);
 		return Napi::ObjectWrap<T>::Unwrap(obj);
 	}
+	static duckdb::Value BindParameter(const Napi::Value source);
 };
 
 Napi::Array EncodeDataChunk(Napi::Env env, duckdb::DataChunk &chunk, bool with_types, bool with_data);

package/src/statement.cpp

@@ -85,51 +85,6 @@ Statement::~Statement() {
 	connection_ref = nullptr;
 }
 
-// A Napi InstanceOf for Javascript Objects "Date" and "RegExp"
-static bool OtherInstanceOf(Napi::Object source, const char *object_type) {
-	if (strcmp(object_type, "Date") == 0) {
-		return source.InstanceOf(source.Env().Global().Get(object_type).As<Napi::Function>());
-	} else if (strcmp(object_type, "RegExp") == 0) {
-		return source.InstanceOf(source.Env().Global().Get(object_type).As<Napi::Function>());
-	}
-
-	return false;
-}
-
-static duckdb::Value BindParameter(const Napi::Value source) {
-	if (source.IsString()) {
-		return duckdb::Value(source.As<Napi::String>().Utf8Value());
-	} else if (OtherInstanceOf(source.As<Napi::Object>(), "RegExp")) {
-		return duckdb::Value(source.ToString().Utf8Value());
-	} else if (source.IsNumber()) {
-		if (Utils::OtherIsInt(source.As<Napi::Number>())) {
-			return duckdb::Value::INTEGER(source.As<Napi::Number>().Int32Value());
-		} else {
-			return duckdb::Value::DOUBLE(source.As<Napi::Number>().DoubleValue());
-		}
-	} else if (source.IsBoolean()) {
-		return duckdb::Value::BOOLEAN(source.As<Napi::Boolean>().Value());
-	} else if (source.IsNull()) {
-		return duckdb::Value();
-	} else if (source.IsBuffer()) {
-		Napi::Buffer<char> buffer = source.As<Napi::Buffer<char>>();
-		return duckdb::Value::BLOB(std::string(buffer.Data(), buffer.Length()));
-#if (NAPI_VERSION > 4)
-	} else if (source.IsDate()) {
-		const auto micros = int64_t(source.As<Napi::Date>().ValueOf()) * duckdb::Interval::MICROS_PER_MSEC;
-		if (micros % duckdb::Interval::MICROS_PER_DAY) {
-			return duckdb::Value::TIMESTAMP(duckdb::timestamp_t(micros));
-		} else {
-			const auto days = int32_t(micros / duckdb::Interval::MICROS_PER_DAY);
-			return duckdb::Value::DATE(duckdb::date_t(days));
-		}
-#endif
-	} else if (source.IsObject()) {
-		return duckdb::Value(source.ToString().Utf8Value());
-	}
-	return duckdb::Value();
-}
-
 static Napi::Value convert_col_val(Napi::Env &env, duckdb::Value dval, duckdb::LogicalTypeId id) {
 	Napi::Value value;
 
@@ -472,7 +427,7 @@ duckdb::unique_ptr<StatementParam> Statement::HandleArgs(const Napi::CallbackInf
 		if (p.IsUndefined()) {
 			continue;
 		}
-		params->params.push_back(BindParameter(p));
+		params->params.push_back(Utils::BindParameter(p));
 	}
 	return params;
 }

package/src/utils.cpp

@@ -20,5 +20,47 @@ Napi::Value Utils::CreateError(Napi::Env env, std::string msg) {
 
 	return obj;
 }
+// A Napi InstanceOf for Javascript Objects "Date" and "RegExp"
+static bool OtherInstanceOf(Napi::Object source, const char *object_type) {
+	if (strcmp(object_type, "Date") == 0) {
+		return source.InstanceOf(source.Env().Global().Get(object_type).As<Napi::Function>());
+	} else if (strcmp(object_type, "RegExp") == 0) {
+		return source.InstanceOf(source.Env().Global().Get(object_type).As<Napi::Function>());
+	}
 
+	return false;
+}
+duckdb::Value Utils::BindParameter(const Napi::Value source) {
+	if (source.IsString()) {
+		return duckdb::Value(source.As<Napi::String>().Utf8Value());
+	} else if (OtherInstanceOf(source.As<Napi::Object>(), "RegExp")) {
+		return duckdb::Value(source.ToString().Utf8Value());
+	} else if (source.IsNumber()) {
+		if (Utils::OtherIsInt(source.As<Napi::Number>())) {
+			return duckdb::Value::INTEGER(source.As<Napi::Number>().Int32Value());
+		} else {
+			return duckdb::Value::DOUBLE(source.As<Napi::Number>().DoubleValue());
+		}
+	} else if (source.IsBoolean()) {
+		return duckdb::Value::BOOLEAN(source.As<Napi::Boolean>().Value());
+	} else if (source.IsNull()) {
+		return duckdb::Value();
+	} else if (source.IsBuffer()) {
+		Napi::Buffer<char> buffer = source.As<Napi::Buffer<char>>();
+		return duckdb::Value::BLOB(std::string(buffer.Data(), buffer.Length()));
+#if (NAPI_VERSION > 4)
+	} else if (source.IsDate()) {
+		const auto micros = int64_t(source.As<Napi::Date>().ValueOf()) * duckdb::Interval::MICROS_PER_MSEC;
+		if (micros % duckdb::Interval::MICROS_PER_DAY) {
+			return duckdb::Value::TIMESTAMP(duckdb::timestamp_t(micros));
+		} else {
+			const auto days = int32_t(micros / duckdb::Interval::MICROS_PER_DAY);
+			return duckdb::Value::DATE(duckdb::date_t(days));
+		}
+#endif
+	} else if (source.IsObject()) {
+		return duckdb::Value(source.ToString().Utf8Value());
+	}
+	return duckdb::Value();
+}
 } // namespace node_duckdb

package/test/replacement_scan.test.ts

@@ -0,0 +1,144 @@
+import * as sqlite3 from "../lib/duckdb";
+import type { TableData } from "../lib/duckdb";
+import { expect } from "chai";
+
+const replacementScan = (table: string) => {
+  if (table.endsWith(".csv")) {
+    return null;
+  } else {
+    return {
+      function: "read_csv_auto",
+      parameters: [`test/support/${table}.csv`],
+    };
+  }
+};
+
+const invalidTableFunction = (table: string) => {
+  return {
+    function: "foo",
+    parameters: ["bar"],
+  };
+};
+
+const invalidResultType = (table: string) => {
+  return "hello" as unknown as sqlite3.ReplacementScanResult;
+};
+
+const invalidResultKeys = (table: string) => {
+  return {
+    foo: "foo",
+    bar: "bar",
+  } as unknown as sqlite3.ReplacementScanResult;
+};
+
+describe("replacement scan", () => {
+  var db: sqlite3.Database;
+  describe("without replacement scan", () => {
+    before(function (done) {
+      db = new sqlite3.Database(":memory:", done);
+    });
+
+    it("is not found", (done) => {
+      db.all(
+        "SELECT * FROM 'prepare' LIMIT 5",
+        function (err: null | Error, rows: TableData) {
+          expect(err).not.to.be.null;
+          expect(err!.message).to.match(
+            /Table with name prepare does not exist/
+          );
+          done();
+        }
+      );
+    });
+  });
+
+  describe("with replacement scan", () => {
+    before((done) => {
+      db = new sqlite3.Database(":memory:", () => {
+        db.registerReplacementScan(replacementScan).then(done);
+      });
+    });
+
+    it("is found when pattern matches", (done) => {
+      db.all(
+        "SELECT * FROM 'prepare' LIMIT 5",
+        function (err: null | Error, rows: TableData) {
+          expect(rows.length).to.equal(5);
+          done();
+        }
+      );
+    });
+
+    it("handles null response", (done) => {
+      db.all(
+        "SELECT * FROM 'test/support/prepare.csv' LIMIT 5",
+        function (err: null | Error, rows: TableData) {
+          expect(rows.length).to.equal(5);
+          done();
+        }
+      );
+    });
+
+    it("errors with invalid table", (done) => {
+      db.all(
+        "SELECT * FROM 'missing' LIMIT 5",
+        function (err: null | Error, rows: TableData) {
+          expect(err).not.to.be.null;
+          expect(err!.message).to.match(
+            /No files found that match the pattern "test\/support\/missing.csv"/
+          );
+          done();
+        }
+      );
+    });
+  });
+
+  describe("with invalid replacement scan functions", () => {
+    it("does not crash with bad return values", (done) => {
+      db = new sqlite3.Database(":memory:", () => {
+        db.registerReplacementScan(invalidTableFunction).then(() => {
+          db.all(
+            "SELECT * FROM 'missing' LIMIT 5",
+            function (err: null | Error, rows: TableData) {
+              expect(err).not.to.be.null;
+              expect(err!.message).to.match(
+                /Table Function with name foo does not exist/
+              );
+              done();
+            }
+          );
+        });
+      });
+    });
+
+    it("does not crash with invalid response", (done) => {
+      db = new sqlite3.Database(":memory:", () => {
+        db.registerReplacementScan(invalidResultType).then(() => {
+          db.all(
+            "SELECT * FROM 'missing' LIMIT 5",
+            function (err: null | Error, rows: TableData) {
+              expect(err).not.to.be.null;
+              expect(err!.message).to.match(/Invalid scan replacement result/);
+              done();
+            }
+          );
+        });
+      });
+    });
+
+    it("does not crash with invalid response object", (done) => {
+      db = new sqlite3.Database(":memory:", () => {
+        db.registerReplacementScan(invalidResultKeys).then(() => {
+          db.all(
+            "SELECT * FROM 'missing' LIMIT 5",
+            function (err: null | Error, rows: TableData) {
+              expect(err).not.to.be.null;
+              expect(err!.message).to.match(/Expected parameter array/);
+              done();
+            }
+          );
+        });
+      });
+    });
+  });
+});