duckdb 0.9.1 → 0.9.2-dev11.0

package/.github/workflows/HighPriorityIssues.yml

@@ -0,0 +1,36 @@
+name: Create Internal issue when the "High Priority" label is applied
+on:
+  issues:
+    types:
+      - labeled
+
+env:
+  GH_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }}
+  # an event triggering this workflow is either an issue or a pull request,
+  # hence only one of the numbers will be filled in the TITLE_PREFIX
+  TITLE_PREFIX: "[duckdb-node/#${{ github.event.issue.number }}]"
+  PUBLIC_ISSUE_TITLE: ${{ github.event.issue.title }}
+
+jobs:
+  create_or_label_issue:
+    if: github.event.label.name == 'High Priority'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get mirror issue number
+        run: |
+          gh issue list --repo duckdblabs/duckdb-internal --search "${TITLE_PREFIX}" --json title,number --jq ".[] | select(.title | startswith(\"$TITLE_PREFIX\")).number" > mirror_issue_number.txt
+          echo "MIRROR_ISSUE_NUMBER=$(cat mirror_issue_number.txt)" >> $GITHUB_ENV
+
+      - name: Print whether mirror issue exists
+        run: |
+          if [ "$MIRROR_ISSUE_NUMBER" == "" ]; then
+            echo "Mirror issue with title prefix '$TITLE_PREFIX' does not exist yet"
+          else
+            echo "Mirror issue with title prefix '$TITLE_PREFIX' exists with number $MIRROR_ISSUE_NUMBER"
+          fi
+
+      - name: Create or label issue
+        run: |
+          if [ "$MIRROR_ISSUE_NUMBER" == "" ]; then
+            gh issue create --repo duckdblabs/duckdb-internal --label "Node.js" --label "High Priority" --title "$TITLE_PREFIX - $PUBLIC_ISSUE_TITLE" --body "See https://github.com/duckdb/duckdb/issues/${{ github.event.issue.number }}"
+          fi

package/LICENSE

@@ -0,0 +1,7 @@
+Copyright 2018-2023 Stichting DuckDB Foundation
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/lib/duckdb.d.ts

@@ -78,6 +78,8 @@ export type ArrowArray = Uint8Array[];
 export class Connection {
   constructor(db: Database, callback?: Callback<any>);
 
+  close(callback?: Callback<void>): void;
+
   all(sql: string, ...args: [...any, Callback<TableData>] | []): void;
   arrowIPCAll(sql: string, ...args: [...any, Callback<ArrowArray>] | []): void;
   each(sql: string, ...args: [...any, Callback<RowData>] | []): void;
@@ -127,6 +129,20 @@ export type ReplacementScanCallback = (
   table: string
 ) => ReplacementScanResult | null;
 
+export enum TokenType {
+  IDENTIFIER = 0,
+  NUMERIC_CONSTANT = 1,
+  STRING_CONSTANT = 2,
+  OPERATOR = 3,
+  KEYWORD = 4,
+  COMMENT = 5,
+}
+
+export interface ScriptTokens {
+  offsets: number[];
+  types: TokenType[];
+}
+
 export class Database {
   constructor(path: string, accessMode?: number | Record<string,string>, callback?: Callback<any>);
   constructor(path: string, callback?: Callback<any>);
@@ -169,6 +185,8 @@ export class Database {
   registerReplacementScan(
     replacementScan: ReplacementScanCallback
   ): Promise<void>;
+
+  tokenize(text: string): ScriptTokens;
 }
 
 export type GenericTypeInfo = {

package/lib/duckdb.js

@@ -64,6 +64,10 @@ var Statement = duckdb.Statement;
  * @class
  */
 var QueryResult = duckdb.QueryResult;
+/**
+ * Types of tokens return by `tokenize`.
+ */
+var TokenType = duckdb.TokenType;
 
 /**
  * @method
@@ -631,6 +635,14 @@ Database.prototype.unregister_udf = function () {
 
 Database.prototype.registerReplacementScan;
 
+/**
+ * Return positions and types of tokens in given text
+ * @method
+ * @arg text
+ * @return {ScriptTokens}
+ */
+Database.prototype.tokenize;
+
 /**
  * Not implemented
  */

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.9.1",
+  "version": "0.9.2-dev11.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/database.cpp

@@ -1,5 +1,6 @@
 #include "duckdb/parser/expression/constant_expression.hpp"
 #include "duckdb/parser/expression/function_expression.hpp"
+#include "duckdb/parser/parser.hpp"
 #include "duckdb/parser/tableref/table_function_ref.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
 #include "duckdb_node.hpp"
@@ -18,7 +19,8 @@ Napi::FunctionReference Database::Init(Napi::Env env, Napi::Object exports) {
 	    {InstanceMethod("close_internal", &Database::Close), InstanceMethod("wait", &Database::Wait),
 	     InstanceMethod("serialize", &Database::Serialize), InstanceMethod("parallelize", &Database::Parallelize),
 	     InstanceMethod("connect", &Database::Connect), InstanceMethod("interrupt", &Database::Interrupt),
-	     InstanceMethod("registerReplacementScan", &Database::RegisterReplacementScan)});
+	     InstanceMethod("registerReplacementScan", &Database::RegisterReplacementScan),
+	     InstanceMethod("tokenize", &Database::Tokenize)});
 
 	exports.Set("Database", t);
 
@@ -364,4 +366,31 @@ Napi::Value Database::RegisterReplacementScan(const Napi::CallbackInfo &info) {
 	return deferred.Promise();
 }
 
+Napi::Value Database::Tokenize(const Napi::CallbackInfo &info) {
+	auto env = info.Env();
+
+	if (info.Length() < 1) {
+		throw Napi::TypeError::New(env, "Text argument expected");
+	}
+
+	std::string text = info[0].As<Napi::String>();
+
+	auto tokens = duckdb::Parser::Tokenize(text);
+	auto numTokens = tokens.size();
+
+	auto offsets = Napi::Array::New(env, numTokens);
+	auto types = Napi::Array::New(env, numTokens);
+
+	for (size_t i = 0; i < numTokens; i++) {
+		auto token = tokens[i];
+		offsets.Set(i, token.start);
+		types.Set(i, (uint8_t)token.type);
+	}
+
+	auto result = Napi::Object::New(env);
+	result.Set("offsets", offsets);
+	result.Set("types", types);
+	return result;
+}
+
 } // namespace node_duckdb

package/src/duckdb_node.cpp

@@ -12,15 +12,36 @@ NodeDuckDB::NodeDuckDB(Napi::Env env, Napi::Object exports) {
 	statement_constructor = node_duckdb::Statement::Init(env, exports);
 	query_result_constructor = node_duckdb::QueryResult::Init(env, exports);
 
-	exports.DefineProperties({
-	    DEFINE_CONSTANT_INTEGER(exports, node_duckdb::Database::DUCKDB_NODEJS_ERROR, ERROR) DEFINE_CONSTANT_INTEGER(
+	auto token_type_enum = Napi::Object::New(env);
+
+	token_type_enum.Set("IDENTIFIER", 0);
+	token_type_enum.Set("NUMERIC_CONSTANT", 1);
+	token_type_enum.Set("STRING_CONSTANT", 2);
+	token_type_enum.Set("OPERATOR", 3);
+	token_type_enum.Set("KEYWORD", 4);
+	token_type_enum.Set("COMMENT", 5);
+
+	// TypeScript enums expose an inverse mapping.
+	token_type_enum.Set((uint32_t)0, "IDENTIFIER");
+	token_type_enum.Set((uint32_t)1, "NUMERIC_CONSTANT");
+	token_type_enum.Set((uint32_t)2, "STRING_CONSTANT");
+	token_type_enum.Set((uint32_t)3, "OPERATOR");
+	token_type_enum.Set((uint32_t)4, "KEYWORD");
+	token_type_enum.Set((uint32_t)5, "COMMENT");
+
+	token_type_enum_ref = Napi::ObjectReference::New(token_type_enum);
+
+	exports.DefineProperties(
+	    {DEFINE_CONSTANT_INTEGER(exports, node_duckdb::Database::DUCKDB_NODEJS_ERROR, ERROR) DEFINE_CONSTANT_INTEGER(
 	        exports, node_duckdb::Database::DUCKDB_NODEJS_READONLY, OPEN_READONLY) // same as SQLite
-	    DEFINE_CONSTANT_INTEGER(exports, 0, OPEN_READWRITE)                        // ignored
-	    DEFINE_CONSTANT_INTEGER(exports, 0, OPEN_CREATE)                           // ignored
-	    DEFINE_CONSTANT_INTEGER(exports, 0, OPEN_FULLMUTEX)                        // ignored
-	    DEFINE_CONSTANT_INTEGER(exports, 0, OPEN_SHAREDCACHE)                      // ignored
-	    DEFINE_CONSTANT_INTEGER(exports, 0, OPEN_PRIVATECACHE)                     // ignored
-	});
+	     DEFINE_CONSTANT_INTEGER(exports, 0, OPEN_READWRITE)                       // ignored
+	     DEFINE_CONSTANT_INTEGER(exports, 0, OPEN_CREATE)                          // ignored
+	     DEFINE_CONSTANT_INTEGER(exports, 0, OPEN_FULLMUTEX)                       // ignored
+	     DEFINE_CONSTANT_INTEGER(exports, 0, OPEN_SHAREDCACHE)                     // ignored
+	     DEFINE_CONSTANT_INTEGER(exports, 0, OPEN_PRIVATECACHE)                    // ignored
+
+	     Napi::PropertyDescriptor::Value("TokenType", token_type_enum,
+	                                     static_cast<napi_property_attributes>(napi_enumerable | napi_configurable))});
 }
 
 NODE_API_ADDON(NodeDuckDB);

package/src/duckdb_node.hpp

@@ -23,6 +23,7 @@ public:
 	Napi::FunctionReference connection_constructor;
 	Napi::FunctionReference statement_constructor;
 	Napi::FunctionReference query_result_constructor;
+	Napi::ObjectReference token_type_enum_ref;
 };
 
 namespace node_duckdb {
@@ -109,6 +110,7 @@ public:
 	Napi::Value Interrupt(const Napi::CallbackInfo &info);
 	Napi::Value Close(const Napi::CallbackInfo &info);
 	Napi::Value RegisterReplacementScan(const Napi::CallbackInfo &info);
+	Napi::Value Tokenize(const Napi::CallbackInfo &info);
 
 public:
 	constexpr static int DUCKDB_NODEJS_ERROR = -1;

package/src/statement.cpp

@@ -187,6 +187,15 @@ static Napi::Value convert_col_val(Napi::Env &env, duckdb::Value dval, duckdb::L
 		const auto scale = duckdb::Interval::SECS_PER_DAY * duckdb::Interval::MSECS_PER_SEC;
 		value = Napi::Date::New(env, double(dval.GetValue<int32_t>() * scale));
 	} break;
+	case duckdb::LogicalTypeId::TIMESTAMP_NS: {
+		value = Napi::Date::New(env, double(dval.GetValue<int64_t>() / (duckdb::Interval::MICROS_PER_MSEC * 1000)));
+	} break;
+	case duckdb::LogicalTypeId::TIMESTAMP_MS: {
+		value = Napi::Date::New(env, double(dval.GetValue<int64_t>()));
+	} break;
+	case duckdb::LogicalTypeId::TIMESTAMP_SEC: {
+		value = Napi::Date::New(env, double(dval.GetValue<int64_t>() * duckdb::Interval::MSECS_PER_SEC));
+	} break;
 	case duckdb::LogicalTypeId::TIMESTAMP:
 	case duckdb::LogicalTypeId::TIMESTAMP_TZ: {
 		value = Napi::Date::New(env, double(dval.GetValue<int64_t>() / duckdb::Interval::MICROS_PER_MSEC));

package/test/test_all_types.test.ts

@@ -22,10 +22,12 @@ function timedelta(obj: { days: number; micros: number; months: number }) {
 const replacement_values: Record<string, string> = {
   timestamp:
     "'1990-01-01 00:00:00'::TIMESTAMP, '9999-12-31 23:59:59'::TIMESTAMP, NULL::TIMESTAMP",
-  // TODO: fix these, they are currently being returned as strings
-  //   timestamp_s: "'1990-01-01 00:00:00'::TIMESTAMP_S",
-  //   timestamp_ns: "'1990-01-01 00:00:00'::TIMESTAMP_NS",
-  //   timestamp_ms: "'1990-01-01 00:00:00'::TIMESTAMP_MS",
+  timestamp_s:
+    "'1990-01-01 00:00:00'::TIMESTAMP_S, '9999-12-31 23:59:59'::TIMESTAMP_S, NULL::TIMESTAMP_S",
+  // note: timestamp_ns does not support extreme values
+  timestamp_ns: "'1990-01-01 00:00:00'::TIMESTAMP_NS,  NULL::TIMESTAMP_NS",
+  timestamp_ms:
+    "'1990-01-01 00:00:00'::TIMESTAMP_MS,  '9999-12-31 23:59:59'::TIMESTAMP_MS, NULL::TIMESTAMP_MS",
   timestamp_tz:
     "'1990-01-01 00:00:00Z'::TIMESTAMPTZ, '9999-12-31 23:59:59.999999Z'::TIMESTAMPTZ, NULL::TIMESTAMPTZ",
   date: "'1990-01-01'::DATE, '9999-12-31'::DATE, NULL::DATE",
@@ -157,7 +159,7 @@ const correct_answer_map: Record<string, any[]> = {
     null,
   ],
   map: ["{}", "{key1=🦆🦆🦆🦆🦆🦆, key2=goose}", null],
-  union: ['Frank', '5', null],
+  union: ["Frank", "5", null],
 
   time_tz: ["00:00:00-1559", "23:59:59.999999+1559", null],
   interval: [
@@ -176,16 +178,15 @@ const correct_answer_map: Record<string, any[]> = {
     null,
   ],
   date: [new Date("1990-01-01"), new Date("9999-12-31"), null],
-  timestamp_s: ["290309-12-22 (BC) 00:00:00", "294247-01-10 04:00:54", null],
-
-  timestamp_ns: [
-    "1677-09-21 00:12:43.145225",
-    "2262-04-11 23:47:16.854775",
+  timestamp_s: [
+    new Date(Date.UTC(1990, 0, 1)),
+    new Date("9999-12-31T23:59:59.000Z"),
     null,
   ],
+  timestamp_ns: [new Date(Date.UTC(1990, 0, 1)), null],
   timestamp_ms: [
-    "290309-12-22 (BC) 00:00:00",
-    "294247-01-10 04:00:54.775",
+    new Date(Date.UTC(1990, 0, 1)),
+    new Date("9999-12-31T23:59:59.000Z"),
     null,
   ],
   timestamp_tz: [

package/test/tokenize.test.ts

@@ -0,0 +1,74 @@
+import * as assert from 'assert';
+import * as duckdb from '..';
+
+describe('tokenize', function () {
+  it('should return correct tokens for a single statement', function () {
+    const db = new duckdb.Database(':memory:');
+    const output = db.tokenize('select 1;');
+    assert.deepStrictEqual(output, {
+      offsets: [0, 7, 8],
+      types: [duckdb.TokenType.KEYWORD, duckdb.TokenType.NUMERIC_CONSTANT, duckdb.TokenType.OPERATOR]
+    });
+  });
+  it('should return correct tokens for a multiple statements', function () {
+    const db = new duckdb.Database(':memory:');
+    const output = db.tokenize('select 1; select 2;');
+    assert.deepStrictEqual(output, {
+      offsets: [0, 7, 8, 10, 17, 18],
+      types: [
+        duckdb.TokenType.KEYWORD, duckdb.TokenType.NUMERIC_CONSTANT, duckdb.TokenType.OPERATOR,
+        duckdb.TokenType.KEYWORD, duckdb.TokenType.NUMERIC_CONSTANT, duckdb.TokenType.OPERATOR
+      ]
+    });
+  });
+  it('should return no tokens for an empty string', function () {
+    const db = new duckdb.Database(':memory:');
+    const output = db.tokenize('');
+    assert.deepStrictEqual(output, {
+      offsets: [],
+      types: []
+    });
+  });
+  it('should handle quoted semicolons in string constants', function () {
+    const db = new duckdb.Database(':memory:');
+    const output = db.tokenize(`select ';';`);
+    assert.deepStrictEqual(output, {
+      offsets: [0, 7, 10],
+      types: [duckdb.TokenType.KEYWORD, duckdb.TokenType.STRING_CONSTANT, duckdb.TokenType.OPERATOR]
+    });
+  });
+  it('should handle quoted semicolons in identifiers', function () {
+    const db = new duckdb.Database(':memory:');
+    const output = db.tokenize(`from ";";`);
+    assert.deepStrictEqual(output, {
+      offsets: [0, 5, 8],
+      types: [duckdb.TokenType.KEYWORD, duckdb.TokenType.IDENTIFIER, duckdb.TokenType.OPERATOR]
+    });
+  });
+  it('should handle comments', function () {
+    const db = new duckdb.Database(':memory:');
+    const output = db.tokenize(`select /* comment */ 1`);
+    // Note that the tokenizer doesn't return tokens for comments.
+    assert.deepStrictEqual(output, {
+      offsets: [0, 21],
+      types: [duckdb.TokenType.KEYWORD, duckdb.TokenType.NUMERIC_CONSTANT]
+    });
+  });
+  it('should handle invalid syntax', function () {
+    const db = new duckdb.Database(':memory:');
+    const output = db.tokenize(`selec 1`);
+    // The misspelled keyword is scanned as an identifier.
+    assert.deepStrictEqual(output, {
+      offsets: [0, 6],
+      types: [duckdb.TokenType.IDENTIFIER, duckdb.TokenType.NUMERIC_CONSTANT]
+    });
+  });
+  it('should support inverse TokenType mapping', function () {
+    assert.equal(duckdb.TokenType[duckdb.TokenType.IDENTIFIER], "IDENTIFIER");
+    assert.equal(duckdb.TokenType[duckdb.TokenType.NUMERIC_CONSTANT], "NUMERIC_CONSTANT");
+    assert.equal(duckdb.TokenType[duckdb.TokenType.STRING_CONSTANT], "STRING_CONSTANT");
+    assert.equal(duckdb.TokenType[duckdb.TokenType.OPERATOR], "OPERATOR");
+    assert.equal(duckdb.TokenType[duckdb.TokenType.KEYWORD], "KEYWORD");
+    assert.equal(duckdb.TokenType[duckdb.TokenType.COMMENT], "COMMENT");
+  });
+});