duckdb 0.9.2-dev2.0 → 0.9.2-dev22.0

package/.github/workflows/HighPriorityIssues.yml

@@ -0,0 +1,36 @@
+name: Create Internal issue when the "High Priority" label is applied
+on:
+  issues:
+    types:
+      - labeled
+
+env:
+  GH_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }}
+  # an event triggering this workflow is either an issue or a pull request,
+  # hence only one of the numbers will be filled in the TITLE_PREFIX
+  TITLE_PREFIX: "[duckdb-node/#${{ github.event.issue.number }}]"
+  PUBLIC_ISSUE_TITLE: ${{ github.event.issue.title }}
+
+jobs:
+  create_or_label_issue:
+    if: github.event.label.name == 'High Priority'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get mirror issue number
+        run: |
+          gh issue list --repo duckdblabs/duckdb-internal --search "${TITLE_PREFIX}" --json title,number --jq ".[] | select(.title | startswith(\"$TITLE_PREFIX\")).number" > mirror_issue_number.txt
+          echo "MIRROR_ISSUE_NUMBER=$(cat mirror_issue_number.txt)" >> $GITHUB_ENV
+
+      - name: Print whether mirror issue exists
+        run: |
+          if [ "$MIRROR_ISSUE_NUMBER" == "" ]; then
+            echo "Mirror issue with title prefix '$TITLE_PREFIX' does not exist yet"
+          else
+            echo "Mirror issue with title prefix '$TITLE_PREFIX' exists with number $MIRROR_ISSUE_NUMBER"
+          fi
+
+      - name: Create or label issue
+        run: |
+          if [ "$MIRROR_ISSUE_NUMBER" == "" ]; then
+            gh issue create --repo duckdblabs/duckdb-internal --label "Node.js" --label "High Priority" --title "$TITLE_PREFIX - $PUBLIC_ISSUE_TITLE" --body "See https://github.com/duckdb/duckdb-node/issues/${{ github.event.issue.number }}"
+          fi

package/.github/workflows/NodeJS.yml

@@ -2,6 +2,8 @@ name: NodeJS
 on:
   push:
   pull_request:
+  workflow_dispatch:
+  repository_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}
@@ -34,55 +36,36 @@ jobs:
     name: node.js Linux
     runs-on: ubuntu-20.04
     needs: set-up-npm
+    continue-on-error: ${{ matrix.node != '18' && matrix.node != '20' && matrix.node != '21' }}
     env:
       TARGET_ARCH: ${{ matrix.target_arch }}
       DUCKDB_NODE_BUILD_CACHE: 0
     strategy:
       matrix:
         # node.js current support policy to be found at https://github.com/duckdb/duckdb-node/tree/main/#Supported-Node-versions
-        node: [ '12', '14', '16', '17', '18', '19', '20' ]
+        node: [ '12', '14', '16', '17', '18', '19', '20', '21']
         target_arch: [ x64, arm64 ]
         isRelease:
           - ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}
         exclude:
           - isRelease: false
             node: 12
-            target_arch: x64
           - isRelease: false
             node: 14
-            target_arch: x64
           - isRelease: false
             node: 16
-            target_arch: x64
           - isRelease: false
             node: 17
-            target_arch: x64
-          - isRelease: false
-            node: 18
-            target_arch: x64
           - isRelease: false
             node: 19
-            target_arch: x64
-          - isRelease: false
-            node: 12
-            target_arch: arm64
-          - isRelease: false
-            node: 14
-            target_arch: arm64
-          - isRelease: false
-            node: 16
-            target_arch: arm64
-          - isRelease: false
-            node: 17
-            target_arch: arm64
           - isRelease: false
             node: 18
             target_arch: arm64
           - isRelease: false
-            node: 19
+            node: 20
             target_arch: arm64
           - isRelease: false
-            node: 20
+            node: 21
             target_arch: arm64
 
     steps:
@@ -127,10 +110,11 @@ jobs:
     name: node.js OSX
     runs-on: macos-latest
     needs: linux-nodejs
+    continue-on-error: ${{ matrix.node != '18' && matrix.node != '20' && matrix.node != '21' }}
     strategy:
       matrix:
         target_arch: [ x64, arm64 ]
-        node: [ '12', '14', '16', '17', '18', '19', '20' ]
+        node: [ '12', '14', '16', '17', '18', '19', '20', '21']
         isRelease:
           - ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}
         exclude:
@@ -143,7 +127,7 @@ jobs:
           - isRelease: false
             node: 17
           - isRelease: false
-            node: 18
+            node: 19
           - target_arch: arm64
             node: 12
           - target_arch: arm64
@@ -158,6 +142,11 @@ jobs:
         with:
           fetch-depth: 0
 
+      # Default Python (3.12) doesn't have support for distutils
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
       - name: Setup Ccache
         uses: hendrikmuhs/ccache-action@main
         with:
@@ -184,12 +173,13 @@ jobs:
     name: node.js Windows
     runs-on: windows-latest
     needs: linux-nodejs
+    continue-on-error: ${{ matrix.node != '18' && matrix.node != '20' && matrix.node != '21' }}
     env:
       npm_config_msvs_version: 2019
 
     strategy:
       matrix:
-        node: [ '12', '14', '16', '17', '18', '19', '20' ]
+        node: [ '12', '14', '16', '17', '18', '19', '20', '21']
         isRelease:
           - ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}
         exclude:
@@ -205,6 +195,8 @@ jobs:
             node: 18
           - isRelease: false
             node: 19
+          - isRelease: false
+            node: 20
 
     steps:
       - uses: actions/setup-python@v4

package/README.md

@@ -101,10 +101,10 @@ var stmt = con.prepare('select ?::INTEGER as fortytwo', function(err, stmt) {
 ```
 
 ## Supported Node versions
-We actively support only LTS and In-Support Node versions, as per July 2023, they are: Node 16, Node 18 and Node 20.
+We actively support only LTS and In-Support Node versions, as per July 2023, they are: Node 18, Node 20 and Node 21.
 Release schedule for Node.js can be checked here: https://github.com/nodejs/release#release-schedule.
 
-We currently bundle and test DuckDB also for Node 10, 12, 14, 17 and 19. We plan of going so going forward as long as the tooling supports it.
+We currently bundle and test DuckDB also for Node 10, 12, 14, 16, 17 and 19. We plan of going so going forward as long as the tooling supports it.
 As per July 2023, Node 15 has been removed from the supported versions.
 
 ## Development

package/lib/duckdb.d.ts

@@ -78,6 +78,8 @@ export type ArrowArray = Uint8Array[];
 export class Connection {
   constructor(db: Database, callback?: Callback<any>);
 
+  close(callback?: Callback<void>): void;
+
   all(sql: string, ...args: [...any, Callback<TableData>] | []): void;
   arrowIPCAll(sql: string, ...args: [...any, Callback<ArrowArray>] | []): void;
   each(sql: string, ...args: [...any, Callback<RowData>] | []): void;
@@ -127,6 +129,20 @@ export type ReplacementScanCallback = (
   table: string
 ) => ReplacementScanResult | null;
 
+export enum TokenType {
+  IDENTIFIER = 0,
+  NUMERIC_CONSTANT = 1,
+  STRING_CONSTANT = 2,
+  OPERATOR = 3,
+  KEYWORD = 4,
+  COMMENT = 5,
+}
+
+export interface ScriptTokens {
+  offsets: number[];
+  types: TokenType[];
+}
+
 export class Database {
   constructor(path: string, accessMode?: number | Record<string,string>, callback?: Callback<any>);
   constructor(path: string, callback?: Callback<any>);
@@ -169,6 +185,8 @@ export class Database {
   registerReplacementScan(
     replacementScan: ReplacementScanCallback
   ): Promise<void>;
+
+  tokenize(text: string): ScriptTokens;
 }
 
 export type GenericTypeInfo = {

package/lib/duckdb.js

@@ -64,6 +64,10 @@ var Statement = duckdb.Statement;
  * @class
  */
 var QueryResult = duckdb.QueryResult;
+/**
+ * Types of tokens return by `tokenize`.
+ */
+var TokenType = duckdb.TokenType;
 
 /**
  * @method
@@ -631,6 +635,14 @@ Database.prototype.unregister_udf = function () {
 
 Database.prototype.registerReplacementScan;
 
+/**
+ * Return positions and types of tokens in given text
+ * @method
+ * @arg text
+ * @return {ScriptTokens}
+ */
+Database.prototype.tokenize;
+
 /**
  * Not implemented
  */

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.9.2-dev2.0",
+  "version": "0.9.2-dev22.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/database.cpp

@@ -1,5 +1,6 @@
 #include "duckdb/parser/expression/constant_expression.hpp"
 #include "duckdb/parser/expression/function_expression.hpp"
+#include "duckdb/parser/parser.hpp"
 #include "duckdb/parser/tableref/table_function_ref.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
 #include "duckdb_node.hpp"
@@ -18,7 +19,8 @@ Napi::FunctionReference Database::Init(Napi::Env env, Napi::Object exports) {
 	    {InstanceMethod("close_internal", &Database::Close), InstanceMethod("wait", &Database::Wait),
 	     InstanceMethod("serialize", &Database::Serialize), InstanceMethod("parallelize", &Database::Parallelize),
 	     InstanceMethod("connect", &Database::Connect), InstanceMethod("interrupt", &Database::Interrupt),
-	     InstanceMethod("registerReplacementScan", &Database::RegisterReplacementScan)});
+	     InstanceMethod("registerReplacementScan", &Database::RegisterReplacementScan),
+	     InstanceMethod("tokenize", &Database::Tokenize)});
 
 	exports.Set("Database", t);
 
@@ -364,4 +366,31 @@ Napi::Value Database::RegisterReplacementScan(const Napi::CallbackInfo &info) {
 	return deferred.Promise();
 }
 
+Napi::Value Database::Tokenize(const Napi::CallbackInfo &info) {
+	auto env = info.Env();
+
+	if (info.Length() < 1) {
+		throw Napi::TypeError::New(env, "Text argument expected");
+	}
+
+	std::string text = info[0].As<Napi::String>();
+
+	auto tokens = duckdb::Parser::Tokenize(text);
+	auto numTokens = tokens.size();
+
+	auto offsets = Napi::Array::New(env, numTokens);
+	auto types = Napi::Array::New(env, numTokens);
+
+	for (size_t i = 0; i < numTokens; i++) {
+		auto token = tokens[i];
+		offsets.Set(i, token.start);
+		types.Set(i, (uint8_t)token.type);
+	}
+
+	auto result = Napi::Object::New(env);
+	result.Set("offsets", offsets);
+	result.Set("types", types);
+	return result;
+}
+
 } // namespace node_duckdb

package/src/duckdb_node.cpp

@@ -12,15 +12,36 @@ NodeDuckDB::NodeDuckDB(Napi::Env env, Napi::Object exports) {
 	statement_constructor = node_duckdb::Statement::Init(env, exports);
 	query_result_constructor = node_duckdb::QueryResult::Init(env, exports);
 
-	exports.DefineProperties({
-	    DEFINE_CONSTANT_INTEGER(exports, node_duckdb::Database::DUCKDB_NODEJS_ERROR, ERROR) DEFINE_CONSTANT_INTEGER(
+	auto token_type_enum = Napi::Object::New(env);
+
+	token_type_enum.Set("IDENTIFIER", 0);
+	token_type_enum.Set("NUMERIC_CONSTANT", 1);
+	token_type_enum.Set("STRING_CONSTANT", 2);
+	token_type_enum.Set("OPERATOR", 3);
+	token_type_enum.Set("KEYWORD", 4);
+	token_type_enum.Set("COMMENT", 5);
+
+	// TypeScript enums expose an inverse mapping.
+	token_type_enum.Set((uint32_t)0, "IDENTIFIER");
+	token_type_enum.Set((uint32_t)1, "NUMERIC_CONSTANT");
+	token_type_enum.Set((uint32_t)2, "STRING_CONSTANT");
+	token_type_enum.Set((uint32_t)3, "OPERATOR");
+	token_type_enum.Set((uint32_t)4, "KEYWORD");
+	token_type_enum.Set((uint32_t)5, "COMMENT");
+
+	token_type_enum_ref = Napi::ObjectReference::New(token_type_enum);
+
+	exports.DefineProperties(
+	    {DEFINE_CONSTANT_INTEGER(exports, node_duckdb::Database::DUCKDB_NODEJS_ERROR, ERROR) DEFINE_CONSTANT_INTEGER(
 	        exports, node_duckdb::Database::DUCKDB_NODEJS_READONLY, OPEN_READONLY) // same as SQLite
-	    DEFINE_CONSTANT_INTEGER(exports, 0, OPEN_READWRITE)                        // ignored
-	    DEFINE_CONSTANT_INTEGER(exports, 0, OPEN_CREATE)                           // ignored
-	    DEFINE_CONSTANT_INTEGER(exports, 0, OPEN_FULLMUTEX)                        // ignored
-	    DEFINE_CONSTANT_INTEGER(exports, 0, OPEN_SHAREDCACHE)                      // ignored
-	    DEFINE_CONSTANT_INTEGER(exports, 0, OPEN_PRIVATECACHE)                     // ignored
-	});
+	     DEFINE_CONSTANT_INTEGER(exports, 0, OPEN_READWRITE)                       // ignored
+	     DEFINE_CONSTANT_INTEGER(exports, 0, OPEN_CREATE)                          // ignored
+	     DEFINE_CONSTANT_INTEGER(exports, 0, OPEN_FULLMUTEX)                       // ignored
+	     DEFINE_CONSTANT_INTEGER(exports, 0, OPEN_SHAREDCACHE)                     // ignored
+	     DEFINE_CONSTANT_INTEGER(exports, 0, OPEN_PRIVATECACHE)                    // ignored
+
+	     Napi::PropertyDescriptor::Value("TokenType", token_type_enum,
+	                                     static_cast<napi_property_attributes>(napi_enumerable | napi_configurable))});
 }
 
 NODE_API_ADDON(NodeDuckDB);

package/src/duckdb_node.hpp

@@ -23,6 +23,7 @@ public:
 	Napi::FunctionReference connection_constructor;
 	Napi::FunctionReference statement_constructor;
 	Napi::FunctionReference query_result_constructor;
+	Napi::ObjectReference token_type_enum_ref;
 };
 
 namespace node_duckdb {
@@ -109,6 +110,7 @@ public:
 	Napi::Value Interrupt(const Napi::CallbackInfo &info);
 	Napi::Value Close(const Napi::CallbackInfo &info);
 	Napi::Value RegisterReplacementScan(const Napi::CallbackInfo &info);
+	Napi::Value Tokenize(const Napi::CallbackInfo &info);
 
 public:
 	constexpr static int DUCKDB_NODEJS_ERROR = -1;

package/src/statement.cpp

@@ -187,6 +187,15 @@ static Napi::Value convert_col_val(Napi::Env &env, duckdb::Value dval, duckdb::L
 		const auto scale = duckdb::Interval::SECS_PER_DAY * duckdb::Interval::MSECS_PER_SEC;
 		value = Napi::Date::New(env, double(dval.GetValue<int32_t>() * scale));
 	} break;
+	case duckdb::LogicalTypeId::TIMESTAMP_NS: {
+		value = Napi::Date::New(env, double(dval.GetValue<int64_t>() / (duckdb::Interval::MICROS_PER_MSEC * 1000)));
+	} break;
+	case duckdb::LogicalTypeId::TIMESTAMP_MS: {
+		value = Napi::Date::New(env, double(dval.GetValue<int64_t>()));
+	} break;
+	case duckdb::LogicalTypeId::TIMESTAMP_SEC: {
+		value = Napi::Date::New(env, double(dval.GetValue<int64_t>() * duckdb::Interval::MSECS_PER_SEC));
+	} break;
 	case duckdb::LogicalTypeId::TIMESTAMP:
 	case duckdb::LogicalTypeId::TIMESTAMP_TZ: {
 		value = Napi::Date::New(env, double(dval.GetValue<int64_t>() / duckdb::Interval::MICROS_PER_MSEC));

package/test/test_all_types.test.ts

@@ -22,10 +22,12 @@ function timedelta(obj: { days: number; micros: number; months: number }) {
 const replacement_values: Record<string, string> = {
   timestamp:
     "'1990-01-01 00:00:00'::TIMESTAMP, '9999-12-31 23:59:59'::TIMESTAMP, NULL::TIMESTAMP",
-  // TODO: fix these, they are currently being returned as strings
-  //   timestamp_s: "'1990-01-01 00:00:00'::TIMESTAMP_S",
-  //   timestamp_ns: "'1990-01-01 00:00:00'::TIMESTAMP_NS",
-  //   timestamp_ms: "'1990-01-01 00:00:00'::TIMESTAMP_MS",
+  timestamp_s:
+    "'1990-01-01 00:00:00'::TIMESTAMP_S, '9999-12-31 23:59:59'::TIMESTAMP_S, NULL::TIMESTAMP_S",
+  // note: timestamp_ns does not support extreme values
+  timestamp_ns: "'1990-01-01 00:00:00'::TIMESTAMP_NS,  NULL::TIMESTAMP_NS",
+  timestamp_ms:
+    "'1990-01-01 00:00:00'::TIMESTAMP_MS,  '9999-12-31 23:59:59'::TIMESTAMP_MS, NULL::TIMESTAMP_MS",
   timestamp_tz:
     "'1990-01-01 00:00:00Z'::TIMESTAMPTZ, '9999-12-31 23:59:59.999999Z'::TIMESTAMPTZ, NULL::TIMESTAMPTZ",
   date: "'1990-01-01'::DATE, '9999-12-31'::DATE, NULL::DATE",
@@ -157,7 +159,7 @@ const correct_answer_map: Record<string, any[]> = {
     null,
   ],
   map: ["{}", "{key1=🦆🦆🦆🦆🦆🦆, key2=goose}", null],
-  union: ['Frank', '5', null],
+  union: ["Frank", "5", null],
 
   time_tz: ["00:00:00-1559", "23:59:59.999999+1559", null],
   interval: [
@@ -176,16 +178,15 @@ const correct_answer_map: Record<string, any[]> = {
     null,
   ],
   date: [new Date("1990-01-01"), new Date("9999-12-31"), null],
-  timestamp_s: ["290309-12-22 (BC) 00:00:00", "294247-01-10 04:00:54", null],
-
-  timestamp_ns: [
-    "1677-09-21 00:12:43.145225",
-    "2262-04-11 23:47:16.854775",
+  timestamp_s: [
+    new Date(Date.UTC(1990, 0, 1)),
+    new Date("9999-12-31T23:59:59.000Z"),
     null,
   ],
+  timestamp_ns: [new Date(Date.UTC(1990, 0, 1)), null],
   timestamp_ms: [
-    "290309-12-22 (BC) 00:00:00",
-    "294247-01-10 04:00:54.775",
+    new Date(Date.UTC(1990, 0, 1)),
+    new Date("9999-12-31T23:59:59.000Z"),
     null,
   ],
   timestamp_tz: [

package/test/tokenize.test.ts

@@ -0,0 +1,74 @@
+import * as assert from 'assert';
+import * as duckdb from '..';
+
+describe('tokenize', function () {
+  it('should return correct tokens for a single statement', function () {
+    const db = new duckdb.Database(':memory:');
+    const output = db.tokenize('select 1;');
+    assert.deepStrictEqual(output, {
+      offsets: [0, 7, 8],
+      types: [duckdb.TokenType.KEYWORD, duckdb.TokenType.NUMERIC_CONSTANT, duckdb.TokenType.OPERATOR]
+    });
+  });
+  it('should return correct tokens for a multiple statements', function () {
+    const db = new duckdb.Database(':memory:');
+    const output = db.tokenize('select 1; select 2;');
+    assert.deepStrictEqual(output, {
+      offsets: [0, 7, 8, 10, 17, 18],
+      types: [
+        duckdb.TokenType.KEYWORD, duckdb.TokenType.NUMERIC_CONSTANT, duckdb.TokenType.OPERATOR,
+        duckdb.TokenType.KEYWORD, duckdb.TokenType.NUMERIC_CONSTANT, duckdb.TokenType.OPERATOR
+      ]
+    });
+  });
+  it('should return no tokens for an empty string', function () {
+    const db = new duckdb.Database(':memory:');
+    const output = db.tokenize('');
+    assert.deepStrictEqual(output, {
+      offsets: [],
+      types: []
+    });
+  });
+  it('should handle quoted semicolons in string constants', function () {
+    const db = new duckdb.Database(':memory:');
+    const output = db.tokenize(`select ';';`);
+    assert.deepStrictEqual(output, {
+      offsets: [0, 7, 10],
+      types: [duckdb.TokenType.KEYWORD, duckdb.TokenType.STRING_CONSTANT, duckdb.TokenType.OPERATOR]
+    });
+  });
+  it('should handle quoted semicolons in identifiers', function () {
+    const db = new duckdb.Database(':memory:');
+    const output = db.tokenize(`from ";";`);
+    assert.deepStrictEqual(output, {
+      offsets: [0, 5, 8],
+      types: [duckdb.TokenType.KEYWORD, duckdb.TokenType.IDENTIFIER, duckdb.TokenType.OPERATOR]
+    });
+  });
+  it('should handle comments', function () {
+    const db = new duckdb.Database(':memory:');
+    const output = db.tokenize(`select /* comment */ 1`);
+    // Note that the tokenizer doesn't return tokens for comments.
+    assert.deepStrictEqual(output, {
+      offsets: [0, 21],
+      types: [duckdb.TokenType.KEYWORD, duckdb.TokenType.NUMERIC_CONSTANT]
+    });
+  });
+  it('should handle invalid syntax', function () {
+    const db = new duckdb.Database(':memory:');
+    const output = db.tokenize(`selec 1`);
+    // The misspelled keyword is scanned as an identifier.
+    assert.deepStrictEqual(output, {
+      offsets: [0, 6],
+      types: [duckdb.TokenType.IDENTIFIER, duckdb.TokenType.NUMERIC_CONSTANT]
+    });
+  });
+  it('should support inverse TokenType mapping', function () {
+    assert.equal(duckdb.TokenType[duckdb.TokenType.IDENTIFIER], "IDENTIFIER");
+    assert.equal(duckdb.TokenType[duckdb.TokenType.NUMERIC_CONSTANT], "NUMERIC_CONSTANT");
+    assert.equal(duckdb.TokenType[duckdb.TokenType.STRING_CONSTANT], "STRING_CONSTANT");
+    assert.equal(duckdb.TokenType[duckdb.TokenType.OPERATOR], "OPERATOR");
+    assert.equal(duckdb.TokenType[duckdb.TokenType.KEYWORD], "KEYWORD");
+    assert.equal(duckdb.TokenType[duckdb.TokenType.COMMENT], "COMMENT");
+  });
+});