duckdb 0.6.2-dev604.0 → 0.6.2-dev623.0

package/Makefile

@@ -1,15 +1,15 @@
 all: build
 
-src/duckdb.cpp:
+src/duckdb/src/include/duckdb.hpp:
 	./configure
 
 ./node_modules:
 	npm install --build-from-source
 
-build: ./node_modules src/duckdb.cpp
+build: ./node_modules src/duckdb/src/include/duckdb.hpp
 	./node_modules/.bin/node-pre-gyp build -j max --loglevel=silent
 
-debug: ./node_modules src/duckdb.cpp
+debug: ./node_modules src/duckdb/src/include/duckdb.hpp
 	./node_modules/.bin/node-pre-gyp build -j max --debug --verbose
 
 clean:
@@ -17,9 +17,12 @@ clean:
 	rm -rf lib/binding/
 	rm -f test/support/big.db-journal
 	rm -rf ./node_modules/
-	rm -rf src/duckdb.*
-	rm -rf src/parquet-amalgamation.*
+	rm -rf src/duckdb/
+	rm -f binding.gyp
 
+complete_build:
+	./configure
+	npm install
 
 test:
 	npm test

package/binding.gyp

@@ -1,90 +1,116 @@
 {
     "targets": [
         {
-            "target_name": "<(module_name)",
+            "target_name": "<(module_name)", 
             "sources": [
-                "src/duckdb_node.cpp",
-                "src/database.cpp",
-                "src/data_chunk.cpp",
-                "src/connection.cpp",
-                "src/statement.cpp",
-                "src/utils.cpp",
-                "src/parquet-amalgamation.cpp",
-                "src/duckdb.cpp" # comment this out to build against existing lib
-            ],
+                "src/duckdb_node.cpp", 
+                "src/database.cpp", 
+                "src/data_chunk.cpp", 
+                "src/connection.cpp", 
+                "src/statement.cpp", 
+                "src/utils.cpp"
+            ], 
             "include_dirs": [
-                "<!@(node -p \"require('node-addon-api').include\")"
-            ],
-            'defines': [
-                'NAPI_DISABLE_CPP_EXCEPTIONS=1',
-                "NAPI_VERSION=5"
-            ],
+                "<!@(node -p \"require('node-addon-api').include\")", 
+                "/home/runner/work/duckdb/duckdb/scripts/../src/include", 
+                "/home/runner/work/duckdb/duckdb/scripts/..", 
+                "/home/runner/work/duckdb/duckdb/scripts/../third_party/utf8proc/include", 
+                "/home/runner/work/duckdb/duckdb/scripts/../extension/parquet/include", 
+                "/home/runner/work/duckdb/duckdb/scripts/../extension/icu/include", 
+                "/home/runner/work/duckdb/duckdb/scripts/../extension/json/include"
+            ], 
+            "defines": [
+                "NAPI_DISABLE_CPP_EXCEPTIONS=1", 
+                "NAPI_VERSION=5", 
+                "BUILD_PARQUET_EXTENSION", 
+                "BUILD_ICU_EXTENSION", 
+                "BUILD_JSON_EXTENSION"
+            ], 
             "cflags_cc": [
-                "-frtti",
-                "-fexceptions",
-                "-Wno-redundant-move",
-            ],
+                "-frtti", 
+                "-fexceptions", 
+                "-Wno-redundant-move"
+            ], 
             "cflags_cc!": [
-                "-fno-rrti",
-                "-fno-exceptions",
-            ],
+                "-fno-rrti", 
+                "-fno-exceptions"
+            ], 
             "cflags": [
-                "-frtti",
-                "-fexceptions",
-                "-Wno-redundant-move",
-            ],
+                "-frtti", 
+                "-fexceptions", 
+                "-Wno-redundant-move"
+            ], 
             "cflags!": [
-                "-fno-rrti",
-                "-fno-exceptions",
-            ],
+                "-fno-rrti", 
+                "-fno-exceptions"
+            ], 
             "xcode_settings": {
-                "GCC_ENABLE_CPP_EXCEPTIONS": "YES",
-                "GCC_ENABLE_CPP_RTTI": "YES",
-                "CLANG_CXX_LIBRARY": "libc++",
-                "MACOSX_DEPLOYMENT_TARGET": "10.15",
-                'CLANG_CXX_LANGUAGE_STANDARD':'c++11',
-                'OTHER_CFLAGS' : ['-fexceptions', '-frtti', '-Wno-redundant-move']
-
-            },
+                "GCC_ENABLE_CPP_EXCEPTIONS": "YES", 
+                "GCC_ENABLE_CPP_RTTI": "YES", 
+                "CLANG_CXX_LIBRARY": "libc++", 
+                "MACOSX_DEPLOYMENT_TARGET": "10.15", 
+                "CLANG_CXX_LANGUAGE_STANDARD": "c++11", 
+                "OTHER_CFLAGS": [
+                    "-fexceptions", 
+                    "-frtti", 
+                    "-Wno-redundant-move"
+                ]
+            }, 
             "msvs_settings": {
                 "VCCLCompilerTool": {
-                    "ExceptionHandling": 1,
+                    "ExceptionHandling": 1, 
                     "AdditionalOptions": [
                         "/bigobj"
                     ]
                 }
-            },
+            }, 
             "conditions": [
                 [
-                    'OS=="win"',
+                    "OS==\"win\"", 
                     {
                         "defines": [
                             "DUCKDB_BUILD_LIBRARY"
                         ]
-                    },
-                ],  # OS=="win"
-            ],  # conditions
-          #  uncomment this to build against existing lib
-#             "libraries": [
-#                 "/Users/hannes/source/duckdb/build/release/src/libduckdb_static.a",
-#                 "/Users/hannes/source/duckdb/build/release/third_party/fmt/libduckdb_fmt.a",
-#                 "/Users/hannes/source/duckdb/build/release/third_party/libpg_query/libduckdb_pg_query.a",
-#                 "/Users/hannes/source/duckdb/build/release/third_party/utf8proc/libduckdb_utf8proc.a",
-#                 "/Users/hannes/source/duckdb/build/release/third_party/re2/libduckdb_re2.a"
-#             ]
-        },
+                    }
+                ]
+            ], 
+            "libraries": [
+                "/home/runner/work/duckdb/duckdb/build/release/src/libduckdb_static.a", 
+                "/home/runner/work/duckdb/duckdb/build/release/extension/parquet/libparquet_extension.a", 
+                "/home/runner/work/duckdb/duckdb/build/release/extension/icu/libicu_extension.a", 
+                "/home/runner/work/duckdb/duckdb/build/release/extension/json/libjson_extension.a", 
+                "/home/runner/work/duckdb/duckdb/build/release/third_party/fsst/libduckdb_fsst.a", 
+                "/home/runner/work/duckdb/duckdb/build/release/third_party/fmt/libduckdb_fmt.a", 
+                "/home/runner/work/duckdb/duckdb/build/release/third_party/libpg_query/libduckdb_pg_query.a", 
+                "/home/runner/work/duckdb/duckdb/build/release/third_party/re2/libduckdb_re2.a", 
+                "/home/runner/work/duckdb/duckdb/build/release/third_party/miniz/libduckdb_miniz.a", 
+                "/home/runner/work/duckdb/duckdb/build/release/third_party/utf8proc/libduckdb_utf8proc.a", 
+                "/home/runner/work/duckdb/duckdb/build/release/third_party/hyperloglog/libduckdb_hyperloglog.a", 
+                "/home/runner/work/duckdb/duckdb/build/release/third_party/fastpforlib/libduckdb_fastpforlib.a", 
+                "/home/runner/work/duckdb/duckdb/build/release/third_party/mbedtls/libduckdb_mbedtls.a", 
+                "/home/runner/work/duckdb/duckdb/build/release/extension/jemalloc/libjemalloc_extension.a", 
+                "/home/runner/work/duckdb/duckdb/build/release/extension/parquet/libparquet_extension.a", 
+                "/home/runner/work/duckdb/duckdb/build/release/extension/icu/libicu_extension.a", 
+                "/home/runner/work/duckdb/duckdb/build/release/extension/tpch/libtpch_extension.a", 
+                "/home/runner/work/duckdb/duckdb/build/release/extension/httpfs/libhttpfs_extension.a", 
+                "/home/runner/work/duckdb/duckdb/build/release/extension/visualizer/libvisualizer_extension.a", 
+                "/home/runner/work/duckdb/duckdb/build/release/extension/json/libjson_extension.a"
+            ]
+        }, 
         {
-            "target_name": "action_after_build",
-            "type": "none",
+            "target_name": "action_after_build", 
+            "type": "none", 
             "dependencies": [
                 "<(module_name)"
-            ],
+            ], 
             "copies": [
                 {
-                    "files": [ "<(PRODUCT_DIR)/<(module_name).node" ],
+                    "files": [
+                        "<(PRODUCT_DIR)/<(module_name).node"
+                    ], 
                     "destination": "<(module_path)"
                 }
             ]
         }
     ]
-}
+}

package/binding.gyp.in

@@ -0,0 +1,88 @@
+{
+    "targets": [
+        {
+            "target_name": "<(module_name)",
+            "sources": [
+                "src/duckdb_node.cpp",
+                "src/database.cpp",
+                "src/data_chunk.cpp",
+                "src/connection.cpp",
+                "src/statement.cpp",
+                "src/utils.cpp",
+                "${SOURCE_FILES}"
+            ],
+            "include_dirs": [
+                "<!@(node -p \"require('node-addon-api').include\")",
+                "${INCLUDE_FILES}"
+            ],
+            "defines": [
+                "NAPI_DISABLE_CPP_EXCEPTIONS=1",
+                "NAPI_VERSION=5",
+                "${DEFINES}"
+            ],
+            "cflags_cc": [
+                "-frtti",
+                "-fexceptions",
+                "-Wno-redundant-move"
+            ],
+            "cflags_cc!": [
+                "-fno-rrti",
+                "-fno-exceptions"
+            ],
+            "cflags": [
+                "-frtti",
+                "-fexceptions",
+                "-Wno-redundant-move",
+                "${CFLAGS}"
+            ],
+            "cflags!": [
+                "-fno-rrti",
+                "-fno-exceptions"
+            ],
+            "xcode_settings": {
+                "GCC_ENABLE_CPP_EXCEPTIONS": "YES",
+                "GCC_ENABLE_CPP_RTTI": "YES",
+                "CLANG_CXX_LIBRARY": "libc++",
+                "MACOSX_DEPLOYMENT_TARGET": "10.15",
+                "CLANG_CXX_LANGUAGE_STANDARD": "c++11",
+                "OTHER_CFLAGS" : ["-fexceptions", "-frtti", "-Wno-redundant-move"]
+
+            },
+            "msvs_settings": {
+                "VCCLCompilerTool": {
+                    "ExceptionHandling": 1,
+                    "AdditionalOptions": [
+                        "/bigobj",
+                        "${WINDOWS_OPTIONS}"
+                    ]
+                }
+            },
+            "conditions": [
+                [
+                    "OS==\"win\"",
+                    {
+                        "defines": [
+                            "DUCKDB_BUILD_LIBRARY"
+                        ]
+                    }
+                ]
+            ],
+            "libraries": [
+                "${LIBRARY_FILES}"
+            ]
+        },
+        {
+            "target_name": "action_after_build",
+            "type": "none",
+            "dependencies": [
+                "<(module_name)"
+            ],
+            "copies": [
+                {
+                    "files": [ "<(PRODUCT_DIR)/<(module_name).node" ],
+                    "destination": "<(module_path)"
+                }
+            ]
+        }
+    ]
+}

package/configure

@@ -5,10 +5,4 @@ set -x
 
 cd `dirname $0`
 
-if [ ! -f "../../scripts/amalgamation.py" ]; then
-	echo "Could not find the amalgamation build script"
-	exit 1
-fi
-
-(cd ../.. && python3 scripts/amalgamation.py --extended --source=tools/nodejs/src/duckdb.cpp --header=tools/nodejs/src/duckdb.hpp)
-(cd ../.. && python3 scripts/parquet_amalgamation.py && cp src/amalgamation/parquet-amalgamation* tools/nodejs/src/)
+python3 configure.py

package/configure.py

@@ -0,0 +1,126 @@
+import os
+import sys
+import json
+import pickle
+
+# list of extensions to bundle
+extensions = ['parquet', 'icu', 'json']
+
+# path to target
+basedir = os.getcwd()
+target_dir = os.path.join(basedir, 'src', 'duckdb')
+gyp_in = os.path.join(basedir, 'binding.gyp.in')
+gyp_out = os.path.join(basedir, 'binding.gyp')
+cache_file = os.path.join(basedir, 'filelist.cache')
+
+# path to package_build.py
+os.chdir(os.path.join('..', '..'))
+scripts_dir = 'scripts'
+
+sys.path.append(scripts_dir)
+import package_build
+
+defines = ['BUILD_{}_EXTENSION'.format(ext.upper()) for ext in extensions]
+
+if 'DUCKDB_NODE_BUILD_CACHE' in os.environ and os.path.isfile(cache_file):
+    with open(cache_file, 'rb') as f:
+        cache = pickle.load(f)
+    source_list = cache['source_list']
+    include_list = cache['include_list']
+    libraries = cache['libraries']
+    windows_options = cache['windows_options']
+    cflags = cache['cflags']
+elif 'DUCKDB_NODE_BINDIR' in os.environ:
+    def find_library_path(libdir, libname):
+        flist = os.listdir(libdir)
+        for fname in flist:
+            fpath = os.path.join(libdir, fname)
+            if os.path.isfile(fpath) and package_build.file_is_lib(fname, libname):
+                return fpath
+        raise Exception(f"Failed to find library {libname} in {libdir}")
+    # existing build
+    existing_duckdb_dir = os.environ['DUCKDB_NODE_BINDIR']
+    cflags = os.environ['DUCKDB_NODE_CFLAGS']
+    libraries = os.environ['DUCKDB_NODE_LIBS'].split(' ')
+
+    include_directories = [os.path.join('..', '..', include) for include in package_build.third_party_includes()]
+    include_list = package_build.includes(extensions)
+
+    result_libraries = package_build.get_libraries(existing_duckdb_dir, libraries, extensions)
+    libraries = []
+    for (libdir, libname) in result_libraries:
+        if libdir is None:
+            continue
+        libraries.append(find_library_path(libdir, libname))
+
+    source_list = []
+    cflags = []
+    windows_options = []
+    if os.name == 'nt':
+        windows_options = [x for x in os.environ['DUCKDB_NODE_CFLAGS'].split(' ') if x.startswith('/')]
+    else:
+        if '-g' in os.environ['DUCKDB_NODE_CFLAGS']:
+            cflags += ['-g']
+        if '-O0' in os.environ['DUCKDB_NODE_CFLAGS']:
+            cflags += ['-O0']
+
+    if 'DUCKDB_NODE_BUILD_CACHE' in os.environ:
+        cache = {
+            'source_list': source_list,
+            'include_list': include_list,
+            'libraries': libraries,
+            'cflags': cflags,
+            'windows_options': windows_options
+        }
+        with open(cache_file, 'wb+') as f:
+            pickle.dump(cache, f)
+else:
+    # fresh build - copy over all of the files
+    (source_list, include_list, original_sources) = package_build.build_package(target_dir, extensions, False)
+
+    # # the list of all source files (.cpp files) that have been copied into the `duckdb_source_copy` directory
+    # print(source_list)
+    # # the list of all include files
+    # print(include_list)
+    source_list = [os.path.relpath(x, basedir) if os.path.isabs(x) else os.path.join('src', x) for x in source_list]
+    include_list = [os.path.join('src', 'duckdb', x) for x in include_list]
+    libraries = []
+    windows_options = []
+    cflags = []
+
+def sanitize_path(x):
+    return x.replace('\\', '/')
+
+source_list = [sanitize_path(x) for x in source_list]
+include_list = [sanitize_path(x) for x in include_list]
+libraries = [sanitize_path(x) for x in libraries]
+
+with open(gyp_in, 'r') as f:
+    input_json = json.load(f)
+
+def replace_entries(node, replacement_map):
+    if type(node) == type([]):
+        for key in replacement_map.keys():
+            if key in node:
+                node.remove(key)
+                node += replacement_map[key]
+        for entry in node:
+            if type(entry) == type([]) or type(entry) == type({}):
+                replace_entries(entry, replacement_map)
+    if type(node) == type({}):
+        for key in node.keys():
+            replace_entries(node[key], replacement_map)
+
+
+replacement_map = {}
+replacement_map['${SOURCE_FILES}'] = source_list
+replacement_map['${INCLUDE_FILES}'] = include_list
+replacement_map['${DEFINES}'] = defines
+replacement_map['${LIBRARY_FILES}'] = libraries
+replacement_map['${CFLAGS}'] = cflags
+replacement_map['${WINDOWS_OPTIONS}'] = windows_options
+
+replace_entries(input_json, replacement_map)
+
+with open(gyp_out, 'w+') as f:
+    json.dump(input_json, f, indent=4, separators=(", ", ": "))

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.6.2-dev604.0",
+  "version": "0.6.2-dev623.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/connection.cpp

@@ -1,7 +1,9 @@
 #include "duckdb.hpp"
 #include "duckdb_node.hpp"
 #include "napi.h"
-
+#include "duckdb/parser/parser.hpp"
+#include "duckdb/parser/parsed_data/drop_info.hpp"
+#include "duckdb/parser/expression/cast_expression.hpp"
 #include <iostream>
 #include <thread>
 
@@ -242,6 +244,7 @@ struct RegisterUdfTask : public Task {
 			// here we can do only DuckDB stuff because we do not have a functioning env
 
 			// Flatten all args to simplify udfs
+			bool all_constant = args.AllConstant();
 			args.Flatten();
 
 			JSArgs jsargs;
@@ -257,6 +260,9 @@ struct RegisterUdfTask : public Task {
 			if (jsargs.error) {
 				jsargs.error.Throw();
 			}
+			if (all_constant) {
+				result.SetVectorType(duckdb::VectorType::CONSTANT_VECTOR);
+			}
 		};
 
 		auto expr = duckdb::Parser::ParseExpressionList(duckdb::StringUtil::Format("asdf::%s", return_type_name));

package/src/database.cpp

@@ -1,6 +1,6 @@
 #include "duckdb_node.hpp"
+#include "duckdb/storage/buffer_manager.hpp"
 #include "napi.h"
-#include "parquet-amalgamation.hpp"
 
 namespace node_duckdb {
 
@@ -56,8 +56,6 @@ struct OpenTask : public Task {
 	void DoWork() override {
 		try {
 			Get<Database>().database = duckdb::make_unique<duckdb::DuckDB>(filename, &duckdb_config);
-			duckdb::ParquetExtension extension;
-			extension.Load(*Get<Database>().database);
 			success = true;
 
 		} catch (const duckdb::Exception &ex) {