duckdb 0.3.3-dev105.0 → 0.3.3-dev149.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.3.3-dev105.0",
+  "version": "0.3.3-dev149.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -15826,6 +15826,10 @@ public:
 		return "LocalFileSystem";
 	}
 
+	//! Returns the last Win32 error, in string format. Returns an empty string if there is no error, or on non-Windows
+	//! systems.
+	static std::string GetLastErrorAsString();
+
 private:
 	//! Set the file pointer of a file handle to a specified location. Reads and writes will happen from this location
 	void SetFilePointer(FileHandle &handle, idx_t location);
@@ -16273,12 +16277,16 @@ void LocalFileSystem::MoveFile(const string &source, const string &target) {
 	}
 }
 
+std::string LocalFileSystem::GetLastErrorAsString() {
+	return string();
+}
+
 #else
 
 constexpr char PIPE_PREFIX[] = "\\\\.\\pipe\\";
 
 // Returns the last Win32 error, in string format. Returns an empty string if there is no error.
-std::string GetLastErrorAsString() {
+std::string LocalFileSystem::GetLastErrorAsString() {
 	// Get the error message, if any.
 	DWORD errorMessageID = GetLastError();
 	if (errorMessageID == 0)
@@ -16354,7 +16362,7 @@ unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path, uint8_t fla
 	HANDLE hFile = CreateFileW(unicode_path.c_str(), desired_access, share_mode, NULL, creation_disposition,
 	                           flags_and_attributes, NULL);
 	if (hFile == INVALID_HANDLE_VALUE) {
-		auto error = GetLastErrorAsString();
+		auto error = LocalFileSystem::GetLastErrorAsString();
 		throw IOException("Cannot open file \"%s\": %s", path.c_str(), error);
 	}
 	auto handle = make_unique<WindowsFileHandle>(*this, path.c_str(), hFile);
@@ -16383,7 +16391,7 @@ static DWORD FSInternalRead(FileHandle &handle, HANDLE hFile, void *buffer, int6
 	ov.hEvent = 0;
 	auto rc = ReadFile(hFile, buffer, (DWORD)nr_bytes, &bytes_read, &ov);
 	if (!rc) {
-		auto error = GetLastErrorAsString();
+		auto error = LocalFileSystem::GetLastErrorAsString();
 		throw IOException("Could not read file \"%s\" (error in ReadFile): %s", handle.path, error);
 	}
 	return bytes_read;
@@ -16417,7 +16425,7 @@ static DWORD FSInternalWrite(FileHandle &handle, HANDLE hFile, void *buffer, int
 	ov.hEvent = 0;
 	auto rc = WriteFile(hFile, buffer, (DWORD)nr_bytes, &bytes_written, &ov);
 	if (!rc) {
-		auto error = GetLastErrorAsString();
+		auto error = LocalFileSystem::GetLastErrorAsString();
 		throw IOException("Could not write file \"%s\" (error in WriteFile): %s", handle.path, error);
 	}
 	return bytes_written;
@@ -16481,7 +16489,7 @@ void LocalFileSystem::Truncate(FileHandle &handle, int64_t new_size) {
 	SetFilePointer(handle, new_size);
 	// now set the end of file position
 	if (!SetEndOfFile(hFile)) {
-		auto error = GetLastErrorAsString();
+		auto error = LocalFileSystem::GetLastErrorAsString();
 		throw IOException("Failure in SetEndOfFile call on file \"%s\": %s", handle.path, error);
 	}
 }
@@ -42511,12 +42519,18 @@ void VectorOperations::Copy(const Vector &source, Vector &target, idx_t source_c
 namespace duckdb {
 
 struct HashOp {
+	static const hash_t NULL_HASH = 0xbf58476d1ce4e5b9;
+
 	template <class T>
 	static inline hash_t Operation(T input, bool is_null) {
-		return duckdb::Hash<T>(is_null ? duckdb::NullValue<T>() : input);
+		return is_null ? NULL_HASH : duckdb::Hash<T>(input);
 	}
 };
 
+static inline hash_t CombineHashScalar(hash_t a, hash_t b) {
+	return (a * UINT64_C(0xbf58476d1ce4e5b9)) ^ b;
+}
+
 template <bool HAS_RSEL, class T>
 static inline void TightLoopHash(T *__restrict ldata, hash_t *__restrict result_data, const SelectionVector *rsel,
                                  idx_t count, const SelectionVector *__restrict sel_vector, ValidityMask &mask) {
@@ -42558,7 +42572,7 @@ template <bool HAS_RSEL, bool FIRST_HASH>
 static inline void StructLoopHash(Vector &input, Vector &hashes, const SelectionVector *rsel, idx_t count) {
 	auto &children = StructVector::GetEntries(input);
 
-	D_ASSERT(children.size() > 0);
+	D_ASSERT(!children.empty());
 	idx_t col_no = 0;
 	if (HAS_RSEL) {
 		if (FIRST_HASH) {
@@ -42588,14 +42602,81 @@ static inline void ListLoopHash(Vector &input, Vector &hashes, const SelectionVe
 	VectorData idata;
 	input.Orrify(count, idata);
 	const auto ldata = (const list_entry_t *)idata.data;
-	for (idx_t i = 0; i < count; i++) {
+
+	// Hash the children into a temporary
+	auto &child = ListVector::GetEntry(input);
+	const auto child_count = ListVector::GetListSize(input);
+
+	Vector child_hashes(LogicalType::HASH, child_count);
+	VectorOperations::Hash(child, child_hashes, child_count);
+	auto chdata = FlatVector::GetData<hash_t>(child_hashes);
+
+	// Reduce the number of entries to check to the non-empty ones
+	SelectionVector unprocessed(count);
+	SelectionVector cursor(HAS_RSEL ? STANDARD_VECTOR_SIZE : count);
+	idx_t remaining = 0;
+	for (idx_t i = 0; i < count; ++i) {
 		const idx_t ridx = HAS_RSEL ? rsel->get_index(i) : i;
 		const auto lidx = idata.sel->get_index(ridx);
 		const auto &entry = ldata[lidx];
-		if (idata.validity.RowIsValid(lidx)) {
-			hdata[ridx] = duckdb::Hash(entry.length);
-		} else {
-			hdata[ridx] = 0;
+		if (idata.validity.RowIsValid(lidx) && entry.length > 0) {
+			unprocessed.set_index(remaining++, ridx);
+			cursor.set_index(ridx, entry.offset);
+		} else if (FIRST_HASH) {
+			hdata[ridx] = HashOp::NULL_HASH;
+		}
+		// Empty or NULL non-first elements have no effect.
+	}
+
+	count = remaining;
+	if (count == 0) {
+		return;
+	}
+
+	// Merge the first position hash into the main hash
+	idx_t position = 1;
+	if (FIRST_HASH) {
+		remaining = 0;
+		for (idx_t i = 0; i < count; ++i) {
+			const auto ridx = unprocessed.get_index(i);
+			const auto cidx = cursor.get_index(ridx);
+			hdata[ridx] = chdata[cidx];
+
+			const auto lidx = idata.sel->get_index(ridx);
+			const auto &entry = ldata[lidx];
+			if (entry.length > position) {
+				// Entry still has values to hash
+				unprocessed.set_index(remaining++, ridx);
+				cursor.set_index(ridx, cidx + 1);
+			}
+		}
+		count = remaining;
+		if (count == 0) {
+			return;
+		}
+		++position;
+	}
+
+	// Combine the hashes for the remaining positions until there are none left
+	for (;; ++position) {
+		remaining = 0;
+		for (idx_t i = 0; i < count; ++i) {
+			const auto ridx = unprocessed.get_index(i);
+			const auto cidx = cursor.get_index(ridx);
+			hdata[ridx] = CombineHashScalar(hdata[ridx], chdata[cidx]);
+
+			const auto lidx = idata.sel->get_index(ridx);
+			const auto &entry = ldata[lidx];
+			if (entry.length > position) {
+				// Entry still has values to hash
+				unprocessed.set_index(remaining++, ridx);
+				cursor.set_index(ridx, cidx + 1);
+			}
+		}
+
+		count = remaining;
+		if (count == 0) {
+			break;
 		}
 	}
 }
@@ -42664,10 +42745,6 @@ void VectorOperations::Hash(Vector &input, Vector &result, const SelectionVector
 	HashTypeSwitch<true>(input, result, &sel, count);
 }
 
-static inline hash_t CombineHashScalar(hash_t a, hash_t b) {
-	return (a * UINT64_C(0xbf58476d1ce4e5b9)) ^ b;
-}
-
 template <bool HAS_RSEL, class T>
 static inline void TightLoopCombineHashConstant(T *__restrict ldata, hash_t constant_hash, hash_t *__restrict hash_data,
                                                 const SelectionVector *rsel, idx_t count,
@@ -80222,6 +80299,10 @@ struct AliasFun {
 	static void RegisterFunction(BuiltinFunctions &set);
 };
 
+struct HashFun {
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
 struct LeastFun {
 	static void RegisterFunction(BuiltinFunctions &set);
 };
@@ -80423,6 +80504,21 @@ void CurrentSettingFun::RegisterFunction(BuiltinFunctions &set) {
 } // namespace duckdb
 
 
+namespace duckdb {
+
+static void HashFunction(DataChunk &args, ExpressionState &state, Vector &result) {
+	args.Hash(result);
+}
+
+void HashFun::RegisterFunction(BuiltinFunctions &set) {
+	auto hash_fun = ScalarFunction("hash", {LogicalType::ANY}, LogicalType::HASH, HashFunction);
+	hash_fun.varargs = LogicalType::ANY;
+	set.AddFunction(hash_fun);
+}
+
+} // namespace duckdb
+
+
 
 namespace duckdb {
 
@@ -80623,6 +80719,7 @@ namespace duckdb {
 
 void BuiltinFunctions::RegisterGenericFunctions() {
 	Register<AliasFun>();
+	Register<HashFun>();
 	Register<LeastFun>();
 	Register<GreatestFun>();
 	Register<StatsFun>();
@@ -101741,35 +101838,35 @@ void DuckDB::SetExtensionLoaded(const std::string &name) {
 
 
 
-#ifdef BUILD_ICU_EXTENSION
+#if defined(BUILD_ICU_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
 #include "icu-extension.hpp"
 #endif
 
-#ifdef BUILD_PARQUET_EXTENSION
+#if defined(BUILD_PARQUET_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
 #include "parquet-extension.hpp"
 #endif
 
-#ifdef BUILD_TPCH_EXTENSION
+#if defined(BUILD_TPCH_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
 #include "tpch-extension.hpp"
 #endif
 
-#ifdef BUILD_TPCDS_EXTENSION
+#if defined(BUILD_TPCDS_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
 #include "tpcds-extension.hpp"
 #endif
 
-#ifdef BUILD_FTS_EXTENSION
+#if defined(BUILD_FTS_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
 #include "fts-extension.hpp"
 #endif
 
-#ifdef BUILD_HTTPFS_EXTENSION
+#if defined(BUILD_HTTPFS_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
 #include "httpfs-extension.hpp"
 #endif
 
-#ifdef BUILD_VISUALIZER_EXTENSION
+#if defined(BUILD_VISUALIZER_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
 #include "visualizer-extension.hpp"
 #endif
 
-#ifdef BUILD_EXCEL_EXTENSION
+#if defined(BUILD_EXCEL_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
 #include "excel-extension.hpp"
 #endif
 
@@ -101808,55 +101905,55 @@ ExtensionLoadResult ExtensionHelper::LoadExtensionInternal(DuckDB &db, const std
 	}
 #endif
 	if (extension == "parquet") {
-#ifdef BUILD_PARQUET_EXTENSION
+#if defined(BUILD_PARQUET_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
 		db.LoadExtension<ParquetExtension>();
 #else
 		// parquet extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "icu") {
-#ifdef BUILD_ICU_EXTENSION
+#if defined(BUILD_ICU_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
 		db.LoadExtension<ICUExtension>();
 #else
 		// icu extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "tpch") {
-#ifdef BUILD_TPCH_EXTENSION
+#if defined(BUILD_TPCH_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
 		db.LoadExtension<TPCHExtension>();
 #else
 		// icu extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "tpcds") {
-#ifdef BUILD_TPCDS_EXTENSION
+#if defined(BUILD_TPCDS_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
 		db.LoadExtension<TPCDSExtension>();
 #else
 		// icu extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "fts") {
-#ifdef BUILD_FTS_EXTENSION
+#if defined(BUILD_FTS_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
 		db.LoadExtension<FTSExtension>();
 #else
 		// fts extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "httpfs") {
-#ifdef BUILD_HTTPFS_EXTENSION
+#if defined(BUILD_HTTPFS_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
 		db.LoadExtension<HTTPFsExtension>();
 #else
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "visualizer") {
-#ifdef BUILD_VISUALIZER_EXTENSION
+#if defined(BUILD_VISUALIZER_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
 		db.LoadExtension<VisualizerExtension>();
 #else
 		// visualizer extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "excel") {
-#ifdef BUILD_EXCEL_EXTENSION
+#if defined(BUILD_EXCEL_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
 		db.LoadExtension<EXCELExtension>();
 #else
 		// excel extension required but not build: skip this test
@@ -109432,6 +109529,7 @@ void ExtensionHelper::InstallExtension(DatabaseInstance &db, const string &exten
 
 
 
+
 #ifndef _WIN32
 #include <dlfcn.h>
 #else
@@ -109452,6 +109550,17 @@ void *dlsym(void *handle, const char *name) {
 	D_ASSERT(handle);
 	return (void *)GetProcAddress((HINSTANCE)handle, name);
 }
+
+std::string GetDLError(void) {
+	return LocalFileSystem::GetLastErrorAsString();
+}
+
+#else
+
+std::string GetDLError(void) {
+	return dlerror();
+}
+
 #endif
 
 } // namespace duckdb
@@ -109469,7 +109578,7 @@ template <class T>
 static T LoadFunctionFromDLL(void *dll, const string &function_name, const string &filename) {
 	auto function = dlsym(dll, function_name.c_str());
 	if (!function) {
-		throw IOException("File \"%s\" did not contain function \"%s\"", filename, function_name);
+		throw IOException("File \"%s\" did not contain function \"%s\": %s", filename, function_name, GetDLError());
 	}
 	return (T)function;
 }
@@ -109497,7 +109606,7 @@ void ExtensionHelper::LoadExternalExtension(DatabaseInstance &db, const string &
 	}
 	auto lib_hdl = dlopen(filename.c_str(), RTLD_LAZY | RTLD_LOCAL);
 	if (!lib_hdl) {
-		throw IOException("File \"%s\" could not be loaded", filename);
+		throw IOException("File \"%s\" could not be loaded: %s", filename, GetDLError());
 	}
 
 	auto basename = fs.ExtractBaseName(filename);

package/src/duckdb.hpp

@@ -11,8 +11,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 #pragma once
 #define DUCKDB_AMALGAMATION 1
 #define DUCKDB_AMALGAMATION_EXTENDED 1
-#define DUCKDB_SOURCE_ID "e25230541"
-#define DUCKDB_VERSION "v0.3.3-dev105"
+#define DUCKDB_SOURCE_ID "2230af054"
+#define DUCKDB_VERSION "v0.3.3-dev149"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //