duckdb 0.4.1-dev23.0 → 0.4.1-dev33.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.4.1-dev23.0",
+  "version": "0.4.1-dev33.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -6039,6 +6039,7 @@ private:
 } // namespace duckdb
 
 
+
 namespace duckdb {
 
 ArrowSchemaWrapper::~ArrowSchemaWrapper() {
@@ -6235,6 +6236,7 @@ unique_ptr<DataChunk> ArrowUtil::FetchChunk(QueryResult *result, idx_t chunk_siz
 } // namespace duckdb
 
 
+
 namespace duckdb {
 
 void DuckDBAssertInternal(bool condition, const char *condition_name, const char *file, int linenr) {
@@ -6842,6 +6844,7 @@ uint64_t CycleCounter::Tick() const {
 
 
 
+
 namespace duckdb {
 
 // LCOV_EXCL_START
@@ -13922,6 +13925,7 @@ void FileBuffer::Clear() {
 #undef RemoveDirectory
 
 #endif
+
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -17282,6 +17286,7 @@ public:
 
 
 
+
 #include <cstdint>
 #include <cstdio>
 
@@ -30782,7 +30787,8 @@ std::vector<Match> RegexFindAll(const std::string &input, const Regex &regex) {
 	return matches;
 }
 
-} // namespace duckdb_re2//===----------------------------------------------------------------------===//
+} // namespace duckdb_re2
+//===----------------------------------------------------------------------===//
 //                         DuckDB
 //
 // duckdb/common/types/row_operations/row_aggregate.cpp
@@ -37172,6 +37178,7 @@ private:
 
 
 
+
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -38252,7 +38259,8 @@ unique_ptr<RenderTree> TreeRenderer::CreateTree(const Pipeline &op) {
 	return CreateRenderTree<PipelineRenderNode>(*node);
 }
 
-} // namespace duckdb//===----------------------------------------------------------------------===//
+} // namespace duckdb
+//===----------------------------------------------------------------------===//
 //                         DuckDB
 //
 // duckdb/common/types/batched_chunk_collection.hpp
@@ -39350,6 +39358,7 @@ struct ArrowAuxiliaryData : VectorAuxiliaryData {
 } // namespace duckdb
 
 
+
 namespace duckdb {
 
 DataChunk::DataChunk() : count(0), capacity(STANDARD_VECTOR_SIZE) {
@@ -41677,6 +41686,7 @@ private:
 
 
 
+
 namespace duckdb {
 
 HyperLogLog::HyperLogLog() : hll(nullptr) {
@@ -61615,6 +61625,7 @@ public:
 
 } // namespace duckdb
 
+
 namespace duckdb {
 
 class PhysicalExecute : public PhysicalOperator {
@@ -62311,6 +62322,12 @@ class DuckDB;
 
 enum class ExtensionLoadResult : uint8_t { LOADED_EXTENSION = 0, EXTENSION_UNKNOWN = 1, NOT_LOADED = 2 };
 
+struct DefaultExtension {
+	const char *name;
+	const char *description;
+	bool statically_loaded;
+};
+
 class ExtensionHelper {
 public:
 	static void LoadAllExtensions(DuckDB &db);
@@ -62320,6 +62337,11 @@ public:
 	static void InstallExtension(DatabaseInstance &db, const string &extension, bool force_install);
 	static void LoadExternalExtension(DatabaseInstance &db, const string &extension);
 
+	static string ExtensionDirectory(FileSystem &fs);
+
+	static idx_t DefaultExtensionCount();
+	static DefaultExtension GetDefaultExtension(idx_t index);
+
 private:
 	static const vector<string> PathComponents();
 
@@ -73810,6 +73832,7 @@ public:
 
 
 
+
 namespace duckdb {
 
 PhysicalCreateType::PhysicalCreateType(unique_ptr<CreateTypeInfo> info, idx_t estimated_cardinality)
@@ -95106,6 +95129,7 @@ void ListPositionFun::RegisterFunction(BuiltinFunctions &set) {
 } // namespace duckdb
 
 
+
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -96523,6 +96547,7 @@ void ListSortFun::RegisterFunction(BuiltinFunctions &set) {
 
 
 
+
 namespace duckdb {
 
 static void ListValueFunction(DataChunk &args, ExpressionState &state, Vector &result) {
@@ -107057,6 +107082,10 @@ struct DuckDBDependenciesFun {
 	static void RegisterFunction(BuiltinFunctions &set);
 };
 
+struct DuckDBExtensionsFun {
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
 struct DuckDBFunctionsFun {
 	static void RegisterFunction(BuiltinFunctions &set);
 };
@@ -108655,6 +108684,152 @@ void DuckDBDependenciesFun::RegisterFunction(BuiltinFunctions &set) {
 
 
 
+namespace duckdb {
+
+struct ExtensionInformation {
+	string name;
+	bool loaded = false;
+	bool installed = false;
+	string file_path;
+	string description;
+};
+
+struct DuckDBExtensionsData : public GlobalTableFunctionState {
+	DuckDBExtensionsData() : offset(0) {
+	}
+
+	vector<ExtensionInformation> entries;
+	idx_t offset;
+};
+
+static unique_ptr<FunctionData> DuckDBExtensionsBind(ClientContext &context, TableFunctionBindInput &input,
+                                                     vector<LogicalType> &return_types, vector<string> &names) {
+	names.emplace_back("extension_name");
+	return_types.emplace_back(LogicalType::VARCHAR);
+
+	names.emplace_back("loaded");
+	return_types.emplace_back(LogicalType::BOOLEAN);
+
+	names.emplace_back("installed");
+	return_types.emplace_back(LogicalType::BOOLEAN);
+
+	names.emplace_back("install_path");
+	return_types.emplace_back(LogicalType::VARCHAR);
+
+	names.emplace_back("description");
+	return_types.emplace_back(LogicalType::VARCHAR);
+
+	return nullptr;
+}
+
+unique_ptr<GlobalTableFunctionState> DuckDBExtensionsInit(ClientContext &context, TableFunctionInitInput &input) {
+	auto result = make_unique<DuckDBExtensionsData>();
+
+	auto &fs = FileSystem::GetFileSystem(context);
+	auto &db = DatabaseInstance::GetDatabase(context);
+
+	map<string, ExtensionInformation> installed_extensions;
+	auto extension_count = ExtensionHelper::DefaultExtensionCount();
+	for (idx_t i = 0; i < extension_count; i++) {
+		auto extension = ExtensionHelper::GetDefaultExtension(i);
+		ExtensionInformation info;
+		info.name = extension.name;
+		info.installed = extension.statically_loaded;
+		info.loaded = false;
+		info.file_path = extension.statically_loaded ? "(BUILT-IN)" : string();
+		info.description = extension.description;
+		installed_extensions[info.name] = move(info);
+	}
+
+	// scan the install directory for installed extensions
+	auto ext_directory = ExtensionHelper::ExtensionDirectory(fs);
+	fs.ListFiles(ext_directory, [&](const string &path, bool is_directory) {
+		if (!StringUtil::EndsWith(path, ".duckdb_extension")) {
+			return;
+		}
+		ExtensionInformation info;
+		info.name = fs.ExtractBaseName(path);
+		info.loaded = false;
+		info.file_path = fs.JoinPath(ext_directory, path);
+		auto entry = installed_extensions.find(info.name);
+		if (entry == installed_extensions.end()) {
+			installed_extensions[info.name] = move(info);
+		} else {
+			if (!entry->second.loaded) {
+				entry->second.file_path = info.file_path;
+			}
+			entry->second.installed = true;
+		}
+	});
+
+	// now check the list of currently loaded extensions
+	auto &loaded_extensions = db.LoadedExtensions();
+	for (auto &ext_name : loaded_extensions) {
+		auto entry = installed_extensions.find(ext_name);
+		if (entry == installed_extensions.end()) {
+			ExtensionInformation info;
+			info.name = ext_name;
+			info.loaded = true;
+			installed_extensions[ext_name] = move(info);
+		} else {
+			entry->second.loaded = true;
+		}
+	}
+
+	result->entries.reserve(installed_extensions.size());
+	for (auto &kv : installed_extensions) {
+		result->entries.push_back(move(kv.second));
+	}
+	return move(result);
+}
+
+void DuckDBExtensionsFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) {
+	auto &data = (DuckDBExtensionsData &)*data_p.global_state;
+	if (data.offset >= data.entries.size()) {
+		// finished returning values
+		return;
+	}
+	// start returning values
+	// either fill up the chunk or return all the remaining columns
+	idx_t count = 0;
+	while (data.offset < data.entries.size() && count < STANDARD_VECTOR_SIZE) {
+		auto &entry = data.entries[data.offset];
+
+		// return values:
+		// extension_name LogicalType::VARCHAR
+		output.SetValue(0, count, Value(entry.name));
+		// loaded LogicalType::BOOLEAN
+		output.SetValue(1, count, Value::BOOLEAN(entry.loaded));
+		// installed LogicalType::BOOLEAN
+		output.SetValue(2, count, !entry.installed && entry.loaded ? Value() : Value::BOOLEAN(entry.installed));
+		// install_path LogicalType::VARCHAR
+		output.SetValue(3, count, Value(entry.file_path));
+		// description LogicalType::VARCHAR
+		output.SetValue(4, count, Value(entry.description));
+
+		data.offset++;
+		count++;
+	}
+	output.SetCardinality(count);
+}
+
+void DuckDBExtensionsFun::RegisterFunction(BuiltinFunctions &set) {
+	TableFunctionSet functions("duckdb_extensions");
+	functions.AddFunction(TableFunction({}, DuckDBExtensionsFunction, DuckDBExtensionsBind, DuckDBExtensionsInit));
+	set.AddFunction(functions);
+}
+
+} // namespace duckdb
+
+
+
+
+
+
+
+
+
+
 
 
 
@@ -111298,6 +111473,7 @@ void BuiltinFunctions::RegisterSQLiteFunctions() {
 	DuckDBIndexesFun::RegisterFunction(*this);
 	DuckDBSchemasFun::RegisterFunction(*this);
 	DuckDBDependenciesFun::RegisterFunction(*this);
+	DuckDBExtensionsFun::RegisterFunction(*this);
 	DuckDBSequencesFun::RegisterFunction(*this);
 	DuckDBSettingsFun::RegisterFunction(*this);
 	DuckDBTablesFun::RegisterFunction(*this);
@@ -115088,6 +115264,7 @@ private:
 
 
 
+
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -116606,6 +116783,7 @@ bool ClientContextFileOpener::TryGetCurrentSetting(const string &key, Value &res
 
 
 
+
 namespace duckdb {
 
 ClientData::ClientData(ClientContext &context) : catalog_search_path(make_unique<CatalogSearchPath>(context)) {
@@ -117863,6 +118041,10 @@ idx_t DatabaseInstance::NumberOfThreads() {
 	return scheduler->NumberOfThreads();
 }
 
+const unordered_set<std::string> &DatabaseInstance::LoadedExtensions() {
+	return loaded_extensions;
+}
+
 idx_t DuckDB::NumberOfThreads() {
 	return instance->NumberOfThreads();
 }
@@ -117891,31 +118073,52 @@ string ClientConfig::ExtractTimezoneFromConfig(ClientConfig &config) {
 
 
 #if defined(BUILD_ICU_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
+#define ICU_STATICALLY_LOADED true
 #include "icu-extension.hpp"
+#else
+#define ICU_STATICALLY_LOADED false
 #endif
 
 #if defined(BUILD_PARQUET_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
+#define PARQUET_STATICALLY_LOADED true
 #include "parquet-extension.hpp"
+#else
+#define PARQUET_STATICALLY_LOADED false
 #endif
 
 #if defined(BUILD_TPCH_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
+#define TPCH_STATICALLY_LOADED true
 #include "tpch-extension.hpp"
+#else
+#define TPCH_STATICALLY_LOADED false
 #endif
 
 #if defined(BUILD_TPCDS_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
+#define TPCDS_STATICALLY_LOADED true
 #include "tpcds-extension.hpp"
+#else
+#define TPCDS_STATICALLY_LOADED false
 #endif
 
 #if defined(BUILD_SUBSTRAIT_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
+#define SUBSTRAIT_STATICALLY_LOADED true
 #include "substrait-extension.hpp"
+#else
+#define SUBSTRAIT_STATICALLY_LOADED false
 #endif
 
 #if defined(BUILD_FTS_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
+#define FTS_STATICALLY_LOADED true
 #include "fts-extension.hpp"
+#else
+#define FTS_STATICALLY_LOADED false
 #endif
 
 #if defined(BUILD_HTTPFS_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
+#define HTTPFS_STATICALLY_LOADED true
 #include "httpfs-extension.hpp"
+#else
+#define HTTPFS_STATICALLY_LOADED false
 #endif
 
 #if defined(BUILD_VISUALIZER_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
@@ -117923,7 +118126,10 @@ string ClientConfig::ExtractTimezoneFromConfig(ClientConfig &config) {
 #endif
 
 #if defined(BUILD_JSON_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
+#define JSON_STATICALLY_LOADED true
 #include "json-extension.hpp"
+#else
+#define JSON_STATICALLY_LOADED false
 #endif
 
 #if defined(BUILD_EXCEL_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
@@ -117936,6 +118142,37 @@ string ClientConfig::ExtractTimezoneFromConfig(ClientConfig &config) {
 
 namespace duckdb {
 
+//===--------------------------------------------------------------------===//
+// Default Extensions
+//===--------------------------------------------------------------------===//
+static DefaultExtension internal_extensions[] = {
+    {"icu", "Adds support for time zones and collations using the ICU library", ICU_STATICALLY_LOADED},
+    {"parquet", "Adds support for reading and writing parquet files", PARQUET_STATICALLY_LOADED},
+    {"tpch", "Adds TPC-H data generation and query support", TPCH_STATICALLY_LOADED},
+    {"tpcds", "Adds TPC-DS data generation and query support", TPCDS_STATICALLY_LOADED},
+    {"substrait", "Adds support for the Substrait integration", SUBSTRAIT_STATICALLY_LOADED},
+    {"fts", "Adds support for Full-Text Search Indexes", FTS_STATICALLY_LOADED},
+    {"httpfs", "Adds support for reading and writing files over a HTTP(S) connection", HTTPFS_STATICALLY_LOADED},
+    {"json", "Adds support for JSON operations", JSON_STATICALLY_LOADED},
+    {"sqlite_scanner", "Adds support for reading SQLite database files", false},
+    {"postgres_scanner", "Adds support for reading from a Postgres database", false},
+    {nullptr, nullptr, false}};
+
+idx_t ExtensionHelper::DefaultExtensionCount() {
+	idx_t index;
+	for (index = 0; internal_extensions[index].name != nullptr; index++) {
+	}
+	return index;
+}
+
+DefaultExtension ExtensionHelper::GetDefaultExtension(idx_t index) {
+	D_ASSERT(index < DefaultExtensionCount());
+	return internal_extensions[index];
+}
+
+//===--------------------------------------------------------------------===//
+// Load Statically Compiled Extension
+//===--------------------------------------------------------------------===//
 void ExtensionHelper::LoadAllExtensions(DuckDB &db) {
 	unordered_set<string> extensions {"parquet",   "icu",        "tpch", "tpcds", "fts",     "httpfs",
 	                                  "substrait", "visualizer", "json", "excel", "sqlsmith"};
@@ -117944,9 +118181,6 @@ void ExtensionHelper::LoadAllExtensions(DuckDB &db) {
 	}
 }
 
-//===--------------------------------------------------------------------===//
-// Load Statically Compiled Extension
-//===--------------------------------------------------------------------===//
 ExtensionLoadResult ExtensionHelper::LoadExtension(DuckDB &db, const std::string &extension) {
 	return LoadExtensionInternal(db, extension, false);
 }
@@ -117970,28 +118204,28 @@ ExtensionLoadResult ExtensionHelper::LoadExtensionInternal(DuckDB &db, const std
 	}
 #endif
 	if (extension == "parquet") {
-#if defined(BUILD_PARQUET_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
+#if PARQUET_STATICALLY_LOADED
 		db.LoadExtension<ParquetExtension>();
 #else
 		// parquet extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "icu") {
-#if defined(BUILD_ICU_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
+#if ICU_STATICALLY_LOADED
 		db.LoadExtension<ICUExtension>();
 #else
 		// icu extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "tpch") {
-#if defined(BUILD_TPCH_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
+#if TPCH_STATICALLY_LOADED
 		db.LoadExtension<TPCHExtension>();
 #else
 		// icu extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "substrait") {
-#if defined(BUILD_SUBSTRAIT_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
+#if SUBSTRAIT_STATICALLY_LOADED
 
 		db.LoadExtension<SubstraitExtension>();
 #else
@@ -117999,21 +118233,21 @@ ExtensionLoadResult ExtensionHelper::LoadExtensionInternal(DuckDB &db, const std
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "tpcds") {
-#if defined(BUILD_TPCDS_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
+#if TPCDS_STATICALLY_LOADED
 		db.LoadExtension<TPCDSExtension>();
 #else
 		// icu extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "fts") {
-#if defined(BUILD_FTS_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
+#if FTS_STATICALLY_LOADED
 		db.LoadExtension<FTSExtension>();
 #else
 		// fts extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "httpfs") {
-#if defined(BUILD_HTTPFS_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
+#if HTTPFS_STATICALLY_LOADED
 		db.LoadExtension<HTTPFsExtension>();
 #else
 		return ExtensionLoadResult::NOT_LOADED;
@@ -118026,7 +118260,7 @@ ExtensionLoadResult ExtensionHelper::LoadExtensionInternal(DuckDB &db, const std
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "json") {
-#if defined(BUILD_JSON_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
+#if JSON_STATICALLY_LOADED
 		db.LoadExtension<JSONExtension>();
 #else
 		// json extension required but not build: skip this test
@@ -126269,13 +126503,7 @@ const vector<string> ExtensionHelper::PathComponents() {
 	return vector<string> {".duckdb", "extensions", DuckDB::SourceID(), DuckDB::Platform()};
 }
 
-void ExtensionHelper::InstallExtension(DatabaseInstance &db, const string &extension, bool force_install) {
-	auto &config = DBConfig::GetConfig(db);
-	if (!config.enable_external_access) {
-		throw PermissionException("Installing extensions is disabled through configuration");
-	}
-	auto &fs = FileSystem::GetFileSystem(db);
-
+string ExtensionHelper::ExtensionDirectory(FileSystem &fs) {
 	string local_path = fs.GetHomeDirectory();
 	if (!fs.DirectoryExists(local_path)) {
 		throw InternalException("Can't find the home directory at " + local_path);
@@ -126287,6 +126515,17 @@ void ExtensionHelper::InstallExtension(DatabaseInstance &db, const string &exten
 			fs.CreateDirectory(local_path);
 		}
 	}
+	return local_path;
+}
+
+void ExtensionHelper::InstallExtension(DatabaseInstance &db, const string &extension, bool force_install) {
+	auto &config = DBConfig::GetConfig(db);
+	if (!config.enable_external_access) {
+		throw PermissionException("Installing extensions is disabled through configuration");
+	}
+	auto &fs = FileSystem::GetFileSystem(db);
+
+	string local_path = ExtensionDirectory(fs);
 
 	auto extension_name = fs.ExtractBaseName(extension);
 
@@ -132163,6 +132402,7 @@ private:
 
 
 
+
 namespace duckdb {
 
 using ExpressionValueInformation = FilterCombiner::ExpressionValueInformation;
@@ -135716,6 +135956,7 @@ public:
 
 } // namespace duckdb
 
+
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -147397,6 +147638,7 @@ unique_ptr<ParsedExpression> BetweenExpression::Deserialize(ExpressionType type,
 
 
 
+
 namespace duckdb {
 
 CaseExpression::CaseExpression() : ParsedExpression(ExpressionType::CASE_EXPR, ExpressionClass::CASE) {
@@ -148597,6 +148839,7 @@ string KeywordHelper::WriteOptionallyQuoted(const string &text, char quote) {
 
 
 
+
 namespace duckdb {
 
 AlterInfo::AlterInfo(AlterType type, string schema_p, string name_p)
@@ -160127,6 +160370,7 @@ unique_ptr<AlterStatement> Transformer::TransformAlterSequence(duckdb_libpgquery
 
 
 
+
 namespace duckdb {
 
 unique_ptr<AlterStatement> Transformer::TransformAlter(duckdb_libpgquery::PGNode *node) {
@@ -160844,6 +161088,7 @@ unique_ptr<CreateStatement> Transformer::TransformCreateType(duckdb_libpgquery::
 
 
 
+
 namespace duckdb {
 
 unique_ptr<CreateStatement> Transformer::TransformCreateView(duckdb_libpgquery::PGNode *node) {
@@ -171350,6 +171595,7 @@ BindResult ColumnAliasBinder::BindAlias(ExpressionBinder &enclosing_binder, Colu
 
 } // namespace duckdb
 
+
 namespace duckdb {
 
 ConstantBinder::ConstantBinder(Binder &binder, ClientContext &context, string clause)
@@ -174867,6 +175113,7 @@ public:
 #endif
 
 
+
 namespace duckdb {
 
 BlockHandle::BlockHandle(DatabaseInstance &db, block_id_t block_id_p)
@@ -178625,6 +178872,7 @@ public:
 
 
 
+
 namespace duckdb {
 
 struct StringHash {
@@ -179272,6 +179520,7 @@ bool DictionaryCompressionFun::TypeIsSupported(PhysicalType type) {
 
 
 
+
 namespace duckdb {
 
 //===--------------------------------------------------------------------===//
@@ -188683,6 +188932,7 @@ struct UpdateInfo {
 
 } // namespace duckdb
 
+
 namespace duckdb {
 
 static UpdateSegment::initialize_update_function_t GetInitializeUpdateFunction(PhysicalType type);