duckdb 0.6.2-dev1736.0 → 0.6.2-dev1746.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.6.2-dev1736.0",
+  "version": "0.6.2-dev1746.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/src/catalog/default/default_functions.cpp

@@ -93,6 +93,7 @@ static DefaultMacro internal_macros[] = {
 	{DEFAULT_SCHEMA, "generate_subscripts", {"arr", "dim", nullptr}, "unnest(generate_series(1, array_length(arr, dim)))"},
 	{DEFAULT_SCHEMA, "fdiv", {"x", "y", nullptr}, "floor(x/y)"},
 	{DEFAULT_SCHEMA, "fmod", {"x", "y", nullptr}, "(x-y*floor(x/y))"},
+	{DEFAULT_SCHEMA, "count_if", {"l", nullptr}, "sum(if(l, 1, 0))"},
 
 	// algebraic list aggregates
 	{DEFAULT_SCHEMA, "list_avg", {"l", nullptr}, "list_aggr(l, 'avg')"},

package/src/duckdb/src/common/file_system.cpp

@@ -172,7 +172,7 @@ string FileSystem::ConvertSeparators(const string &path) {
 	return result;
 }
 
-string FileSystem::ExtractBaseName(const string &path) {
+string FileSystem::ExtractName(const string &path) {
 	if (path.empty()) {
 		return string();
 	}
@@ -180,7 +180,14 @@ string FileSystem::ExtractBaseName(const string &path) {
 	auto sep = PathSeparator();
 	auto splits = StringUtil::Split(normalized_path, sep);
 	D_ASSERT(!splits.empty());
-	auto vec = StringUtil::Split(splits.back(), ".");
+	return splits.back();
+}
+
+string FileSystem::ExtractBaseName(const string &path) {
+	if (path.empty()) {
+		return string();
+	}
+	auto vec = StringUtil::Split(ExtractName(path), ".");
 	D_ASSERT(!vec.empty());
 	return vec[0];
 }
@@ -239,6 +246,14 @@ int64_t FileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_bytes) {
 	throw NotImplementedException("%s: Write is not implemented!", GetName());
 }
 
+string FileSystem::GetFileExtension(FileHandle &handle) {
+	auto dot_location = handle.path.rfind('.');
+	if (dot_location != std::string::npos) {
+		return handle.path.substr(dot_location + 1, std::string::npos);
+	}
+	return string();
+}
+
 int64_t FileSystem::GetFileSize(FileHandle &handle) {
 	throw NotImplementedException("%s: GetFileSize is not implemented!", GetName());
 }

package/src/duckdb/src/function/pragma/pragma_queries.cpp

@@ -1,6 +1,9 @@
 #include "duckdb/function/pragma/pragma_functions.hpp"
 #include "duckdb/common/string_util.hpp"
 #include "duckdb/common/file_system.hpp"
+#include "duckdb/parser/statement/export_statement.hpp"
+#include "duckdb/parser/statement/copy_statement.hpp"
+#include "duckdb/parser/parser.hpp"
 #include "duckdb/main/config.hpp"
 
 namespace duckdb {
@@ -73,7 +76,7 @@ string PragmaImportDatabase(ClientContext &context, const FunctionParameters &pa
 	auto &fs = FileSystem::GetFileSystem(context);
 	auto *opener = FileSystem::GetFileOpener(context);
 
-	string query;
+	string final_query;
 	// read the "shema.sql" and "load.sql" files
 	vector<string> files = {"schema.sql", "load.sql"};
 	for (auto &file : files) {
@@ -83,10 +86,25 @@ string PragmaImportDatabase(ClientContext &context, const FunctionParameters &pa
 		auto fsize = fs.GetFileSize(*handle);
 		auto buffer = unique_ptr<char[]>(new char[fsize]);
 		fs.Read(*handle, buffer.get(), fsize);
-
-		query += string(buffer.get(), fsize);
+		auto query = string(buffer.get(), fsize);
+		// Replace the placeholder with the path provided to IMPORT
+		if (file == "load.sql") {
+			Parser parser;
+			parser.ParseQuery(query);
+			auto copy_statements = move(parser.statements);
+			query.clear();
+			for (auto &statement_p : copy_statements) {
+				D_ASSERT(statement_p->type == StatementType::COPY_STATEMENT);
+				auto &statement = (CopyStatement &)*statement_p;
+				auto &info = *statement.info;
+				auto file_name = fs.ExtractName(info.file_path);
+				info.file_path = fs.JoinPath(parameters.values[0].ToString(), file_name);
+				query += statement.ToString() + ";";
+			}
+		}
+		final_query += query;
 	}
-	return query;
+	return final_query;
 }
 
 string PragmaDatabaseSize(ClientContext &context, const FunctionParameters &parameters) {

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.6.2-dev1736"
+#define DUCKDB_VERSION "0.6.2-dev1746"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "424848838c"
+#define DUCKDB_SOURCE_ID "6137998b60"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/common/file_system.hpp

@@ -128,6 +128,8 @@ public:
 	//! Write nr_bytes from the buffer into the file, moving the file pointer forward by nr_bytes.
 	DUCKDB_API virtual int64_t Write(FileHandle &handle, void *buffer, int64_t nr_bytes);
 
+	//! Returns the extension of the file, or empty string if no extension was found.
+	DUCKDB_API string GetFileExtension(FileHandle &handle);
 	//! Returns the file size of a file handle, returns -1 on error
 	DUCKDB_API virtual int64_t GetFileSize(FileHandle &handle);
 	//! Returns the file last modified time of a file handle, returns timespec with zero on all attributes on error
@@ -176,8 +178,10 @@ public:
 	DUCKDB_API static string JoinPath(const string &a, const string &path);
 	//! Convert separators in a path to the local separators (e.g. convert "/" into \\ on windows)
 	DUCKDB_API static string ConvertSeparators(const string &path);
-	//! Extract the base name of a file (e.g. if the input is lib/example.dll the base name is example)
+	//! Extract the base name of a file (e.g. if the input is lib/example.dll the base name is 'example')
 	DUCKDB_API static string ExtractBaseName(const string &path);
+	//! Extract the name of a file (e.g if the input is lib/example.dll the name is 'example.dll')
+	DUCKDB_API static string ExtractName(const string &path);
 
 	//! Runs a glob on the file system, returning a list of matching files
 	DUCKDB_API virtual vector<string> Glob(const string &path, FileOpener *opener = nullptr);

package/src/duckdb/src/include/duckdb/parser/parsed_data/copy_info.hpp

@@ -27,12 +27,12 @@ struct CopyInfo : public ParseInfo {
 	string table;
 	//! List of columns to copy to/from
 	vector<string> select_list;
-	//! The file path to copy to/from
-	string file_path;
 	//! Whether or not this is a copy to file (false) or copy from a file (true)
 	bool is_from;
 	//! The file format of the external file
 	string format;
+	//! The file path to copy to/from
+	string file_path;
 	//! Set of (key, value) options
 	unordered_map<string, vector<Value>> options;
 

package/src/duckdb/src/include/duckdb/parser/statement/copy_statement.hpp

@@ -21,11 +21,15 @@ public:
 	unique_ptr<CopyInfo> info;
 	// The SQL statement used instead of a table when copying data out to a file
 	unique_ptr<QueryNode> select_statement;
+	string ToString() const override;
+	string CopyOptionsToString(const string &format, const unordered_map<string, vector<Value>> &options) const;
 
 protected:
 	CopyStatement(const CopyStatement &other);
 
 public:
 	unique_ptr<SQLStatement> Copy() const override;
+
+private:
 };
 } // namespace duckdb

package/src/duckdb/src/main/client_context.cpp

@@ -665,6 +665,7 @@ unique_ptr<PendingQueryResult> ClientContext::PendingStatementOrPreparedStatemen
 			statement = std::move(copied_statement);
 			break;
 		}
+		case StatementType::COPY_STATEMENT:
 		case StatementType::INSERT_STATEMENT:
 		case StatementType::DELETE_STATEMENT:
 		case StatementType::UPDATE_STATEMENT: {

package/src/duckdb/src/parser/statement/copy_statement.cpp

@@ -11,6 +11,108 @@ CopyStatement::CopyStatement(const CopyStatement &other) : SQLStatement(other),
 	}
 }
 
+string ConvertOptionValueToString(const Value &val) {
+	auto type = val.type().id();
+	switch (type) {
+	case LogicalTypeId::VARCHAR:
+		return KeywordHelper::WriteOptionallyQuoted(val.ToString());
+	default:
+		return val.ToString();
+	}
+}
+
+string CopyStatement::CopyOptionsToString(const string &format,
+                                          const unordered_map<string, vector<Value>> &options) const {
+	if (format.empty() && options.empty()) {
+		return string();
+	}
+	string result;
+
+	result += " (";
+	if (!format.empty()) {
+		result += " FORMAT ";
+		result += format;
+	}
+	for (auto it = options.begin(); it != options.end(); it++) {
+		if (!format.empty() || it != options.begin()) {
+			result += ", ";
+		}
+		auto &name = it->first;
+		auto &values = it->second;
+
+		result += name + " ";
+		if (values.empty()) {
+			// Options like HEADER don't need an explicit value
+			// just providing the name already sets it to true
+		} else if (values.size() == 1) {
+			result += ConvertOptionValueToString(values[0]);
+		} else {
+			result += "( ";
+			for (idx_t i = 0; i < values.size(); i++) {
+				auto &value = values[i];
+				if (i) {
+					result += ", ";
+				}
+				result += KeywordHelper::WriteOptionallyQuoted(value.ToString());
+			}
+			result += " )";
+		}
+	}
+	result += " )";
+	return result;
+}
+
+// COPY table-name (c1, c2, ..)
+string TablePart(const CopyInfo &info) {
+	string result;
+
+	if (!info.catalog.empty()) {
+		result += KeywordHelper::WriteOptionallyQuoted(info.catalog) + ".";
+	}
+	if (!info.schema.empty()) {
+		result += KeywordHelper::WriteOptionallyQuoted(info.schema) + ".";
+	}
+	D_ASSERT(!info.table.empty());
+	result += KeywordHelper::WriteOptionallyQuoted(info.table);
+
+	// (c1, c2, ..)
+	if (!info.select_list.empty()) {
+		result += " (";
+		for (idx_t i = 0; i < info.select_list.size(); i++) {
+			if (i > 0) {
+				result += ", ";
+			}
+			result += KeywordHelper::WriteOptionallyQuoted(info.select_list[i]);
+		}
+		result += " )";
+	}
+	return result;
+}
+
+string CopyStatement::ToString() const {
+	string result;
+
+	result += "COPY ";
+	if (info->is_from) {
+		D_ASSERT(!select_statement);
+		result += TablePart(*info);
+		result += " FROM";
+		result += StringUtil::Format(" '%s'", info->file_path);
+		result += CopyOptionsToString(info->format, info->options);
+	} else {
+		if (select_statement) {
+			// COPY (select-node) TO ...
+			result += "(" + select_statement->ToString() + ")";
+		} else {
+			result += TablePart(*info);
+		}
+		result += " TO";
+		result += StringUtil::Format("'%s'", info->file_path);
+		result += CopyOptionsToString(info->format, info->options);
+	}
+	return result;
+}
+
 unique_ptr<SQLStatement> CopyStatement::Copy() const {
 	return unique_ptr<CopyStatement>(new CopyStatement(*this));
 }

package/src/duckdb/src/planner/binder/statement/bind_export.cpp

@@ -85,6 +85,15 @@ void ReorderTableEntries(vector<TableCatalogEntry *> &tables) {
 	tables = ordered;
 }
 
+string CreateFileName(const string &id_suffix, TableCatalogEntry *table, const string &extension) {
+	auto name = SanitizeExportIdentifier(table->name);
+	if (table->schema->name == DEFAULT_SCHEMA) {
+		return StringUtil::Format("%s%s.%s", name, id_suffix, extension);
+	}
+	auto schema = SanitizeExportIdentifier(table->schema->name);
+	return StringUtil::Format("%s_%s%s.%s", schema, name, id_suffix, extension);
+}
+
 BoundStatement Binder::Bind(ExportStatement &stmt) {
 	// COPY TO a file
 	auto &config = DBConfig::GetConfig(context);
@@ -135,20 +144,13 @@ BoundStatement Binder::Bind(ExportStatement &stmt) {
 		idx_t id = 0;
 		while (true) {
 			string id_suffix = id == 0 ? string() : "_" + to_string(id);
-			if (table->schema->name == DEFAULT_SCHEMA) {
-				info->file_path = fs.JoinPath(stmt.info->file_path,
-				                              StringUtil::Format("%s%s.%s", SanitizeExportIdentifier(table->name),
-				                                                 id_suffix, copy_function->function.extension));
-			} else {
-				info->file_path =
-				    fs.JoinPath(stmt.info->file_path,
-				                StringUtil::Format("%s_%s%s.%s", SanitizeExportIdentifier(table->schema->name),
-				                                   SanitizeExportIdentifier(table->name), id_suffix,
-				                                   copy_function->function.extension));
-			}
-			if (table_name_index.find(info->file_path) == table_name_index.end()) {
+			auto name = CreateFileName(id_suffix, table, copy_function->function.extension);
+			auto directory = stmt.info->file_path;
+			auto full_path = fs.JoinPath(directory, name);
+			info->file_path = full_path;
+			auto insert_result = table_name_index.insert(info->file_path);
+			if (insert_result.second == true) {
 				// this name was not yet taken: take it
-				table_name_index.insert(info->file_path);
 				break;
 			}
 			id++;
@@ -166,6 +168,7 @@ BoundStatement Binder::Bind(ExportStatement &stmt) {
 		exported_data.database_name = catalog;
 		exported_data.table_name = info->table;
 		exported_data.schema_name = info->schema;
+
 		exported_data.file_path = info->file_path;
 
 		ExportedTableInfo table_info;