duckdb 0.3.3-dev392.0 → 0.3.3-dev432.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.3.3-dev392.0",
+  "version": "0.3.3-dev432.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -13658,6 +13658,10 @@ bool FileSystem::FileExists(const string &filename) {
 	throw NotImplementedException("%s: FileExists is not implemented!", GetName());
 }
 
+bool FileSystem::IsPipe(const string &filename) {
+	throw NotImplementedException("%s: IsPipe is not implemented!", GetName());
+}
+
 void FileSystem::RemoveFile(const string &filename) {
 	throw NotImplementedException("%s: RemoveFile is not implemented!", GetName());
 }
@@ -15773,6 +15777,9 @@ public:
 	void MoveFile(const string &source, const string &target) override;
 	//! Check if a file exists
 	bool FileExists(const string &filename) override;
+
+	//! Check if path is a pipe
+	bool IsPipe(const string &filename) override;
 	//! Remove a file from disk
 	void RemoveFile(const string &filename) override;
 	//! Sync a file handle to disk
@@ -15865,8 +15872,10 @@ public:
 } // namespace duckdb
 
 #include <string>
+#include <io.h>
 
 #ifdef __MINGW32__
+#include <sys/stat.h>
 // need to manually define this for mingw
 extern "C" WINBASEAPI BOOL WINAPI GetPhysicallyInstalledSystemMemory(PULONGLONG);
 #endif
@@ -15891,6 +15900,90 @@ static void AssertValidFileFlags(uint8_t flags) {
 #endif
 }
 
+#ifdef __MINGW32__
+bool LocalFileSystem::FileExists(const string &filename) {
+	auto unicode_path = WindowsUtil::UTF8ToUnicode(filename.c_str());
+	const wchar_t *wpath = unicode_path.c_str();
+	if (_waccess(wpath, 0) == 0) {
+		struct _stat64i32 status;
+		_wstat(wpath, &status);
+		if (status.st_size > 0) {
+			return true;
+		}
+	}
+	return false;
+}
+bool LocalFileSystem::IsPipe(const string &filename) {
+	auto unicode_path = WindowsUtil::UTF8ToUnicode(filename.c_str());
+	const wchar_t *wpath = unicode_path.c_str();
+	if (_waccess(wpath, 0) == 0) {
+		struct _stat64i32 status;
+		_wstat(wpath, &status);
+		if (status.st_size == 0) {
+			return true;
+		}
+	}
+	return false;
+}
+
+#else
+#ifndef _WIN32
+bool LocalFileSystem::FileExists(const string &filename) {
+	if (!filename.empty()) {
+		if (access(filename.c_str(), 0) == 0) {
+			struct stat status;
+			stat(filename.c_str(), &status);
+			if (S_ISREG(status.st_mode)) {
+				return true;
+			}
+		}
+	}
+	// if any condition fails
+	return false;
+}
+
+bool LocalFileSystem::IsPipe(const string &filename) {
+	if (!filename.empty()) {
+		if (access(filename.c_str(), 0) == 0) {
+			struct stat status;
+			stat(filename.c_str(), &status);
+			if (S_ISFIFO(status.st_mode)) {
+				return true;
+			}
+		}
+	}
+	// if any condition fails
+	return false;
+}
+
+#else
+bool LocalFileSystem::FileExists(const string &filename) {
+	auto unicode_path = WindowsUtil::UTF8ToUnicode(filename.c_str());
+	const wchar_t *wpath = unicode_path.c_str();
+	if (_waccess(wpath, 0) == 0) {
+		struct _stat64i32 status;
+		_wstat(wpath, &status);
+		if (status.st_mode & S_IFREG) {
+			return true;
+		}
+	}
+	return false;
+}
+bool LocalFileSystem::IsPipe(const string &filename) {
+	auto unicode_path = WindowsUtil::UTF8ToUnicode(filename.c_str());
+	const wchar_t *wpath = unicode_path.c_str();
+	if (_waccess(wpath, 0) == 0) {
+		struct _stat64i32 status;
+		_wstat(wpath, &status);
+		if (status.st_mode & _S_IFCHR) {
+			return true;
+		}
+	}
+	return false;
+}
+#endif
+#endif
+
 #ifndef _WIN32
 // somehow sometimes this is missing
 #ifndef O_CLOEXEC
@@ -16127,20 +16220,6 @@ bool LocalFileSystem::DirectoryExists(const string &directory) {
 	return false;
 }
 
-bool LocalFileSystem::FileExists(const string &filename) {
-	if (!filename.empty()) {
-		if (access(filename.c_str(), 0) == 0) {
-			struct stat status;
-			stat(filename.c_str(), &status);
-			if (!(status.st_mode & S_IFDIR)) {
-				return true;
-			}
-		}
-	}
-	// if any condition fails
-	return false;
-}
-
 void LocalFileSystem::CreateDirectory(const string &directory) {
 	struct stat st;
 
@@ -16479,11 +16558,6 @@ bool LocalFileSystem::DirectoryExists(const string &directory) {
 	return (attrs != INVALID_FILE_ATTRIBUTES && (attrs & FILE_ATTRIBUTE_DIRECTORY));
 }
 
-bool LocalFileSystem::FileExists(const string &filename) {
-	DWORD attrs = WindowsGetFileAttributes(filename);
-	return (attrs != INVALID_FILE_ATTRIBUTES && !(attrs & FILE_ATTRIBUTE_DIRECTORY));
-}
-
 void LocalFileSystem::CreateDirectory(const string &directory) {
 	if (DirectoryExists(directory)) {
 		return;
@@ -16642,9 +16716,9 @@ vector<string> LocalFileSystem::Glob(const string &path) {
 	}
 	// first check if the path has a glob at all
 	if (!HasGlob(path)) {
-		// no glob: return only the file (if it exists)
+		// no glob: return only the file (if it exists or is a pipe)
 		vector<string> result;
-		if (FileExists(path)) {
+		if (FileExists(path) || IsPipe(path)) {
 			result.push_back(path);
 		}
 		return result;
@@ -45465,6 +45539,9 @@ public:
 		return FindFileSystem(filename)->FileExists(filename);
 	}
 
+	bool IsPipe(const string &filename) override {
+		return FindFileSystem(filename)->IsPipe(filename);
+	}
 	virtual void RemoveFile(const string &filename) override {
 		FindFileSystem(filename)->RemoveFile(filename);
 	}
@@ -61229,6 +61306,8 @@ public:
 
 	CopyFunction function;
 	unique_ptr<FunctionData> bind_data;
+	string file_path;
+	bool use_tmp_file;
 
 public:
 	// Source interface
@@ -61254,6 +61333,7 @@ public:
 
 
 
+
 #include <algorithm>
 
 namespace duckdb {
@@ -61278,6 +61358,15 @@ public:
 //===--------------------------------------------------------------------===//
 // Sink
 //===--------------------------------------------------------------------===//
+void MoveTmpFile(ClientContext &context, const string &tmp_file_path) {
+	auto &fs = FileSystem::GetFileSystem(context);
+	auto file_path = tmp_file_path.substr(0, tmp_file_path.length() - 4);
+	if (fs.FileExists(file_path)) {
+		fs.RemoveFile(file_path);
+	}
+	fs.MoveFile(tmp_file_path, file_path);
+}
+
 PhysicalCopyToFile::PhysicalCopyToFile(vector<LogicalType> types, CopyFunction function_p,
                                        unique_ptr<FunctionData> bind_data, idx_t estimated_cardinality)
     : PhysicalOperator(PhysicalOperatorType::COPY_TO_FILE, move(types), estimated_cardinality),
@@ -61308,6 +61397,10 @@ SinkFinalizeType PhysicalCopyToFile::Finalize(Pipeline &pipeline, Event &event,
 	auto &gstate = (CopyToFunctionGlobalState &)gstate_p;
 	if (function.copy_to_finalize) {
 		function.copy_to_finalize(context, *bind_data, *gstate.global_state);
+
+		if (use_tmp_file) {
+			MoveTmpFile(context, file_path);
+		}
 	}
 	return SinkFinalizeType::READY;
 }
@@ -61316,7 +61409,7 @@ unique_ptr<LocalSinkState> PhysicalCopyToFile::GetLocalSinkState(ExecutionContex
 	return make_unique<CopyToFunctionLocalState>(function.copy_to_initialize_local(context.client, *bind_data));
 }
 unique_ptr<GlobalSinkState> PhysicalCopyToFile::GetGlobalSinkState(ClientContext &context) const {
-	return make_unique<CopyToFunctionGlobalState>(function.copy_to_initialize_global(context, *bind_data));
+	return make_unique<CopyToFunctionGlobalState>(function.copy_to_initialize_global(context, *bind_data, file_path));
 }
 
 //===--------------------------------------------------------------------===//
@@ -65351,6 +65444,7 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalComparison
 
 
 
+
 namespace duckdb {
 
 class LogicalCopyToFile : public LogicalOperator {
@@ -65360,6 +65454,9 @@ public:
 	}
 	CopyFunction function;
 	unique_ptr<FunctionData> bind_data;
+	std::string file_path;
+	bool use_tmp_file;
+	bool is_file_and_exists;
 
 protected:
 	void ResolveTypes() override {
@@ -65373,8 +65470,14 @@ namespace duckdb {
 
 unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalCopyToFile &op) {
 	auto plan = CreatePlan(*op.children[0]);
+	bool use_tmp_file = op.is_file_and_exists && op.use_tmp_file;
+	if (use_tmp_file) {
+		op.file_path += ".tmp";
+	}
 	// COPY from select statement to file
 	auto copy = make_unique<PhysicalCopyToFile>(op.types, op.function, move(op.bind_data), op.estimated_cardinality);
+	copy->file_path = op.file_path;
+	copy->use_tmp_file = use_tmp_file;
 
 	copy->children.push_back(move(plan));
 	return move(copy);
@@ -95093,10 +95196,11 @@ static unique_ptr<LocalFunctionData> WriteCSVInitializeLocal(ClientContext &cont
 	return move(local_data);
 }
 
-static unique_ptr<GlobalFunctionData> WriteCSVInitializeGlobal(ClientContext &context, FunctionData &bind_data) {
+static unique_ptr<GlobalFunctionData> WriteCSVInitializeGlobal(ClientContext &context, FunctionData &bind_data,
+                                                               const string &file_path) {
 	auto &csv_data = (WriteCSVData &)bind_data;
 	auto &options = csv_data.options;
-	auto global_data = make_unique<GlobalWriteCSVData>(FileSystem::GetFileSystem(context), csv_data.files[0],
+	auto global_data = make_unique<GlobalWriteCSVData>(FileSystem::GetFileSystem(context), file_path,
 	                                                   FileSystem::GetFileOpener(context), options.compression);
 
 	if (options.header) {
@@ -95190,6 +95294,7 @@ static void WriteCSVCombine(ClientContext &context, FunctionData &bind_data, Glo
 //===--------------------------------------------------------------------===//
 void WriteCSVFinalize(ClientContext &context, FunctionData &bind_data, GlobalFunctionData &gstate) {
 	auto &global_state = (GlobalWriteCSVData &)gstate;
+
 	global_state.handle->Close();
 	global_state.handle.reset();
 }
@@ -150270,6 +150375,8 @@ BoundStatement Binder::Bind(CallStatement &stmt) {
 
 
 
+
+
 #include <algorithm>
 
 namespace duckdb {
@@ -150293,11 +150400,24 @@ BoundStatement Binder::BindCopyTo(CopyStatement &stmt) {
 	if (!copy_function->function.copy_to_bind) {
 		throw NotImplementedException("COPY TO is not supported for FORMAT \"%s\"", stmt.info->format);
 	}
-
+	bool use_tmp_file = true;
+	for (auto &option : stmt.info->options) {
+		auto loption = StringUtil::Lower(option.first);
+		if (loption == "use_tmp_file") {
+			use_tmp_file = option.second[0].CastAs(LogicalType::BOOLEAN).GetValue<bool>();
+			stmt.info->options.erase(option.first);
+			break;
+		}
+	}
 	auto function_data =
 	    copy_function->function.copy_to_bind(context, *stmt.info, select_node.names, select_node.types);
 	// now create the copy information
 	auto copy = make_unique<LogicalCopyToFile>(copy_function->function, move(function_data));
+	copy->file_path = stmt.info->file_path;
+	copy->use_tmp_file = use_tmp_file;
+	LocalFileSystem fs;
+	copy->is_file_and_exists = fs.FileExists(copy->file_path);
+
 	copy->AddChild(move(select_node.plan));
 
 	result.plan = move(copy);

package/src/duckdb.hpp

@@ -11,8 +11,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 #pragma once
 #define DUCKDB_AMALGAMATION 1
 #define DUCKDB_AMALGAMATION_EXTENDED 1
-#define DUCKDB_SOURCE_ID "27c81ef25"
-#define DUCKDB_VERSION "v0.3.3-dev392"
+#define DUCKDB_SOURCE_ID "851e8cfd3"
+#define DUCKDB_VERSION "v0.3.3-dev432"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -1930,11 +1930,12 @@ public:
 	DUCKDB_API virtual void MoveFile(const string &source, const string &target);
 	//! Check if a file exists
 	DUCKDB_API virtual bool FileExists(const string &filename);
+	//! Check if path is pipe
+	DUCKDB_API virtual bool IsPipe(const string &filename);
 	//! Remove a file from disk
 	DUCKDB_API virtual void RemoveFile(const string &filename);
 	//! Sync a file handle to disk
 	DUCKDB_API virtual void FileSync(FileHandle &handle);
-
 	//! Sets the working directory
 	DUCKDB_API static void SetWorkingDirectory(const string &path);
 	//! Gets the working directory
@@ -18076,7 +18077,8 @@ struct GlobalFunctionData {
 typedef unique_ptr<FunctionData> (*copy_to_bind_t)(ClientContext &context, CopyInfo &info, vector<string> &names,
                                                    vector<LogicalType> &sql_types);
 typedef unique_ptr<LocalFunctionData> (*copy_to_initialize_local_t)(ClientContext &context, FunctionData &bind_data);
-typedef unique_ptr<GlobalFunctionData> (*copy_to_initialize_global_t)(ClientContext &context, FunctionData &bind_data);
+typedef unique_ptr<GlobalFunctionData> (*copy_to_initialize_global_t)(ClientContext &context, FunctionData &bind_data,
+                                                                      const string &file_path);
 typedef void (*copy_to_sink_t)(ClientContext &context, FunctionData &bind_data, GlobalFunctionData &gstate,
                                LocalFunctionData &lstate, DataChunk &input);
 typedef void (*copy_to_combine_t)(ClientContext &context, FunctionData &bind_data, GlobalFunctionData &gstate,