duckdb 0.9.3-dev2.0 → 0.10.0

package/src/duckdb/src/common/http_state.cpp

@@ -10,13 +10,14 @@ CachedFileHandle::CachedFileHandle(shared_ptr<CachedFile> &file_p) {
 	file = file_p;
 }
 
-void CachedFileHandle::SetInitialized() {
+void CachedFileHandle::SetInitialized(idx_t total_size) {
 	if (file->initialized) {
 		throw InternalException("Cannot set initialized on cached file that was already initialized");
 	}
 	if (!lock) {
 		throw InternalException("Cannot set initialized on cached file without lock");
 	}
+	file->size = total_size;
 	file->initialized = true;
 	lock = nullptr;
 }
@@ -57,10 +58,26 @@ void HTTPState::Reset() {
 	cached_files.clear();
 }
 
-shared_ptr<HTTPState> HTTPState::TryGetState(FileOpener *opener) {
+shared_ptr<HTTPState> HTTPState::TryGetState(ClientContext &context, bool create_on_missing) {
+	auto lookup = context.registered_state.find("http_state");
+
+	if (lookup != context.registered_state.end()) {
+		return std::static_pointer_cast<HTTPState, ClientContextState>(lookup->second);
+	}
+
+	if (!create_on_missing) {
+		return nullptr;
+	}
+
+	auto http_state = make_shared<HTTPState>();
+	context.registered_state["http_state"] = http_state;
+	return http_state;
+}
+
+shared_ptr<HTTPState> HTTPState::TryGetState(FileOpener *opener, bool create_on_missing) {
 	auto client_context = FileOpener::TryGetClientContext(opener);
 	if (client_context) {
-		return client_context->client_data->http_state;
+		return TryGetState(*client_context, create_on_missing);
 	}
 	return nullptr;
 }

package/src/duckdb/src/common/local_file_system.cpp

@@ -34,6 +34,25 @@ extern "C" WINBASEAPI BOOL WINAPI GetPhysicallyInstalledSystemMemory(PULONGLONG)
 #undef FILE_CREATE // woo mingw
 #endif
 
+// includes for giving a better error message on lock conflicts
+#if defined(__linux__) || defined(__APPLE__)
+#include <pwd.h>
+#endif
+
+#if defined(__linux__)
+#include <libgen.h>
+// See e.g.:
+// https://opensource.apple.com/source/CarbonHeaders/CarbonHeaders-18.1/TargetConditionals.h.auto.html
+#elif defined(__APPLE__)
+#include <TargetConditionals.h>                             // NOLINT
+#if not(defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE == 1) // NOLINT
+#include <libproc.h>                                        // NOLINT
+#endif                                                      // NOLINT
+#elif defined(_WIN32)
+#include <RestartManager.h>
+#pragma comment(lib, "rstrtmgr.lib")
+#endif
+
 namespace duckdb {
 
 static void AssertValidFileFlags(uint8_t flags) {
@@ -162,6 +181,88 @@ static FileType GetFileTypeInternal(int fd) { // LCOV_EXCL_START
 	}
 } // LCOV_EXCL_STOP
 
+#if __APPLE__ && !TARGET_OS_IPHONE
+
+static string AdditionalProcessInfo(FileSystem &fs, pid_t pid) {
+	if (pid == getpid()) {
+		return "Lock is already held in current process, likely another DuckDB instance";
+	}
+
+	string process_name, process_owner;
+	// try to find out more about the process holding the lock
+	struct proc_bsdshortinfo proc;
+	if (proc_pidinfo(pid, PROC_PIDT_SHORTBSDINFO, 0, &proc, PROC_PIDT_SHORTBSDINFO_SIZE) ==
+	    PROC_PIDT_SHORTBSDINFO_SIZE) {
+		process_name = proc.pbsi_comm; // only a short version however, let's take it in case proc_pidpath() below fails
+		// try to get actual name of conflicting process owner
+		auto pw = getpwuid(proc.pbsi_uid);
+		if (pw) {
+			process_owner = pw->pw_name;
+		}
+	}
+	// try to get a better process name (full path)
+	char full_exec_path[PROC_PIDPATHINFO_MAXSIZE];
+	if (proc_pidpath(pid, full_exec_path, PROC_PIDPATHINFO_MAXSIZE) > 0) {
+		// somehow could not get the path, lets use some sensible fallback
+		process_name = full_exec_path;
+	}
+	return StringUtil::Format("Conflicting lock is held in %s%s",
+	                          !process_name.empty() ? StringUtil::Format("%s (PID %d)", process_name, pid)
+	                                                : StringUtil::Format("PID %d", pid),
+	                          !process_owner.empty() ? StringUtil::Format(" by user %s", process_owner) : "");
+}
+
+#elif __linux__
+
+static string AdditionalProcessInfo(FileSystem &fs, pid_t pid) {
+	if (pid == getpid()) {
+		return "Lock is already held in current process, likely another DuckDB instance";
+	}
+	string process_name, process_owner;
+
+	try {
+		auto cmdline_file = fs.OpenFile(StringUtil::Format("/proc/%d/cmdline", pid), FileFlags::FILE_FLAGS_READ);
+		auto cmdline = cmdline_file->ReadLine();
+		process_name = basename(const_cast<char *>(cmdline.c_str()));
+	} catch (std::exception &) {
+		// ignore
+	}
+
+	// we would like to provide a full path to the executable if possible but we might not have rights
+	{
+		char exe_target[PATH_MAX];
+		memset(exe_target, '\0', PATH_MAX);
+		auto proc_exe_link = StringUtil::Format("/proc/%d/exe", pid);
+		auto readlink_n = readlink(proc_exe_link.c_str(), exe_target, PATH_MAX);
+		if (readlink_n > 0) {
+			process_name = exe_target;
+		}
+	}
+
+	// try to find out who created that process
+	try {
+		auto loginuid_file = fs.OpenFile(StringUtil::Format("/proc/%d/loginuid", pid), FileFlags::FILE_FLAGS_READ);
+		auto uid = std::stoi(loginuid_file->ReadLine());
+		auto pw = getpwuid(uid);
+		if (pw) {
+			process_owner = pw->pw_name;
+		}
+	} catch (std::exception &) {
+		// ignore
+	}
+
+	return StringUtil::Format("Conflicting lock is held in %s%s",
+	                          !process_name.empty() ? StringUtil::Format("%s (PID %d)", process_name, pid)
+	                                                : StringUtil::Format("PID %d", pid),
+	                          !process_owner.empty() ? StringUtil::Format(" by user %s", process_owner) : "");
+}
+
+#else
+static string AdditionalProcessInfo(FileSystem &fs, pid_t pid) {
+	return "";
+}
+#endif
+
 unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path_p, uint8_t flags, FileLockType lock_type,
                                                  FileCompressionType compression, FileOpener *opener) {
 	auto path = FileSystem::ExpandPath(path_p, opener);
@@ -199,7 +300,7 @@ unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path_p, uint8_t f
 	}
 	if (flags & FileFlags::FILE_FLAGS_DIRECT_IO) {
 #if defined(__sun) && defined(__SVR4)
-		throw Exception("DIRECT_IO not supported on Solaris");
+		throw InvalidInputException("DIRECT_IO not supported on Solaris");
 #endif
 #if defined(__DARWIN__) || defined(__APPLE__) || defined(__OpenBSD__)
 		// OSX does not have O_DIRECT, instead we need to use fcntl afterwards to support direct IO
@@ -210,7 +311,7 @@ unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path_p, uint8_t f
 	}
 	int fd = open(path.c_str(), open_flags, 0666);
 	if (fd == -1) {
-		throw IOException("Cannot open file \"%s\": %s", path, strerror(errno));
+		throw IOException("Cannot open file \"%s\": %s", {{"errno", std::to_string(errno)}}, path, strerror(errno));
 	}
 	// #if defined(__DARWIN__) || defined(__APPLE__)
 	// 	if (flags & FileFlags::FILE_FLAGS_DIRECT_IO) {
@@ -233,8 +334,30 @@ unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path_p, uint8_t f
 			fl.l_start = 0;
 			fl.l_len = 0;
 			rc = fcntl(fd, F_SETLK, &fl);
+			// Retain the original error.
+			int retained_errno = errno;
 			if (rc == -1) {
-				throw IOException("Could not set lock on file \"%s\": %s", path, strerror(errno));
+				string message;
+				// try to find out who is holding the lock using F_GETLK
+				rc = fcntl(fd, F_GETLK, &fl);
+				if (rc == -1) { // fnctl does not want to help us
+					message = strerror(errno);
+				} else {
+					message = AdditionalProcessInfo(*this, fl.l_pid);
+				}
+
+				if (lock_type == FileLockType::WRITE_LOCK) {
+					// maybe we can get a read lock instead and tell this to the user.
+					fl.l_type = F_RDLCK;
+					rc = fcntl(fd, F_SETLK, &fl);
+					if (rc != -1) { // success!
+						message += ". However, you would be able to open this database in read-only mode, e.g. by "
+						           "using the -readonly parameter in the CLI";
+					}
+				}
+				message += ". See also https://duckdb.org/docs/connect/concurrency";
+				throw IOException("Could not set lock on file \"%s\": %s", {{"errno", std::to_string(retained_errno)}},
+				                  path, message);
 			}
 		}
 	}
@@ -245,8 +368,8 @@ void LocalFileSystem::SetFilePointer(FileHandle &handle, idx_t location) {
 	int fd = handle.Cast<UnixFileHandle>().fd;
 	off_t offset = lseek(fd, location, SEEK_SET);
 	if (offset == (off_t)-1) {
-		throw IOException("Could not seek to location %lld for file \"%s\": %s", location, handle.path,
-		                  strerror(errno));
+		throw IOException("Could not seek to location %lld for file \"%s\": %s", {{"errno", std::to_string(errno)}},
+		                  location, handle.path, strerror(errno));
 	}
 }
 
@@ -254,7 +377,8 @@ idx_t LocalFileSystem::GetFilePointer(FileHandle &handle) {
 	int fd = handle.Cast<UnixFileHandle>().fd;
 	off_t position = lseek(fd, 0, SEEK_CUR);
 	if (position == (off_t)-1) {
-		throw IOException("Could not get file position file \"%s\": %s", handle.path, strerror(errno));
+		throw IOException("Could not get file position file \"%s\": %s", {{"errno", std::to_string(errno)}},
+		                  handle.path, strerror(errno));
 	}
 	return position;
 }
@@ -265,7 +389,8 @@ void LocalFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes, i
 	while (nr_bytes > 0) {
 		int64_t bytes_read = pread(fd, read_buffer, nr_bytes, location);
 		if (bytes_read == -1) {
-			throw IOException("Could not read from file \"%s\": %s", handle.path, strerror(errno));
+			throw IOException("Could not read from file \"%s\": %s", {{"errno", std::to_string(errno)}}, handle.path,
+			                  strerror(errno));
 		}
 		if (bytes_read == 0) {
 			throw IOException(
@@ -281,7 +406,8 @@ int64_t LocalFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes
 	int fd = handle.Cast<UnixFileHandle>().fd;
 	int64_t bytes_read = read(fd, buffer, nr_bytes);
 	if (bytes_read == -1) {
-		throw IOException("Could not read from file \"%s\": %s", handle.path, strerror(errno));
+		throw IOException("Could not read from file \"%s\": %s", {{"errno", std::to_string(errno)}}, handle.path,
+		                  strerror(errno));
 	}
 	return bytes_read;
 }
@@ -292,7 +418,8 @@ void LocalFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_bytes,
 	while (nr_bytes > 0) {
 		int64_t bytes_written = pwrite(fd, write_buffer, nr_bytes, location);
 		if (bytes_written < 0) {
-			throw IOException("Could not write file \"%s\": %s", handle.path, strerror(errno));
+			throw IOException("Could not write file \"%s\": %s", {{"errno", std::to_string(errno)}}, handle.path,
+			                  strerror(errno));
 		}
 		D_ASSERT(bytes_written >= 0 && bytes_written);
 		write_buffer += bytes_written;
@@ -304,7 +431,8 @@ int64_t LocalFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_byte
 	int fd = handle.Cast<UnixFileHandle>().fd;
 	int64_t bytes_written = write(fd, buffer, nr_bytes);
 	if (bytes_written == -1) {
-		throw IOException("Could not write file \"%s\": %s", handle.path, strerror(errno));
+		throw IOException("Could not write file \"%s\": %s", {{"errno", std::to_string(errno)}}, handle.path,
+		                  strerror(errno));
 	}
 	return bytes_written;
 }
@@ -335,7 +463,8 @@ FileType LocalFileSystem::GetFileType(FileHandle &handle) {
 void LocalFileSystem::Truncate(FileHandle &handle, int64_t new_size) {
 	int fd = handle.Cast<UnixFileHandle>().fd;
 	if (ftruncate(fd, new_size) != 0) {
-		throw IOException("Could not truncate file \"%s\": %s", handle.path, strerror(errno));
+		throw IOException("Could not truncate file \"%s\": %s", {{"errno", std::to_string(errno)}}, handle.path,
+		                  strerror(errno));
 	}
 }
 
@@ -359,10 +488,11 @@ void LocalFileSystem::CreateDirectory(const string &directory) {
 	if (stat(directory.c_str(), &st) != 0) {
 		/* Directory does not exist. EEXIST for race condition */
 		if (mkdir(directory.c_str(), 0755) != 0 && errno != EEXIST) {
-			throw IOException("Failed to create directory \"%s\"!", directory);
+			throw IOException("Failed to create directory \"%s\"!", {{"errno", std::to_string(errno)}}, directory);
 		}
 	} else if (!S_ISDIR(st.st_mode)) {
-		throw IOException("Failed to create directory \"%s\": path exists but is not a directory!", directory);
+		throw IOException("Failed to create directory \"%s\": path exists but is not a directory!",
+		                  {{"errno", std::to_string(errno)}}, directory);
 	}
 }
 
@@ -412,7 +542,8 @@ void LocalFileSystem::RemoveDirectory(const string &directory) {
 
 void LocalFileSystem::RemoveFile(const string &filename) {
 	if (std::remove(filename.c_str()) != 0) {
-		throw IOException("Could not remove file \"%s\": %s", filename, strerror(errno));
+		throw IOException("Could not remove file \"%s\": %s", {{"errno", std::to_string(errno)}}, filename,
+		                  strerror(errno));
 	}
 }
 
@@ -461,7 +592,7 @@ void LocalFileSystem::FileSync(FileHandle &handle) {
 void LocalFileSystem::MoveFile(const string &source, const string &target) {
 	//! FIXME: rename does not guarantee atomicity or overwriting target file if it exists
 	if (rename(source.c_str(), target.c_str()) != 0) {
-		throw IOException("Could not rename file!");
+		throw IOException("Could not rename file!", {{"errno", std::to_string(errno)}});
 	}
 }
 
@@ -515,6 +646,68 @@ public:
 	};
 };
 
+static string AdditionalLockInfo(const std::wstring path) {
+	// try to find out if another process is holding the lock
+
+	// init of the somewhat obscure "Windows Restart Manager"
+	// see also https://devblogs.microsoft.com/oldnewthing/20120217-00/?p=8283
+
+	DWORD session, status, reason;
+	WCHAR session_key[CCH_RM_SESSION_KEY + 1] = {0};
+
+	status = RmStartSession(&session, 0, session_key);
+	if (status != ERROR_SUCCESS) {
+		return "";
+	}
+
+	PCWSTR path_ptr = path.c_str();
+	status = RmRegisterResources(session, 1, &path_ptr, 0, NULL, 0, NULL);
+	if (status != ERROR_SUCCESS) {
+		return "";
+	}
+	UINT process_info_size_needed, process_info_size;
+
+	// we first call with nProcInfo = 0 to find out how much to allocate
+	process_info_size = 0;
+	status = RmGetList(session, &process_info_size_needed, &process_info_size, NULL, &reason);
+	if (status != ERROR_MORE_DATA || process_info_size_needed == 0) {
+		return "";
+	}
+
+	// allocate
+	auto process_info_buffer = duckdb::unique_ptr<RM_PROCESS_INFO[]>(new RM_PROCESS_INFO[process_info_size_needed]);
+	auto process_info = process_info_buffer.get();
+
+	// now call again to get actual data
+	process_info_size = process_info_size_needed;
+	status = RmGetList(session, &process_info_size_needed, &process_info_size, process_info, &reason);
+	if (status != ERROR_SUCCESS || process_info_size == 0) {
+		return "";
+	}
+
+	string conflict_string = "File is already open in ";
+
+	for (UINT process_idx = 0; process_idx < process_info_size; process_idx++) {
+		string process_name = WindowsUtil::UnicodeToUTF8(process_info[process_idx].strAppName);
+		auto pid = process_info[process_idx].Process.dwProcessId;
+
+		// find out full path if possible
+		HANDLE process = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, FALSE, pid);
+		if (process) {
+			WCHAR full_path[MAX_PATH];
+			DWORD full_path_size = MAX_PATH;
+			if (QueryFullProcessImageNameW(process, 0, full_path, &full_path_size) && full_path_size <= MAX_PATH) {
+				process_name = WindowsUtil::UnicodeToUTF8(full_path);
+			}
+			CloseHandle(process);
+		}
+		conflict_string += StringUtil::Format("\n%s (PID %d)", process_name, pid);
+	}
+
+	RmEndSession(session);
+	return conflict_string;
+}
+
 unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path_p, uint8_t flags, FileLockType lock_type,
                                                  FileCompressionType compression, FileOpener *opener) {
 	auto path = FileSystem::ExpandPath(path_p, opener);
@@ -556,6 +749,12 @@ unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path_p, uint8_t f
 	                           flags_and_attributes, NULL);
 	if (hFile == INVALID_HANDLE_VALUE) {
 		auto error = LocalFileSystem::GetLastErrorAsString();
+
+		auto better_error = AdditionalLockInfo(unicode_path);
+		if (!better_error.empty()) {
+			throw IOException(better_error);
+		}
+
 		throw IOException("Cannot open file \"%s\": %s", path.c_str(), error);
 	}
 	auto handle = make_uniq<WindowsFileHandle>(*this, path.c_str(), hFile);

package/src/duckdb/src/common/multi_file_reader.cpp

@@ -1,12 +1,15 @@
 #include "duckdb/common/multi_file_reader.hpp"
-#include "duckdb/function/table_function.hpp"
-#include "duckdb/main/config.hpp"
-#include "duckdb/common/types/value.hpp"
-#include "duckdb/planner/operator/logical_get.hpp"
+
 #include "duckdb/common/exception.hpp"
-#include "duckdb/function/function_set.hpp"
 #include "duckdb/common/hive_partitioning.hpp"
 #include "duckdb/common/types.hpp"
+#include "duckdb/common/types/value.hpp"
+#include "duckdb/function/function_set.hpp"
+#include "duckdb/function/table_function.hpp"
+#include "duckdb/main/config.hpp"
+#include "duckdb/planner/operator/logical_get.hpp"
+
+#include <algorithm>
 
 namespace duckdb {
 
@@ -32,6 +35,10 @@ vector<string> MultiFileReader::GetFileList(ClientContext &context, const Value
 	if (input.type().id() == LogicalTypeId::VARCHAR) {
 		auto file_name = StringValue::Get(input);
 		files = fs.GlobFiles(file_name, context, options);
+
+		// Sort the files to ensure that the order is deterministic
+		std::sort(files.begin(), files.end());
+
 	} else if (input.type().id() == LogicalTypeId::LIST) {
 		for (auto &val : ListValue::GetChildren(input)) {
 			if (val.IsNull()) {
@@ -41,6 +48,7 @@ vector<string> MultiFileReader::GetFileList(ClientContext &context, const Value
 				throw ParserException("%s reader can only take a list of strings as a parameter", name);
 			}
 			auto glob_files = fs.GlobFiles(StringValue::Get(val), context, options);
+			std::sort(glob_files.begin(), glob_files.end());
 			files.insert(files.end(), glob_files.begin(), glob_files.end());
 		}
 	} else {
@@ -49,6 +57,7 @@ vector<string> MultiFileReader::GetFileList(ClientContext &context, const Value
 	if (files.empty() && options == FileGlobOptions::DISALLOW_EMPTY) {
 		throw IOException("%s reader needs at least one file to read", name);
 	}
+
 	return files;
 }
 
@@ -308,6 +317,11 @@ void MultiFileReader::CreateMapping(const string &file_name, const vector<Logica
                                     const string &initial_file) {
 	CreateNameMapping(file_name, local_types, local_names, global_types, global_names, global_column_ids, reader_data,
 	                  initial_file);
+	CreateFilterMap(global_types, filters, reader_data);
+}
+
+void MultiFileReader::CreateFilterMap(const vector<LogicalType> &global_types, optional_ptr<TableFilterSet> filters,
+                                      MultiFileReaderData &reader_data) {
 	if (filters) {
 		reader_data.filter_map.resize(global_types.size());
 		for (idx_t c = 0; c < reader_data.column_mapping.size(); c++) {
@@ -363,8 +377,7 @@ void UnionByName::CombineUnionTypes(const vector<string> &col_names, const vecto
 		if (union_find != union_names_map.end()) {
 			// given same name , union_col's type must compatible with col's type
 			auto &current_type = union_col_types[union_find->second];
-			LogicalType compatible_type;
-			compatible_type = LogicalType::MaxLogicalType(current_type, sql_types[col]);
+			auto compatible_type = LogicalType::ForceMaxLogicalType(current_type, sql_types[col]);
 			union_col_types[union_find->second] = compatible_type;
 		} else {
 			union_names_map[col_names[col]] = union_col_names.size();