duckdb 0.10.3-dev3.0 → 0.10.3-dev6.0

package/src/duckdb/src/common/local_file_system.cpp

@@ -40,7 +40,11 @@ extern "C" WINBASEAPI BOOL WINAPI GetPhysicallyInstalledSystemMemory(PULONGLONG)
 #endif
 
 #if defined(__linux__)
-#include <linux/falloc.h>
+// See https://man7.org/linux/man-pages/man2/fallocate.2.html
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE /* See feature_test_macros(7) */
+#endif
+#include <fcntl.h>
 #include <libgen.h>
 // See e.g.:
 // https://opensource.apple.com/source/CarbonHeaders/CarbonHeaders-18.1/TargetConditionals.h.auto.html
@@ -379,6 +383,10 @@ unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path_p, FileOpenF
 						           "using the -readonly parameter in the CLI";
 					}
 				}
+				rc = close(fd);
+				if (rc == -1) {
+					message += ". Also, failed closing file";
+				}
 				message += ". See also https://duckdb.org/docs/connect/concurrency";
 				throw IOException("Could not set lock on file \"%s\": %s", {{"errno", std::to_string(retained_errno)}},
 				                  path, message);
@@ -390,7 +398,7 @@ unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path_p, FileOpenF
 
 void LocalFileSystem::SetFilePointer(FileHandle &handle, idx_t location) {
 	int fd = handle.Cast<UnixFileHandle>().fd;
-	off_t offset = lseek(fd, location, SEEK_SET);
+	off_t offset = lseek(fd, UnsafeNumericCast<off_t>(location), SEEK_SET);
 	if (offset == (off_t)-1) {
 		throw IOException("Could not seek to location %lld for file \"%s\": %s", {{"errno", std::to_string(errno)}},
 		                  location, handle.path, strerror(errno));
@@ -404,14 +412,15 @@ idx_t LocalFileSystem::GetFilePointer(FileHandle &handle) {
 		throw IOException("Could not get file position file \"%s\": %s", {{"errno", std::to_string(errno)}},
 		                  handle.path, strerror(errno));
 	}
-	return position;
+	return UnsafeNumericCast<idx_t>(position);
 }
 
 void LocalFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) {
 	int fd = handle.Cast<UnixFileHandle>().fd;
 	auto read_buffer = char_ptr_cast(buffer);
 	while (nr_bytes > 0) {
-		int64_t bytes_read = pread(fd, read_buffer, nr_bytes, location);
+		int64_t bytes_read =
+		    pread(fd, read_buffer, UnsafeNumericCast<size_t>(nr_bytes), UnsafeNumericCast<off_t>(location));
 		if (bytes_read == -1) {
 			throw IOException("Could not read from file \"%s\": %s", {{"errno", std::to_string(errno)}}, handle.path,
 			                  strerror(errno));
@@ -423,13 +432,13 @@ void LocalFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes, i
 		}
 		read_buffer += bytes_read;
 		nr_bytes -= bytes_read;
-		location += bytes_read;
+		location += UnsafeNumericCast<idx_t>(bytes_read);
 	}
 }
 
 int64_t LocalFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes) {
 	int fd = handle.Cast<UnixFileHandle>().fd;
-	int64_t bytes_read = read(fd, buffer, nr_bytes);
+	int64_t bytes_read = read(fd, buffer, UnsafeNumericCast<size_t>(nr_bytes));
 	if (bytes_read == -1) {
 		throw IOException("Could not read from file \"%s\": %s", {{"errno", std::to_string(errno)}}, handle.path,
 		                  strerror(errno));
@@ -441,7 +450,8 @@ void LocalFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_bytes,
 	int fd = handle.Cast<UnixFileHandle>().fd;
 	auto write_buffer = char_ptr_cast(buffer);
 	while (nr_bytes > 0) {
-		int64_t bytes_written = pwrite(fd, write_buffer, nr_bytes, location);
+		int64_t bytes_written =
+		    pwrite(fd, write_buffer, UnsafeNumericCast<size_t>(nr_bytes), UnsafeNumericCast<off_t>(location));
 		if (bytes_written < 0) {
 			throw IOException("Could not write file \"%s\": %s", {{"errno", std::to_string(errno)}}, handle.path,
 			                  strerror(errno));
@@ -452,7 +462,7 @@ void LocalFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_bytes,
 		}
 		write_buffer += bytes_written;
 		nr_bytes -= bytes_written;
-		location += bytes_written;
+		location += UnsafeNumericCast<idx_t>(bytes_written);
 	}
 }
 
@@ -534,7 +544,8 @@ void LocalFileSystem::CreateDirectory(const string &directory, optional_ptr<File
 	if (stat(directory.c_str(), &st) != 0) {
 		/* Directory does not exist. EEXIST for race condition */
 		if (mkdir(directory.c_str(), 0755) != 0 && errno != EEXIST) {
-			throw IOException("Failed to create directory \"%s\"!", {{"errno", std::to_string(errno)}}, directory);
+			throw IOException("Failed to create directory \"%s\": %s", {{"errno", std::to_string(errno)}}, directory,
+			                  strerror(errno));
 		}
 	} else if (!S_ISDIR(st.st_mode)) {
 		throw IOException("Failed to create directory \"%s\": path exists but is not a directory!",
@@ -598,10 +609,15 @@ bool LocalFileSystem::ListFiles(const string &directory, const std::function<voi
 	if (!DirectoryExists(directory, opener)) {
 		return false;
 	}
-	DIR *dir = opendir(directory.c_str());
+
+	auto dir = opendir(directory.c_str());
 	if (!dir) {
 		return false;
 	}
+
+	// RAII wrapper around DIR to automatically free on exceptions in callback
+	std::unique_ptr<DIR, std::function<void(DIR *)>> dir_unique_ptr(dir, [](DIR *d) { closedir(d); });
+
 	struct dirent *ent;
 	// loop over all files in the directory
 	while ((ent = readdir(dir)) != nullptr) {
@@ -624,7 +640,7 @@ bool LocalFileSystem::ListFiles(const string &directory, const std::function<voi
 		// invoke callback
 		callback(name, status.st_mode & S_IFDIR);
 	}
-	closedir(dir);
+
 	return true;
 }
 
@@ -983,7 +999,8 @@ void LocalFileSystem::CreateDirectory(const string &directory, optional_ptr<File
 	}
 	auto unicode_path = WindowsUtil::UTF8ToUnicode(directory.c_str());
 	if (directory.empty() || !CreateDirectoryW(unicode_path.c_str(), NULL) || !DirectoryExists(directory)) {
-		throw IOException("Could not create directory: \'%s\'", directory.c_str());
+		auto error = LocalFileSystem::GetLastErrorAsString();
+		throw IOException("Failed to create directory \"%s\": %s", directory.c_str(), error);
 	}
 }
 

package/src/duckdb/src/common/multi_file_list.cpp

@@ -0,0 +1,285 @@
+#include "duckdb/common/multi_file_reader.hpp"
+
+#include "duckdb/common/exception.hpp"
+#include "duckdb/common/hive_partitioning.hpp"
+#include "duckdb/common/types.hpp"
+#include "duckdb/function/function_set.hpp"
+#include "duckdb/function/table_function.hpp"
+#include "duckdb/main/config.hpp"
+#include "duckdb/planner/operator/logical_get.hpp"
+#include "duckdb/common/string_util.hpp"
+
+#include <algorithm>
+
+namespace duckdb {
+
+// Helper method to do Filter Pushdown into a MultiFileList
+bool PushdownInternal(ClientContext &context, const MultiFileReaderOptions &options, LogicalGet &get,
+                      vector<unique_ptr<Expression>> &filters, vector<string> &expanded_files) {
+	unordered_map<string, column_t> column_map;
+	for (idx_t i = 0; i < get.column_ids.size(); i++) {
+		if (!IsRowIdColumnId(get.column_ids[i])) {
+			column_map.insert({get.names[get.column_ids[i]], i});
+		}
+	}
+
+	auto start_files = expanded_files.size();
+	HivePartitioning::ApplyFiltersToFileList(context, expanded_files, filters, column_map, get,
+	                                         options.hive_partitioning, options.filename);
+
+	if (expanded_files.size() != start_files) {
+		return true;
+	}
+
+	return false;
+}
+
+//===--------------------------------------------------------------------===//
+// MultiFileListIterator
+//===--------------------------------------------------------------------===//
+MultiFileListIterationHelper MultiFileList::Files() {
+	return MultiFileListIterationHelper(*this);
+}
+
+MultiFileListIterationHelper::MultiFileListIterationHelper(MultiFileList &file_list_p) : file_list(file_list_p) {
+}
+
+MultiFileListIterationHelper::MultiFileListIterator::MultiFileListIterator(MultiFileList *file_list_p)
+    : file_list(file_list_p) {
+	if (!file_list) {
+		return;
+	}
+
+	file_list->InitializeScan(file_scan_data);
+	if (!file_list->Scan(file_scan_data, current_file)) {
+		// There is no first file: move iterator to nop state
+		file_list = nullptr;
+		file_scan_data.current_file_idx = DConstants::INVALID_INDEX;
+	}
+}
+
+void MultiFileListIterationHelper::MultiFileListIterator::Next() {
+	if (!file_list) {
+		return;
+	}
+
+	if (!file_list->Scan(file_scan_data, current_file)) {
+		// exhausted collection: move iterator to nop state
+		file_list = nullptr;
+		file_scan_data.current_file_idx = DConstants::INVALID_INDEX;
+	}
+}
+
+MultiFileListIterationHelper::MultiFileListIterator MultiFileListIterationHelper::begin() { // NOLINT: match stl API
+	return MultiFileListIterationHelper::MultiFileListIterator(
+	    file_list.GetExpandResult() == FileExpandResult::NO_FILES ? nullptr : &file_list);
+}
+MultiFileListIterationHelper::MultiFileListIterator MultiFileListIterationHelper::end() { // NOLINT: match stl API
+	return MultiFileListIterationHelper::MultiFileListIterator(nullptr);
+}
+
+MultiFileListIterationHelper::MultiFileListIterator &MultiFileListIterationHelper::MultiFileListIterator::operator++() {
+	Next();
+	return *this;
+}
+
+bool MultiFileListIterationHelper::MultiFileListIterator::operator!=(const MultiFileListIterator &other) const {
+	return file_list != other.file_list || file_scan_data.current_file_idx != other.file_scan_data.current_file_idx;
+}
+
+const string &MultiFileListIterationHelper::MultiFileListIterator::operator*() const {
+	return current_file;
+}
+
+//===--------------------------------------------------------------------===//
+// MultiFileList
+//===--------------------------------------------------------------------===//
+MultiFileList::MultiFileList(vector<string> paths, FileGlobOptions options)
+    : paths(std::move(paths)), glob_options(options) {
+}
+
+MultiFileList::~MultiFileList() {
+}
+
+const vector<string> MultiFileList::GetPaths() const {
+	return paths;
+}
+
+void MultiFileList::InitializeScan(MultiFileListScanData &iterator) {
+	iterator.current_file_idx = 0;
+}
+
+bool MultiFileList::Scan(MultiFileListScanData &iterator, string &result_file) {
+	D_ASSERT(iterator.current_file_idx != DConstants::INVALID_INDEX);
+	auto maybe_file = GetFile(iterator.current_file_idx);
+
+	if (maybe_file.empty()) {
+		D_ASSERT(iterator.current_file_idx >= GetTotalFileCount());
+		return false;
+	}
+
+	result_file = maybe_file;
+	iterator.current_file_idx++;
+	return true;
+}
+
+unique_ptr<MultiFileList> MultiFileList::ComplexFilterPushdown(ClientContext &context,
+                                                               const MultiFileReaderOptions &options, LogicalGet &get,
+                                                               vector<unique_ptr<Expression>> &filters) {
+	// By default the filter pushdown into a multifilelist does nothing
+	return nullptr;
+}
+
+string MultiFileList::GetFirstFile() {
+	return GetFile(0);
+}
+
+bool MultiFileList::IsEmpty() {
+	return GetExpandResult() == FileExpandResult::NO_FILES;
+}
+
+//===--------------------------------------------------------------------===//
+// SimpleMultiFileList
+//===--------------------------------------------------------------------===//
+SimpleMultiFileList::SimpleMultiFileList(vector<string> paths_p)
+    : MultiFileList(std::move(paths_p), FileGlobOptions::ALLOW_EMPTY) {
+}
+
+unique_ptr<MultiFileList> SimpleMultiFileList::ComplexFilterPushdown(ClientContext &context_p,
+                                                                     const MultiFileReaderOptions &options,
+                                                                     LogicalGet &get,
+                                                                     vector<unique_ptr<Expression>> &filters) {
+	if (!options.hive_partitioning && !options.filename) {
+		return nullptr;
+	}
+
+	// FIXME: don't copy list until first file is filtered
+	auto file_copy = paths;
+	auto res = PushdownInternal(context_p, options, get, filters, file_copy);
+
+	if (res) {
+		return make_uniq<SimpleMultiFileList>(file_copy);
+	}
+
+	return nullptr;
+}
+
+vector<string> SimpleMultiFileList::GetAllFiles() {
+	return paths;
+}
+
+FileExpandResult SimpleMultiFileList::GetExpandResult() {
+	if (paths.size() > 1) {
+		return FileExpandResult::MULTIPLE_FILES;
+	} else if (paths.size() == 1) {
+		return FileExpandResult::SINGLE_FILE;
+	}
+
+	return FileExpandResult::NO_FILES;
+}
+
+string SimpleMultiFileList::GetFile(idx_t i) {
+	if (paths.empty() || i >= paths.size()) {
+		return "";
+	}
+
+	return paths[i];
+}
+
+idx_t SimpleMultiFileList::GetTotalFileCount() {
+	return paths.size();
+}
+
+//===--------------------------------------------------------------------===//
+// GlobMultiFileList
+//===--------------------------------------------------------------------===//
+GlobMultiFileList::GlobMultiFileList(ClientContext &context_p, vector<string> paths_p, FileGlobOptions options)
+    : MultiFileList(std::move(paths_p), options), context(context_p), current_path(0) {
+}
+
+unique_ptr<MultiFileList> GlobMultiFileList::ComplexFilterPushdown(ClientContext &context_p,
+                                                                   const MultiFileReaderOptions &options,
+                                                                   LogicalGet &get,
+                                                                   vector<unique_ptr<Expression>> &filters) {
+	lock_guard<mutex> lck(lock);
+
+	// Expand all
+	// FIXME: lazy expansion
+	// FIXME: push down filters into glob
+	while (ExpandPathInternal()) {
+	}
+
+	if (!options.hive_partitioning && !options.filename) {
+		return nullptr;
+	}
+	auto res = PushdownInternal(context, options, get, filters, expanded_files);
+
+	if (res) {
+		return make_uniq<SimpleMultiFileList>(expanded_files);
+	}
+
+	return nullptr;
+}
+
+vector<string> GlobMultiFileList::GetAllFiles() {
+	lock_guard<mutex> lck(lock);
+	while (ExpandPathInternal()) {
+	}
+	return expanded_files;
+}
+
+idx_t GlobMultiFileList::GetTotalFileCount() {
+	lock_guard<mutex> lck(lock);
+	while (ExpandPathInternal()) {
+	}
+	return expanded_files.size();
+}
+
+FileExpandResult GlobMultiFileList::GetExpandResult() {
+	// GetFile(1) will ensure at least the first 2 files are expanded if they are available
+	GetFile(1);
+
+	if (expanded_files.size() > 1) {
+		return FileExpandResult::MULTIPLE_FILES;
+	} else if (expanded_files.size() == 1) {
+		return FileExpandResult::SINGLE_FILE;
+	}
+
+	return FileExpandResult::NO_FILES;
+}
+
+string GlobMultiFileList::GetFile(idx_t i) {
+	lock_guard<mutex> lck(lock);
+	return GetFileInternal(i);
+}
+
+string GlobMultiFileList::GetFileInternal(idx_t i) {
+	while (expanded_files.size() <= i) {
+		if (!ExpandPathInternal()) {
+			return "";
+		}
+	}
+	D_ASSERT(expanded_files.size() > i);
+	return expanded_files[i];
+}
+
+bool GlobMultiFileList::ExpandPathInternal() {
+	if (IsFullyExpanded()) {
+		return false;
+	}
+
+	auto &fs = FileSystem::GetFileSystem(context);
+	auto glob_files = fs.GlobFiles(paths[current_path], context, glob_options);
+	std::sort(glob_files.begin(), glob_files.end());
+	expanded_files.insert(expanded_files.end(), glob_files.begin(), glob_files.end());
+
+	current_path++;
+
+	return true;
+}
+
+bool GlobMultiFileList::IsFullyExpanded() {
+	return current_path == paths.size();
+}
+
+} // namespace duckdb