duckdb 0.10.2-dev0.0 → 0.10.2-dev5.0

package/src/duckdb/src/common/local_file_system.cpp

@@ -40,43 +40,23 @@ extern "C" WINBASEAPI BOOL WINAPI GetPhysicallyInstalledSystemMemory(PULONGLONG)
 #endif
 
 #if defined(__linux__)
+#include <linux/falloc.h>
 #include <libgen.h>
 // See e.g.:
 // https://opensource.apple.com/source/CarbonHeaders/CarbonHeaders-18.1/TargetConditionals.h.auto.html
 #elif defined(__APPLE__)
-#include <TargetConditionals.h>                             // NOLINT
-#if not(defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE == 1) // NOLINT
-#include <libproc.h>                                        // NOLINT
-#endif                                                      // NOLINT
+#include <TargetConditionals.h>
+#if not(defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE == 1)
+#include <libproc.h>
+#endif
 #elif defined(_WIN32)
 #include <restartmanager.h>
 #endif
 
 namespace duckdb {
 
-static void AssertValidFileFlags(uint8_t flags) {
-#ifdef DEBUG
-	bool is_read = flags & FileFlags::FILE_FLAGS_READ;
-	bool is_write = flags & FileFlags::FILE_FLAGS_WRITE;
-	bool is_create = (flags & FileFlags::FILE_FLAGS_FILE_CREATE) || (flags & FileFlags::FILE_FLAGS_FILE_CREATE_NEW);
-	bool is_private = (flags & FileFlags::FILE_FLAGS_PRIVATE);
-
-	// require either READ or WRITE (or both)
-	D_ASSERT(is_read || is_write);
-	// CREATE/Append flags require writing
-	D_ASSERT(is_write || !(flags & FileFlags::FILE_FLAGS_APPEND));
-	D_ASSERT(is_write || !(flags & FileFlags::FILE_FLAGS_FILE_CREATE));
-	D_ASSERT(is_write || !(flags & FileFlags::FILE_FLAGS_FILE_CREATE_NEW));
-	// cannot combine CREATE and CREATE_NEW flags
-	D_ASSERT(!(flags & FileFlags::FILE_FLAGS_FILE_CREATE && flags & FileFlags::FILE_FLAGS_FILE_CREATE_NEW));
-
-	// For is_private can only be set along with a create flag
-	D_ASSERT(!is_private || is_create);
-#endif
-}
-
 #ifndef _WIN32
-bool LocalFileSystem::FileExists(const string &filename) {
+bool LocalFileSystem::FileExists(const string &filename, optional_ptr<FileOpener> opener) {
 	if (!filename.empty()) {
 		if (access(filename.c_str(), 0) == 0) {
 			struct stat status;
@@ -90,7 +70,7 @@ bool LocalFileSystem::FileExists(const string &filename) {
 	return false;
 }
 
-bool LocalFileSystem::IsPipe(const string &filename) {
+bool LocalFileSystem::IsPipe(const string &filename, optional_ptr<FileOpener> opener) {
 	if (!filename.empty()) {
 		if (access(filename.c_str(), 0) == 0) {
 			struct stat status;
@@ -105,7 +85,7 @@ bool LocalFileSystem::IsPipe(const string &filename) {
 }
 
 #else
-bool LocalFileSystem::FileExists(const string &filename) {
+bool LocalFileSystem::FileExists(const string &filename, optional_ptr<FileOpener> opener) {
 	auto unicode_path = WindowsUtil::UTF8ToUnicode(filename.c_str());
 	const wchar_t *wpath = unicode_path.c_str();
 	if (_waccess(wpath, 0) == 0) {
@@ -117,7 +97,7 @@ bool LocalFileSystem::FileExists(const string &filename) {
 	}
 	return false;
 }
-bool LocalFileSystem::IsPipe(const string &filename) {
+bool LocalFileSystem::IsPipe(const string &filename, optional_ptr<FileOpener> opener) {
 	auto unicode_path = WindowsUtil::UTF8ToUnicode(filename.c_str());
 	const wchar_t *wpath = unicode_path.c_str();
 	if (_waccess(wpath, 0) == 0) {
@@ -233,7 +213,7 @@ static string AdditionalProcessInfo(FileSystem &fs, pid_t pid) {
 	try {
 		auto cmdline_file = fs.OpenFile(StringUtil::Format("/proc/%d/cmdline", pid), FileFlags::FILE_FLAGS_READ);
 		auto cmdline = cmdline_file->ReadLine();
-		process_name = basename(const_cast<char *>(cmdline.c_str()));
+		process_name = basename(const_cast<char *>(cmdline.c_str())); // NOLINT: old C API does not take const
 	} catch (std::exception &) {
 		// ignore
 	}
@@ -292,19 +272,19 @@ bool LocalFileSystem::IsPrivateFile(const string &path_p, FileOpener *opener) {
 	return true;
 }
 
-unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path_p, uint8_t flags, FileLockType lock_type,
-                                                 FileCompressionType compression, FileOpener *opener) {
+unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path_p, FileOpenFlags flags,
+                                                 optional_ptr<FileOpener> opener) {
 	auto path = FileSystem::ExpandPath(path_p, opener);
-	if (compression != FileCompressionType::UNCOMPRESSED) {
+	if (flags.Compression() != FileCompressionType::UNCOMPRESSED) {
 		throw NotImplementedException("Unsupported compression type for default file system");
 	}
 
-	AssertValidFileFlags(flags);
+	flags.Verify();
 
 	int open_flags = 0;
 	int rc;
-	bool open_read = flags & FileFlags::FILE_FLAGS_READ;
-	bool open_write = flags & FileFlags::FILE_FLAGS_WRITE;
+	bool open_read = flags.OpenForReading();
+	bool open_write = flags.OpenForWriting();
 	if (open_read && open_write) {
 		open_flags = O_RDWR;
 	} else if (open_read) {
@@ -316,18 +296,18 @@ unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path_p, uint8_t f
 	}
 	if (open_write) {
 		// need Read or Write
-		D_ASSERT(flags & FileFlags::FILE_FLAGS_WRITE);
+		D_ASSERT(flags.OpenForWriting());
 		open_flags |= O_CLOEXEC;
-		if (flags & FileFlags::FILE_FLAGS_FILE_CREATE) {
+		if (flags.CreateFileIfNotExists()) {
 			open_flags |= O_CREAT;
-		} else if (flags & FileFlags::FILE_FLAGS_FILE_CREATE_NEW) {
+		} else if (flags.OverwriteExistingFile()) {
 			open_flags |= O_CREAT | O_TRUNC;
 		}
-		if (flags & FileFlags::FILE_FLAGS_APPEND) {
+		if (flags.OpenForAppending()) {
 			open_flags |= O_APPEND;
 		}
 	}
-	if (flags & FileFlags::FILE_FLAGS_DIRECT_IO) {
+	if (flags.DirectIO()) {
 #if defined(__sun) && defined(__SVR4)
 		throw InvalidInputException("DIRECT_IO not supported on Solaris");
 #endif
@@ -341,7 +321,7 @@ unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path_p, uint8_t f
 
 	// Determine permissions
 	mode_t filesec;
-	if (flags & FileFlags::FILE_FLAGS_PRIVATE) {
+	if (flags.CreatePrivateFile()) {
 		open_flags |= O_EXCL; // Ensure we error on existing files or the permissions may not set
 		filesec = 0600;
 	} else {
@@ -352,6 +332,9 @@ unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path_p, uint8_t f
 	int fd = open(path.c_str(), open_flags, filesec);
 
 	if (fd == -1) {
+		if (flags.ReturnNullIfNotExists() && errno == ENOENT) {
+			return nullptr;
+		}
 		throw IOException("Cannot open file \"%s\": %s", {{"errno", std::to_string(errno)}}, path, strerror(errno));
 	}
 	// #if defined(__DARWIN__) || defined(__APPLE__)
@@ -363,14 +346,14 @@ unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path_p, uint8_t f
 	// 		}
 	// 	}
 	// #endif
-	if (lock_type != FileLockType::NO_LOCK) {
+	if (flags.Lock() != FileLockType::NO_LOCK) {
 		// set lock on file
 		// but only if it is not an input/output stream
 		auto file_type = GetFileTypeInternal(fd);
 		if (file_type != FileType::FILE_TYPE_FIFO && file_type != FileType::FILE_TYPE_SOCKET) {
 			struct flock fl;
 			memset(&fl, 0, sizeof fl);
-			fl.l_type = lock_type == FileLockType::READ_LOCK ? F_RDLCK : F_WRLCK;
+			fl.l_type = flags.Lock() == FileLockType::READ_LOCK ? F_RDLCK : F_WRLCK;
 			fl.l_whence = SEEK_SET;
 			fl.l_start = 0;
 			fl.l_len = 0;
@@ -387,7 +370,7 @@ unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path_p, uint8_t f
 					message = AdditionalProcessInfo(*this, fl.l_pid);
 				}
 
-				if (lock_type == FileLockType::WRITE_LOCK) {
+				if (flags.Lock() == FileLockType::WRITE_LOCK) {
 					// maybe we can get a read lock instead and tell this to the user.
 					fl.l_type = F_RDLCK;
 					rc = fcntl(fd, F_SETLK, &fl);
@@ -490,6 +473,16 @@ int64_t LocalFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_byte
 	return bytes_written;
 }
 
+bool LocalFileSystem::Trim(FileHandle &handle, idx_t offset_bytes, idx_t length_bytes) {
+#if defined(__linux__)
+	int fd = handle.Cast<UnixFileHandle>().fd;
+	int res = fallocate(fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, offset_bytes, length_bytes);
+	return res == 0;
+#else
+	return false;
+#endif
+}
+
 int64_t LocalFileSystem::GetFileSize(FileHandle &handle) {
 	int fd = handle.Cast<UnixFileHandle>().fd;
 	struct stat s;
@@ -521,7 +514,7 @@ void LocalFileSystem::Truncate(FileHandle &handle, int64_t new_size) {
 	}
 }
 
-bool LocalFileSystem::DirectoryExists(const string &directory) {
+bool LocalFileSystem::DirectoryExists(const string &directory, optional_ptr<FileOpener> opener) {
 	if (!directory.empty()) {
 		if (access(directory.c_str(), 0) == 0) {
 			struct stat status;
@@ -535,7 +528,7 @@ bool LocalFileSystem::DirectoryExists(const string &directory) {
 	return false;
 }
 
-void LocalFileSystem::CreateDirectory(const string &directory) {
+void LocalFileSystem::CreateDirectory(const string &directory, optional_ptr<FileOpener> opener) {
 	struct stat st;
 
 	if (stat(directory.c_str(), &st) != 0) {
@@ -589,11 +582,11 @@ int RemoveDirectoryRecursive(const char *path) {
 	return r;
 }
 
-void LocalFileSystem::RemoveDirectory(const string &directory) {
+void LocalFileSystem::RemoveDirectory(const string &directory, optional_ptr<FileOpener> opener) {
 	RemoveDirectoryRecursive(directory.c_str());
 }
 
-void LocalFileSystem::RemoveFile(const string &filename) {
+void LocalFileSystem::RemoveFile(const string &filename, optional_ptr<FileOpener> opener) {
 	if (std::remove(filename.c_str()) != 0) {
 		throw IOException("Could not remove file \"%s\": %s", {{"errno", std::to_string(errno)}}, filename,
 		                  strerror(errno));
@@ -602,7 +595,7 @@ void LocalFileSystem::RemoveFile(const string &filename) {
 
 bool LocalFileSystem::ListFiles(const string &directory, const std::function<void(const string &, bool)> &callback,
                                 FileOpener *opener) {
-	if (!DirectoryExists(directory)) {
+	if (!DirectoryExists(directory, opener)) {
 		return false;
 	}
 	DIR *dir = opendir(directory.c_str());
@@ -642,7 +635,7 @@ void LocalFileSystem::FileSync(FileHandle &handle) {
 	}
 }
 
-void LocalFileSystem::MoveFile(const string &source, const string &target) {
+void LocalFileSystem::MoveFile(const string &source, const string &target, optional_ptr<FileOpener> opener) {
 	//! FIXME: rename does not guarantee atomicity or overwriting target file if it exists
 	if (rename(source.c_str(), target.c_str()) != 0) {
 		throw IOException("Could not rename file!", {{"errno", std::to_string(errno)}});
@@ -766,20 +759,20 @@ bool LocalFileSystem::IsPrivateFile(const string &path_p, FileOpener *opener) {
 	return true;
 }
 
-unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path_p, uint8_t flags, FileLockType lock_type,
-                                                 FileCompressionType compression, FileOpener *opener) {
+unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path_p, FileOpenFlags flags,
+                                                 optional_ptr<FileOpener> opener) {
 	auto path = FileSystem::ExpandPath(path_p, opener);
-	if (compression != FileCompressionType::UNCOMPRESSED) {
+	if (flags.Compression() != FileCompressionType::UNCOMPRESSED) {
 		throw NotImplementedException("Unsupported compression type for default file system");
 	}
-	AssertValidFileFlags(flags);
+	flags.Verify();
 
 	DWORD desired_access;
 	DWORD share_mode;
 	DWORD creation_disposition = OPEN_EXISTING;
 	DWORD flags_and_attributes = FILE_ATTRIBUTE_NORMAL;
-	bool open_read = flags & FileFlags::FILE_FLAGS_READ;
-	bool open_write = flags & FileFlags::FILE_FLAGS_WRITE;
+	bool open_read = flags.OpenForReading();
+	bool open_write = flags.OpenForWriting();
 	if (open_read && open_write) {
 		desired_access = GENERIC_READ | GENERIC_WRITE;
 		share_mode = 0;
@@ -793,30 +786,33 @@ unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path_p, uint8_t f
 		throw InternalException("READ, WRITE or both should be specified when opening a file");
 	}
 	if (open_write) {
-		if (flags & FileFlags::FILE_FLAGS_FILE_CREATE) {
+		if (flags.CreateFileIfNotExists()) {
 			creation_disposition = OPEN_ALWAYS;
-		} else if (flags & FileFlags::FILE_FLAGS_FILE_CREATE_NEW) {
+		} else if (flags.OverwriteExistingFile()) {
 			creation_disposition = CREATE_ALWAYS;
 		}
 	}
-	if (flags & FileFlags::FILE_FLAGS_DIRECT_IO) {
+	if (flags.DirectIO()) {
 		flags_and_attributes |= FILE_FLAG_NO_BUFFERING;
 	}
 	auto unicode_path = WindowsUtil::UTF8ToUnicode(path.c_str());
 	HANDLE hFile = CreateFileW(unicode_path.c_str(), desired_access, share_mode, NULL, creation_disposition,
 	                           flags_and_attributes, NULL);
 	if (hFile == INVALID_HANDLE_VALUE) {
+		if (flags.ReturnNullIfNotExists() && GetLastError() == ERROR_FILE_NOT_FOUND) {
+			return nullptr;
+		}
 		auto error = LocalFileSystem::GetLastErrorAsString();
 
 		auto better_error = AdditionalLockInfo(unicode_path);
 		if (!better_error.empty()) {
 			throw IOException(better_error);
+		} else {
+			throw IOException("Cannot open file \"%s\": %s", path.c_str(), error);
 		}
-
-		throw IOException("Cannot open file \"%s\": %s", path.c_str(), error);
 	}
 	auto handle = make_uniq<WindowsFileHandle>(*this, path.c_str(), hFile);
-	if (flags & FileFlags::FILE_FLAGS_APPEND) {
+	if (flags.OpenForAppending()) {
 		auto file_size = GetFileSize(*handle);
 		SetFilePointer(*handle, file_size);
 	}
@@ -920,6 +916,11 @@ int64_t LocalFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_byte
 	return bytes_written;
 }
 
+bool LocalFileSystem::Trim(FileHandle &handle, idx_t offset_bytes, idx_t length_bytes) {
+	// TODO: Not yet implemented on windows.
+	return false;
+}
+
 int64_t LocalFileSystem::GetFileSize(FileHandle &handle) {
 	HANDLE hFile = handle.Cast<WindowsFileHandle>().fd;
 	LARGE_INTEGER result;
@@ -971,12 +972,12 @@ static DWORD WindowsGetFileAttributes(const string &filename) {
 	return GetFileAttributesW(unicode_path.c_str());
 }
 
-bool LocalFileSystem::DirectoryExists(const string &directory) {
+bool LocalFileSystem::DirectoryExists(const string &directory, optional_ptr<FileOpener> opener) {
 	DWORD attrs = WindowsGetFileAttributes(directory);
 	return (attrs != INVALID_FILE_ATTRIBUTES && (attrs & FILE_ATTRIBUTE_DIRECTORY));
 }
 
-void LocalFileSystem::CreateDirectory(const string &directory) {
+void LocalFileSystem::CreateDirectory(const string &directory, optional_ptr<FileOpener> opener) {
 	if (DirectoryExists(directory)) {
 		return;
 	}
@@ -1001,7 +1002,7 @@ static void DeleteDirectoryRecursive(FileSystem &fs, string directory) {
 	}
 }
 
-void LocalFileSystem::RemoveDirectory(const string &directory) {
+void LocalFileSystem::RemoveDirectory(const string &directory, optional_ptr<FileOpener> opener) {
 	if (FileExists(directory)) {
 		throw IOException("Attempting to delete directory \"%s\", but it is a file and not a directory!", directory);
 	}
@@ -1011,7 +1012,7 @@ void LocalFileSystem::RemoveDirectory(const string &directory) {
 	DeleteDirectoryRecursive(*this, directory.c_str());
 }
 
-void LocalFileSystem::RemoveFile(const string &filename) {
+void LocalFileSystem::RemoveFile(const string &filename, optional_ptr<FileOpener> opener) {
 	auto unicode_path = WindowsUtil::UTF8ToUnicode(filename.c_str());
 	if (!DeleteFileW(unicode_path.c_str())) {
 		auto error = LocalFileSystem::GetLastErrorAsString();
@@ -1055,7 +1056,7 @@ void LocalFileSystem::FileSync(FileHandle &handle) {
 	}
 }
 
-void LocalFileSystem::MoveFile(const string &source, const string &target) {
+void LocalFileSystem::MoveFile(const string &source, const string &target, optional_ptr<FileOpener> opener) {
 	auto source_unicode = WindowsUtil::UTF8ToUnicode(source.c_str());
 	auto target_unicode = WindowsUtil::UTF8ToUnicode(target.c_str());
 	if (!MoveFileW(source_unicode.c_str(), target_unicode.c_str())) {
@@ -1162,7 +1163,7 @@ static void GlobFilesInternal(FileSystem &fs, const string &path, const string &
 
 vector<string> LocalFileSystem::FetchFileWithoutGlob(const string &path, FileOpener *opener, bool absolute_path) {
 	vector<string> result;
-	if (FileExists(path) || IsPipe(path)) {
+	if (FileExists(path, opener) || IsPipe(path, opener)) {
 		result.push_back(path);
 	} else if (!absolute_path) {
 		Value value;
@@ -1171,7 +1172,7 @@ vector<string> LocalFileSystem::FetchFileWithoutGlob(const string &path, FileOpe
 			vector<std::string> search_paths = StringUtil::Split(search_paths_str, ',');
 			for (const auto &search_path : search_paths) {
 				auto joined_path = JoinPath(search_path, path);
-				if (FileExists(joined_path) || IsPipe(joined_path)) {
+				if (FileExists(joined_path, opener) || IsPipe(joined_path, opener)) {
 					result.push_back(joined_path);
 				}
 			}
@@ -1262,7 +1263,7 @@ vector<string> LocalFileSystem::Glob(const string &path, FileOpener *opener) {
 				if (is_last_chunk) {
 					for (auto &prev_directory : previous_directories) {
 						const string filename = JoinPath(prev_directory, splits[i]);
-						if (FileExists(filename) || DirectoryExists(filename)) {
+						if (FileExists(filename, opener) || DirectoryExists(filename, opener)) {
 							result.push_back(filename);
 						}
 					}

package/src/duckdb/src/common/multi_file_reader.cpp

@@ -172,7 +172,7 @@ MultiFileReaderBindData MultiFileReader::BindOptions(MultiFileReaderOptions &opt
 		}
 
 		for (auto &part : partitions) {
-			idx_t hive_partitioning_index = DConstants::INVALID_INDEX;
+			idx_t hive_partitioning_index;
 			auto lookup = std::find(names.begin(), names.end(), part.first);
 			if (lookup != names.end()) {
 				// hive partitioning column also exists in file - override
@@ -421,39 +421,59 @@ bool MultiFileReaderOptions::AutoDetectHivePartitioningInternal(const vector<str
 	}
 	return true;
 }
-void MultiFileReaderOptions::AutoDetectHiveTypesInternal(const string &file, ClientContext &context) {
+void MultiFileReaderOptions::AutoDetectHiveTypesInternal(const vector<string> &files, ClientContext &context) {
+	const LogicalType candidates[] = {LogicalType::DATE, LogicalType::TIMESTAMP, LogicalType::BIGINT};
+
 	auto &fs = FileSystem::GetFileSystem(context);
 
-	std::map<string, string> partitions;
-	auto splits = StringUtil::Split(file, fs.PathSeparator(file));
-	if (splits.size() < 2) {
-		return;
-	}
-	for (auto it = splits.begin(); it != std::prev(splits.end()); it++) {
-		auto part = StringUtil::Split(*it, "=");
-		if (part.size() == 2) {
-			partitions[part.front()] = part.back();
+	unordered_map<string, LogicalType> detected_types;
+	for (auto &file : files) {
+		unordered_map<string, string> partitions;
+		auto splits = StringUtil::Split(file, fs.PathSeparator(file));
+		if (splits.size() < 2) {
+			return;
 		}
-	}
-	if (partitions.empty()) {
-		return;
-	}
-
-	const LogicalType candidates[] = {LogicalType::DATE, LogicalType::TIMESTAMP, LogicalType::BIGINT};
-	for (auto &part : partitions) {
-		const string &name = part.first;
-		if (hive_types_schema.find(name) != hive_types_schema.end()) {
-			continue;
+		for (auto it = splits.begin(); it != std::prev(splits.end()); it++) {
+			auto part = StringUtil::Split(*it, "=");
+			if (part.size() == 2) {
+				partitions[part.front()] = part.back();
+			}
 		}
-		Value value(part.second);
-		for (auto &candidate : candidates) {
-			const bool success = value.TryCastAs(context, candidate, true);
-			if (success) {
-				hive_types_schema[name] = candidate;
-				break;
+		if (partitions.empty()) {
+			return;
+		}
+
+		for (auto &part : partitions) {
+			const string &name = part.first;
+			if (hive_types_schema.find(name) != hive_types_schema.end()) {
+				// type was explicitly provided by the user
+				continue;
+			}
+			LogicalType detected_type = LogicalType::VARCHAR;
+			Value value(part.second);
+			for (auto &candidate : candidates) {
+				const bool success = value.TryCastAs(context, candidate, true);
+				if (success) {
+					detected_type = candidate;
+					break;
+				}
+			}
+			auto entry = detected_types.find(name);
+			if (entry == detected_types.end()) {
+				// type was not yet detected - insert it
+				detected_types.insert(make_pair(name, std::move(detected_type)));
+			} else {
+				// type was already detected - check if the type matches
+				// if not promote to VARCHAR
+				if (entry->second != detected_type) {
+					entry->second = LogicalType::VARCHAR;
+				}
 			}
 		}
 	}
+	for (auto &entry : detected_types) {
+		hive_types_schema.insert(make_pair(entry.first, std::move(entry.second)));
+	}
 }
 void MultiFileReaderOptions::AutoDetectHivePartitioning(const vector<string> &files, ClientContext &context) {
 	D_ASSERT(!files.empty());
@@ -471,7 +491,7 @@ void MultiFileReaderOptions::AutoDetectHivePartitioning(const vector<string> &fi
 		hive_partitioning = AutoDetectHivePartitioningInternal(files, context);
 	}
 	if (hive_partitioning && hive_types_autocast) {
-		AutoDetectHiveTypesInternal(files.front(), context);
+		AutoDetectHiveTypesInternal(files, context);
 	}
 }
 void MultiFileReaderOptions::VerifyHiveTypesArePartitions(const std::map<string, string> &partitions) const {

package/src/duckdb/src/common/row_operations/row_matcher.cpp

@@ -120,13 +120,13 @@ static idx_t SelectComparison(Vector &, Vector &, const SelectionVector &, idx_t
 template <>
 idx_t SelectComparison<Equals>(Vector &left, Vector &right, const SelectionVector &sel, idx_t count,
                                SelectionVector *true_sel, SelectionVector *false_sel) {
-	return VectorOperations::NestedEquals(left, right, sel, count, true_sel, false_sel);
+	return VectorOperations::NestedEquals(left, right, &sel, count, true_sel, false_sel);
 }
 
 template <>
 idx_t SelectComparison<NotEquals>(Vector &left, Vector &right, const SelectionVector &sel, idx_t count,
                                   SelectionVector *true_sel, SelectionVector *false_sel) {
-	return VectorOperations::NestedNotEquals(left, right, sel, count, true_sel, false_sel);
+	return VectorOperations::NestedNotEquals(left, right, &sel, count, true_sel, false_sel);
 }
 
 template <>

package/src/duckdb/src/common/serializer/buffered_file_reader.cpp

@@ -10,7 +10,13 @@ namespace duckdb {
 BufferedFileReader::BufferedFileReader(FileSystem &fs, const char *path, FileLockType lock_type,
                                        optional_ptr<FileOpener> opener)
     : fs(fs), data(make_unsafe_uniq_array<data_t>(FILE_BUFFER_SIZE)), offset(0), read_data(0), total_read(0) {
-	handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_READ, lock_type, FileSystem::DEFAULT_COMPRESSION, opener.get());
+	handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_READ | lock_type, opener.get());
+	file_size = fs.GetFileSize(*handle);
+}
+
+BufferedFileReader::BufferedFileReader(FileSystem &fs, unique_ptr<FileHandle> handle_p)
+    : fs(fs), data(make_unsafe_uniq_array<data_t>(FILE_BUFFER_SIZE)), offset(0), read_data(0),
+      handle(std::move(handle_p)), total_read(0) {
 	file_size = fs.GetFileSize(*handle);
 }
 
@@ -51,6 +57,12 @@ void BufferedFileReader::Seek(uint64_t location) {
 	read_data = offset = 0;
 }
 
+void BufferedFileReader::Reset() {
+	handle->Reset();
+	total_read = 0;
+	read_data = offset = 0;
+}
+
 uint64_t BufferedFileReader::CurrentOffset() {
 	return total_read + offset;
 }

package/src/duckdb/src/common/serializer/buffered_file_writer.cpp

@@ -1,16 +1,17 @@
 #include "duckdb/common/serializer/buffered_file_writer.hpp"
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/algorithm.hpp"
+#include "duckdb/common/typedefs.hpp"
 #include <cstring>
 
 namespace duckdb {
 
 // Remove this when we switch C++17: https://stackoverflow.com/a/53350948
-constexpr uint8_t BufferedFileWriter::DEFAULT_OPEN_FLAGS;
+constexpr FileOpenFlags BufferedFileWriter::DEFAULT_OPEN_FLAGS;
 
-BufferedFileWriter::BufferedFileWriter(FileSystem &fs, const string &path_p, uint8_t open_flags)
+BufferedFileWriter::BufferedFileWriter(FileSystem &fs, const string &path_p, FileOpenFlags open_flags)
     : fs(fs), path(path_p), data(make_unsafe_uniq_array<data_t>(FILE_BUFFER_SIZE)), offset(0), total_written(0) {
-	handle = fs.OpenFile(path, open_flags, FileLockType::WRITE_LOCK);
+	handle = fs.OpenFile(path, open_flags | FileLockType::WRITE_LOCK);
 }
 
 int64_t BufferedFileWriter::GetFileSize() {
@@ -22,16 +23,34 @@ idx_t BufferedFileWriter::GetTotalWritten() {
 }
 
 void BufferedFileWriter::WriteData(const_data_ptr_t buffer, idx_t write_size) {
-	// first copy anything we can from the buffer
-	const_data_ptr_t end_ptr = buffer + write_size;
-	while (buffer < end_ptr) {
-		idx_t to_write = MinValue<idx_t>((end_ptr - buffer), FILE_BUFFER_SIZE - offset);
-		D_ASSERT(to_write > 0);
-		memcpy(data.get() + offset, buffer, to_write);
-		offset += to_write;
-		buffer += to_write;
-		if (offset == FILE_BUFFER_SIZE) {
-			Flush();
+	if (write_size >= (2ULL * FILE_BUFFER_SIZE - offset)) {
+		idx_t to_copy = 0;
+		// Check before performing direct IO if there is some data in the current internal buffer.
+		// If so, then fill the buffer (to avoid to small write operation), flush it and then write
+		// all the remain data directly.
+		// This is to avoid to split a large buffer into N*FILE_BUFFER_SIZE buffers
+		if (offset != 0) {
+			// Some data are still present in the buffer let write them before
+			to_copy = FILE_BUFFER_SIZE - offset;
+			memcpy(data.get() + offset, buffer, to_copy);
+			offset += to_copy;
+			Flush(); // Flush buffer before writing every things else
+		}
+		idx_t remaining_to_write = write_size - to_copy;
+		fs.Write(*handle, const_cast<data_ptr_t>(buffer + to_copy), remaining_to_write); // NOLINT: wrong API in Write
+		total_written += remaining_to_write;
+	} else {
+		// first copy anything we can from the buffer
+		const_data_ptr_t end_ptr = buffer + write_size;
+		while (buffer < end_ptr) {
+			idx_t to_write = MinValue<idx_t>((end_ptr - buffer), FILE_BUFFER_SIZE - offset);
+			D_ASSERT(to_write > 0);
+			memcpy(data.get() + offset, buffer, to_write);
+			offset += to_write;
+			buffer += to_write;
+			if (offset == FILE_BUFFER_SIZE) {
+				Flush();
+			}
 		}
 	}
 }

package/src/duckdb/src/common/string_util.cpp

@@ -173,7 +173,6 @@ string StringUtil::Join(const set<string> &input, const string &separator) {
 
 string StringUtil::BytesToHumanReadableString(idx_t bytes, idx_t multiplier) {
 	D_ASSERT(multiplier == 1000 || multiplier == 1024);
-	string db_size;
 	idx_t array[6] = {};
 	const char *unit[2][6] = {{"bytes", "KiB", "MiB", "GiB", "TiB", "PiB"}, {"bytes", "kB", "MB", "GB", "TB", "PB"}};
 
@@ -230,7 +229,7 @@ bool StringUtil::CIEquals(const string &l1, const string &l2) {
 	if (l1.size() != l2.size()) {
 		return false;
 	}
-	const auto charmap = LowerFun::ascii_to_lower_map;
+	const auto charmap = LowerFun::ASCII_TO_LOWER_MAP;
 	for (idx_t c = 0; c < l1.size(); c++) {
 		if (charmap[(uint8_t)l1[c]] != charmap[(uint8_t)l2[c]]) {
 			return false;
@@ -240,7 +239,7 @@ bool StringUtil::CIEquals(const string &l1, const string &l2) {
 }
 
 bool StringUtil::CILessThan(const string &s1, const string &s2) {
-	const auto charmap = UpperFun::ascii_to_upper_map;
+	const auto charmap = UpperFun::ASCII_TO_UPPER_MAP;
 
 	unsigned char u1, u2;