duckdb 0.8.2-dev33.0 → 0.8.2-dev3300.0

package/src/duckdb/src/common/hive_partitioning.cpp

@@ -6,6 +6,7 @@
 #include "duckdb/planner/expression/bound_constant_expression.hpp"
 #include "duckdb/planner/expression/bound_reference_expression.hpp"
 #include "duckdb/planner/expression_iterator.hpp"
+#include "duckdb/planner/operator/logical_get.hpp"
 #include "duckdb/planner/table_filter.hpp"
 #include "re2/re2.h"
 
@@ -86,12 +87,15 @@ std::map<string, string> HivePartitioning::Parse(const string &filename) {
 //		 currently, only expressions that cannot be evaluated during pushdown are removed.
 void HivePartitioning::ApplyFiltersToFileList(ClientContext &context, vector<string> &files,
                                               vector<unique_ptr<Expression>> &filters,
-                                              unordered_map<string, column_t> &column_map, idx_t table_index,
+                                              unordered_map<string, column_t> &column_map, LogicalGet &get,
                                               bool hive_enabled, bool filename_enabled) {
+
 	vector<string> pruned_files;
 	vector<bool> have_preserved_filter(filters.size(), false);
 	vector<unique_ptr<Expression>> pruned_filters;
+	unordered_set<idx_t> filters_applied_to_files;
 	duckdb_re2::RE2 regex(REGEX_STRING);
+	auto table_index = get.table_index;
 
 	if ((!filename_enabled && !hive_enabled) || filters.empty()) {
 		return;
@@ -121,11 +125,11 @@ void HivePartitioning::ApplyFiltersToFileList(ClientContext &context, vector<str
 			} else if (!result_value.GetValue<bool>()) {
 				// filter evaluates to false
 				should_prune_file = true;
-			}
-
-			// Use filter combiner to determine that this filter makes
-			if (!should_prune_file && combiner.AddFilter(std::move(filter_copy)) == FilterResult::UNSATISFIABLE) {
-				should_prune_file = true;
+				// convert the filter to a table filter.
+				if (filters_applied_to_files.find(j) == filters_applied_to_files.end()) {
+					get.extra_info.file_filters += filter->ToString();
+					filters_applied_to_files.insert(j);
+				}
 			}
 		}
 

package/src/duckdb/src/common/http_state.cpp

@@ -0,0 +1,78 @@
+#include "duckdb/common/http_state.hpp"
+
+namespace duckdb {
+
+CachedFileHandle::CachedFileHandle(shared_ptr<CachedFile> &file_p) {
+	// If the file was not yet initialized, we need to grab a lock.
+	if (!file_p->initialized) {
+		lock = make_uniq<lock_guard<mutex>>(file_p->lock);
+	}
+	file = file_p;
+}
+
+void CachedFileHandle::SetInitialized() {
+	if (file->initialized) {
+		throw InternalException("Cannot set initialized on cached file that was already initialized");
+	}
+	if (!lock) {
+		throw InternalException("Cannot set initialized on cached file without lock");
+	}
+	file->initialized = true;
+	lock = nullptr;
+}
+
+void CachedFileHandle::AllocateBuffer(idx_t size) {
+	if (file->initialized) {
+		throw InternalException("Cannot allocate a buffer for a cached file that was already initialized");
+	}
+	file->data = std::shared_ptr<char>(new char[size], std::default_delete<char[]>());
+	file->capacity = size;
+}
+
+void CachedFileHandle::GrowBuffer(idx_t new_capacity, idx_t bytes_to_copy) {
+	// copy shared ptr to old data
+	auto old_data = file->data;
+	// allocate new buffer that can hold the new capacity
+	AllocateBuffer(new_capacity);
+	// copy the old data
+	Write(old_data.get(), bytes_to_copy);
+}
+
+void CachedFileHandle::Write(const char *buffer, idx_t length, idx_t offset) {
+	//! Only write to non-initialized files with a lock;
+	D_ASSERT(!file->initialized && lock);
+	memcpy(file->data.get() + offset, buffer, length);
+}
+
+void HTTPState::Reset() {
+	// Reset Counters
+	head_count = 0;
+	get_count = 0;
+	put_count = 0;
+	post_count = 0;
+	total_bytes_received = 0;
+	total_bytes_sent = 0;
+
+	// Reset cached files
+	cached_files.clear();
+}
+
+shared_ptr<HTTPState> HTTPState::TryGetState(FileOpener *opener) {
+	auto client_context = FileOpener::TryGetClientContext(opener);
+	if (client_context) {
+		return client_context->client_data->http_state;
+	}
+	return nullptr;
+}
+
+//! Get cache entry, create if not exists
+shared_ptr<CachedFile> &HTTPState::GetCachedFile(const string &path) {
+	lock_guard<mutex> lock(cached_files_mutex);
+	auto &cache_entry_ref = cached_files[path];
+	if (!cache_entry_ref) {
+		cache_entry_ref = make_shared<CachedFile>();
+	}
+	return cache_entry_ref;
+}
+
+} // namespace duckdb

package/src/duckdb/src/common/local_file_system.cpp

@@ -261,13 +261,19 @@ idx_t LocalFileSystem::GetFilePointer(FileHandle &handle) {
 
 void LocalFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) {
 	int fd = handle.Cast<UnixFileHandle>().fd;
-	int64_t bytes_read = pread(fd, buffer, nr_bytes, location);
-	if (bytes_read == -1) {
-		throw IOException("Could not read from file \"%s\": %s", handle.path, strerror(errno));
-	}
-	if (bytes_read != nr_bytes) {
-		throw IOException("Could not read all bytes from file \"%s\": wanted=%lld read=%lld", handle.path, nr_bytes,
-		                  bytes_read);
+	auto read_buffer = char_ptr_cast(buffer);
+	while (nr_bytes > 0) {
+		int64_t bytes_read = pread(fd, read_buffer, nr_bytes, location);
+		if (bytes_read == -1) {
+			throw IOException("Could not read from file \"%s\": %s", handle.path, strerror(errno));
+		}
+		if (bytes_read == 0) {
+			throw IOException(
+			    "Could not read enough bytes from file \"%s\": attempted to read %llu bytes from location %llu",
+			    handle.path, nr_bytes, location);
+		}
+		read_buffer += bytes_read;
+		nr_bytes -= bytes_read;
 	}
 }
 
@@ -282,13 +288,15 @@ int64_t LocalFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes
 
 void LocalFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) {
 	int fd = handle.Cast<UnixFileHandle>().fd;
-	int64_t bytes_written = pwrite(fd, buffer, nr_bytes, location);
-	if (bytes_written == -1) {
-		throw IOException("Could not write file \"%s\": %s", handle.path, strerror(errno));
-	}
-	if (bytes_written != nr_bytes) {
-		throw IOException("Could not write all bytes to file \"%s\": wanted=%lld wrote=%lld", handle.path, nr_bytes,
-		                  bytes_written);
+	auto write_buffer = char_ptr_cast(buffer);
+	while (nr_bytes > 0) {
+		int64_t bytes_written = pwrite(fd, write_buffer, nr_bytes, location);
+		if (bytes_written < 0) {
+			throw IOException("Could not write file \"%s\": %s", handle.path, strerror(errno));
+		}
+		D_ASSERT(bytes_written >= 0 && bytes_written);
+		write_buffer += bytes_written;
+		nr_bytes -= bytes_written;
 	}
 }
 
@@ -375,7 +383,7 @@ int RemoveDirectoryRecursive(const char *path) {
 				continue;
 			}
 			len = path_len + (idx_t)strlen(p->d_name) + 2;
-			buf = new char[len];
+			buf = new (std::nothrow) char[len];
 			if (buf) {
 				struct stat statbuf;
 				snprintf(buf, len, "%s/%s", path, p->d_name);
@@ -559,7 +567,7 @@ unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path_p, uint8_t f
 }
 
 void LocalFileSystem::SetFilePointer(FileHandle &handle, idx_t location) {
-	auto &whandle = (WindowsFileHandle &)handle;
+	auto &whandle = handle.Cast<WindowsFileHandle>();
 	whandle.position = location;
 	LARGE_INTEGER wlocation;
 	wlocation.QuadPart = location;
@@ -567,7 +575,7 @@ void LocalFileSystem::SetFilePointer(FileHandle &handle, idx_t location) {
 }
 
 idx_t LocalFileSystem::GetFilePointer(FileHandle &handle) {
-	return ((WindowsFileHandle &)handle).position;
+	return handle.Cast<WindowsFileHandle>().position;
 }
 
 static DWORD FSInternalRead(FileHandle &handle, HANDLE hFile, void *buffer, int64_t nr_bytes, idx_t location) {
@@ -597,8 +605,8 @@ void LocalFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes, i
 }
 
 int64_t LocalFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes) {
-	HANDLE hFile = ((WindowsFileHandle &)handle).fd;
-	auto &pos = ((WindowsFileHandle &)handle).position;
+	HANDLE hFile = handle.Cast<WindowsFileHandle>().fd;
+	auto &pos = handle.Cast<WindowsFileHandle>().position;
 	auto n = std::min<idx_t>(std::max<idx_t>(GetFileSize(handle), pos) - pos, nr_bytes);
 	auto bytes_read = FSInternalRead(handle, hFile, buffer, n, pos);
 	pos += bytes_read;
@@ -622,7 +630,7 @@ static DWORD FSInternalWrite(FileHandle &handle, HANDLE hFile, void *buffer, int
 }
 
 void LocalFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) {
-	HANDLE hFile = ((WindowsFileHandle &)handle).fd;
+	HANDLE hFile = handle.Cast<WindowsFileHandle>().fd;
 	auto bytes_written = FSInternalWrite(handle, hFile, buffer, nr_bytes, location);
 	if (bytes_written != nr_bytes) {
 		throw IOException("Could not write all bytes from file \"%s\": wanted=%lld wrote=%lld", handle.path, nr_bytes,
@@ -631,15 +639,15 @@ void LocalFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_bytes,
 }
 
 int64_t LocalFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_bytes) {
-	HANDLE hFile = ((WindowsFileHandle &)handle).fd;
-	auto &pos = ((WindowsFileHandle &)handle).position;
+	HANDLE hFile = handle.Cast<WindowsFileHandle>().fd;
+	auto &pos = handle.Cast<WindowsFileHandle>().position;
 	auto bytes_written = FSInternalWrite(handle, hFile, buffer, nr_bytes, pos);
 	pos += bytes_written;
 	return bytes_written;
 }
 
 int64_t LocalFileSystem::GetFileSize(FileHandle &handle) {
-	HANDLE hFile = ((WindowsFileHandle &)handle).fd;
+	HANDLE hFile = handle.Cast<WindowsFileHandle>().fd;
 	LARGE_INTEGER result;
 	if (!GetFileSizeEx(hFile, &result)) {
 		return -1;
@@ -648,7 +656,7 @@ int64_t LocalFileSystem::GetFileSize(FileHandle &handle) {
 }
 
 time_t LocalFileSystem::GetLastModifiedTime(FileHandle &handle) {
-	HANDLE hFile = ((WindowsFileHandle &)handle).fd;
+	HANDLE hFile = handle.Cast<WindowsFileHandle>().fd;
 
 	// https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-getfiletime
 	FILETIME last_write;
@@ -674,7 +682,7 @@ time_t LocalFileSystem::GetLastModifiedTime(FileHandle &handle) {
 }
 
 void LocalFileSystem::Truncate(FileHandle &handle, int64_t new_size) {
-	HANDLE hFile = ((WindowsFileHandle &)handle).fd;
+	HANDLE hFile = handle.Cast<WindowsFileHandle>().fd;
 	// seek to the location
 	SetFilePointer(handle, new_size);
 	// now set the end of file position
@@ -700,7 +708,7 @@ void LocalFileSystem::CreateDirectory(const string &directory) {
 	}
 	auto unicode_path = WindowsUtil::UTF8ToUnicode(directory.c_str());
 	if (directory.empty() || !CreateDirectoryW(unicode_path.c_str(), NULL) || !DirectoryExists(directory)) {
-		throw IOException("Could not create directory!");
+		throw IOException("Could not create directory: \'%s\'", directory.c_str());
 	}
 }
 
@@ -767,7 +775,7 @@ bool LocalFileSystem::ListFiles(const string &directory, const std::function<voi
 }
 
 void LocalFileSystem::FileSync(FileHandle &handle) {
-	HANDLE hFile = ((WindowsFileHandle &)handle).fd;
+	HANDLE hFile = handle.Cast<WindowsFileHandle>().fd;
 	if (FlushFileBuffers(hFile) == 0) {
 		throw IOException("Could not flush file handle to disk!");
 	}
@@ -782,7 +790,7 @@ void LocalFileSystem::MoveFile(const string &source, const string &target) {
 }
 
 FileType LocalFileSystem::GetFileType(FileHandle &handle) {
-	auto path = ((WindowsFileHandle &)handle).path;
+	auto path = handle.Cast<WindowsFileHandle>().path;
 	// pipes in windows are just files in '\\.\pipe\' folder
 	if (strncmp(path.c_str(), PIPE_PREFIX, strlen(PIPE_PREFIX)) == 0) {
 		return FileType::FILE_TYPE_FIFO;

package/src/duckdb/src/common/multi_file_reader.cpp

@@ -6,6 +6,7 @@
 #include "duckdb/common/exception.hpp"
 #include "duckdb/function/function_set.hpp"
 #include "duckdb/common/hive_partitioning.hpp"
+#include "duckdb/common/types.hpp"
 
 namespace duckdb {
 
@@ -13,6 +14,8 @@ void MultiFileReader::AddParameters(TableFunction &table_function) {
 	table_function.named_parameters["filename"] = LogicalType::BOOLEAN;
 	table_function.named_parameters["hive_partitioning"] = LogicalType::BOOLEAN;
 	table_function.named_parameters["union_by_name"] = LogicalType::BOOLEAN;
+	table_function.named_parameters["hive_types"] = LogicalType::ANY;
+	table_function.named_parameters["hive_types_autocast"] = LogicalType::BOOLEAN;
 }
 
 vector<string> MultiFileReader::GetFileList(ClientContext &context, const Value &input, const string &name,
@@ -49,7 +52,8 @@ vector<string> MultiFileReader::GetFileList(ClientContext &context, const Value
 	return files;
 }
 
-bool MultiFileReader::ParseOption(const string &key, const Value &val, MultiFileReaderOptions &options) {
+bool MultiFileReader::ParseOption(const string &key, const Value &val, MultiFileReaderOptions &options,
+                                  ClientContext &context) {
 	auto loption = StringUtil::Lower(key);
 	if (loption == "filename") {
 		options.filename = BooleanValue::Get(val);
@@ -58,6 +62,28 @@ bool MultiFileReader::ParseOption(const string &key, const Value &val, MultiFile
 		options.auto_detect_hive_partitioning = false;
 	} else if (loption == "union_by_name") {
 		options.union_by_name = BooleanValue::Get(val);
+	} else if (loption == "hive_types_autocast" || loption == "hive_type_autocast") {
+		options.hive_types_autocast = BooleanValue::Get(val);
+	} else if (loption == "hive_types" || loption == "hive_type") {
+		if (val.type().id() != LogicalTypeId::STRUCT) {
+			throw InvalidInputException(
+			    "'hive_types' only accepts a STRUCT('name':VARCHAR, ...), but '%s' was provided",
+			    val.type().ToString());
+		}
+		// verify that that all the children of the struct value are VARCHAR
+		auto &children = StructValue::GetChildren(val);
+		for (idx_t i = 0; i < children.size(); i++) {
+			const Value &child = children[i];
+			if (child.type().id() != LogicalType::VARCHAR) {
+				throw InvalidInputException("hive_types: '%s' must be a VARCHAR, instead: '%s' was provided",
+				                            StructType::GetChildName(val.type(), i), child.type().ToString());
+			}
+			// for every child of the struct, get the logical type
+			LogicalType transformed_type = TransformStringToLogicalType(child.ToString(), context);
+			const string &name = StructType::GetChildName(val.type(), i);
+			options.hive_types_schema[name] = transformed_type;
+		}
+		D_ASSERT(!options.hive_types_schema.empty());
 	} else {
 		return false;
 	}
@@ -80,8 +106,9 @@ bool MultiFileReader::ComplexFilterPushdown(ClientContext &context, vector<strin
 	}
 
 	auto start_files = files.size();
-	HivePartitioning::ApplyFiltersToFileList(context, files, filters, column_map, get.table_index,
-	                                         options.hive_partitioning, options.filename);
+	HivePartitioning::ApplyFiltersToFileList(context, files, filters, column_map, get, options.hive_partitioning,
+	                                         options.filename);
+
 	if (files.size() != start_files) {
 		// we have pruned files
 		return true;
@@ -111,28 +138,28 @@ MultiFileReaderBindData MultiFileReader::BindOptions(MultiFileReaderOptions &opt
 			auto file_partitions = HivePartitioning::Parse(f);
 			for (auto &part_info : partitions) {
 				if (file_partitions.find(part_info.first) == file_partitions.end()) {
+					string error = "Hive partition mismatch between file \"%s\" and \"%s\": key \"%s\" not found";
 					if (options.auto_detect_hive_partitioning == true) {
-						throw BinderException(
-						    "Hive partitioning was enabled automatically, but an error was encountered: Hive partition "
-						    "mismatch between file \"%s\" and \"%s\": key \"%s\" not found\n\nTo switch off hive "
-						    "partition, set: HIVE_PARTITIONING=0",
-						    files[0], f, part_info.first);
+						throw InternalException(error + "(hive partitioning was autodetected)", files[0], f,
+						                        part_info.first);
 					}
-					throw BinderException(
-					    "Hive partition mismatch between file \"%s\" and \"%s\": key \"%s\" not found", files[0], f,
-					    part_info.first);
+					throw BinderException(error.c_str(), files[0], f, part_info.first);
 				}
 			}
 			if (partitions.size() != file_partitions.size()) {
+				string error_msg = "Hive partition mismatch between file \"%s\" and \"%s\"";
 				if (options.auto_detect_hive_partitioning == true) {
-					throw BinderException("Hive partitioning was enabled automatically, but an error was encountered: "
-					                      "Hive partition mismatch between file \"%s\" and \"%s\"\n\nTo switch off "
-					                      "hive partition, set: HIVE_PARTITIONING=0",
-					                      files[0], f);
+					throw InternalException(error_msg + "(hive partitioning was autodetected)", files[0], f);
 				}
-				throw BinderException("Hive partition mismatch between file \"%s\" and \"%s\"", files[0], f);
+				throw BinderException(error_msg.c_str(), files[0], f);
 			}
 		}
+
+		if (!options.hive_types_schema.empty()) {
+			// verify that all hive_types are existing partitions
+			options.VerifyHiveTypesArePartitions(partitions);
+		}
+
 		for (auto &part : partitions) {
 			idx_t hive_partitioning_index = DConstants::INVALID_INDEX;
 			auto lookup = std::find(names.begin(), names.end(), part.first);
@@ -140,11 +167,11 @@ MultiFileReaderBindData MultiFileReader::BindOptions(MultiFileReaderOptions &opt
 				// hive partitioning column also exists in file - override
 				auto idx = lookup - names.begin();
 				hive_partitioning_index = idx;
-				return_types[idx] = LogicalType::VARCHAR;
+				return_types[idx] = options.GetHiveLogicalType(part.first);
 			} else {
 				// hive partitioning column does not exist in file - add a new column containing the key
 				hive_partitioning_index = names.size();
-				return_types.emplace_back(LogicalType::VARCHAR);
+				return_types.emplace_back(options.GetHiveLogicalType(part.first));
 				names.emplace_back(part.first);
 			}
 			bind_data.hive_partitioning_indexes.emplace_back(part.first, hive_partitioning_index);
@@ -156,7 +183,9 @@ MultiFileReaderBindData MultiFileReader::BindOptions(MultiFileReaderOptions &opt
 void MultiFileReader::FinalizeBind(const MultiFileReaderOptions &file_options, const MultiFileReaderBindData &options,
                                    const string &filename, const vector<string> &local_names,
                                    const vector<LogicalType> &global_types, const vector<string> &global_names,
-                                   const vector<column_t> &global_column_ids, MultiFileReaderData &reader_data) {
+                                   const vector<column_t> &global_column_ids, MultiFileReaderData &reader_data,
+                                   ClientContext &context) {
+
 	// create a map of name -> column index
 	case_insensitive_map_t<idx_t> name_map;
 	if (file_options.union_by_name) {
@@ -183,7 +212,8 @@ void MultiFileReader::FinalizeBind(const MultiFileReaderOptions &file_options, c
 			bool found_partition = false;
 			for (auto &entry : options.hive_partitioning_indexes) {
 				if (column_id == entry.index) {
-					reader_data.constant_map.emplace_back(i, Value(partitions[entry.value]));
+					Value value = file_options.GetHivePartitionValue(partitions[entry.value], entry.value, context);
+					reader_data.constant_map.emplace_back(i, value);
 					found_partition = true;
 					break;
 				}
@@ -313,7 +343,16 @@ void MultiFileReaderOptions::Serialize(Serializer &serializer) const {
 	FieldWriter writer(serializer);
 	writer.WriteField<bool>(filename);
 	writer.WriteField<bool>(hive_partitioning);
+	writer.WriteField<bool>(auto_detect_hive_partitioning);
 	writer.WriteField<bool>(union_by_name);
+	writer.WriteField<bool>(hive_types_autocast);
+	// serialize hive_types_schema
+	const uint32_t schema_size = hive_types_schema.size();
+	writer.WriteField<uint32_t>(schema_size);
+	for (auto &hive_type : hive_types_schema) {
+		writer.WriteString(hive_type.first);
+		writer.WriteString(hive_type.second.ToString());
+	}
 	writer.Finalize();
 }
 
@@ -322,7 +361,16 @@ MultiFileReaderOptions MultiFileReaderOptions::Deserialize(Deserializer &source)
 	FieldReader reader(source);
 	result.filename = reader.ReadRequired<bool>();
 	result.hive_partitioning = reader.ReadRequired<bool>();
+	result.auto_detect_hive_partitioning = reader.ReadRequired<bool>();
 	result.union_by_name = reader.ReadRequired<bool>();
+	result.hive_types_autocast = reader.ReadRequired<bool>();
+	// deserialize hive_types_schema
+	const uint32_t schema_size = reader.ReadRequired<uint32_t>();
+	for (idx_t i = 0; i < schema_size; i++) {
+		const string name = reader.ReadRequired<string>();
+		const LogicalType type = TransformStringToLogicalType(reader.ReadRequired<string>());
+		result.hive_types_schema[name] = type;
+	}
 	reader.Finalize();
 	return result;
 }
@@ -365,7 +413,9 @@ HivePartitioningIndex HivePartitioningIndex::Deserialize(Deserializer &source) {
 void MultiFileReaderOptions::AddBatchInfo(BindInfo &bind_info) const {
 	bind_info.InsertOption("filename", Value::BOOLEAN(filename));
 	bind_info.InsertOption("hive_partitioning", Value::BOOLEAN(hive_partitioning));
+	bind_info.InsertOption("auto_detect_hive_partitioning", Value::BOOLEAN(auto_detect_hive_partitioning));
 	bind_info.InsertOption("union_by_name", Value::BOOLEAN(union_by_name));
+	bind_info.InsertOption("hive_types_autocast", Value::BOOLEAN(hive_types_autocast));
 }
 
 void UnionByName::CombineUnionTypes(const vector<string> &col_names, const vector<LogicalType> &sql_types,
@@ -390,4 +440,127 @@ void UnionByName::CombineUnionTypes(const vector<string> &col_names, const vecto
 	}
 }
 
+bool MultiFileReaderOptions::AutoDetectHivePartitioningInternal(const vector<string> &files, ClientContext &context) {
+	std::unordered_set<string> partitions;
+	auto &fs = FileSystem::GetFileSystem(context);
+
+	auto splits_first_file = StringUtil::Split(files.front(), fs.PathSeparator(files.front()));
+	if (splits_first_file.size() < 2) {
+		return false;
+	}
+	for (auto it = splits_first_file.begin(); it != splits_first_file.end(); it++) {
+		auto partition = StringUtil::Split(*it, "=");
+		if (partition.size() == 2) {
+			partitions.insert(partition.front());
+		}
+	}
+	if (partitions.empty()) {
+		return false;
+	}
+	for (auto &file : files) {
+		auto splits = StringUtil::Split(file, fs.PathSeparator(file));
+		if (splits.size() != splits_first_file.size()) {
+			return false;
+		}
+		for (auto it = splits.begin(); it != std::prev(splits.end()); it++) {
+			auto part = StringUtil::Split(*it, "=");
+			if (part.size() != 2) {
+				continue;
+			}
+			if (partitions.find(part.front()) == partitions.end()) {
+				return false;
+			}
+		}
+	}
+	return true;
+}
+void MultiFileReaderOptions::AutoDetectHiveTypesInternal(const string &file, ClientContext &context) {
+	auto &fs = FileSystem::GetFileSystem(context);
+
+	std::map<string, string> partitions;
+	auto splits = StringUtil::Split(file, fs.PathSeparator(file));
+	if (splits.size() < 2) {
+		return;
+	}
+	for (auto it = splits.begin(); it != std::prev(splits.end()); it++) {
+		auto part = StringUtil::Split(*it, "=");
+		if (part.size() == 2) {
+			partitions[part.front()] = part.back();
+		}
+	}
+	if (partitions.empty()) {
+		return;
+	}
+
+	const LogicalType candidates[] = {LogicalType::DATE, LogicalType::TIMESTAMP, LogicalType::BIGINT};
+	for (auto &part : partitions) {
+		const string &name = part.first;
+		if (hive_types_schema.find(name) != hive_types_schema.end()) {
+			continue;
+		}
+		Value value(part.second);
+		for (auto &candidate : candidates) {
+			const bool success = value.TryCastAs(context, candidate);
+			if (success) {
+				hive_types_schema[name] = candidate;
+				break;
+			}
+		}
+	}
+}
+void MultiFileReaderOptions::AutoDetectHivePartitioning(const vector<string> &files, ClientContext &context) {
+	D_ASSERT(!files.empty());
+	const bool hp_explicitly_disabled = !auto_detect_hive_partitioning && !hive_partitioning;
+	const bool ht_enabled = !hive_types_schema.empty();
+	if (hp_explicitly_disabled && ht_enabled) {
+		throw InvalidInputException("cannot disable hive_partitioning when hive_types is enabled");
+	}
+	if (ht_enabled && auto_detect_hive_partitioning && !hive_partitioning) {
+		// hive_types flag implies hive_partitioning
+		hive_partitioning = true;
+		auto_detect_hive_partitioning = false;
+	}
+	if (auto_detect_hive_partitioning) {
+		hive_partitioning = AutoDetectHivePartitioningInternal(files, context);
+	}
+	if (hive_partitioning && hive_types_autocast) {
+		AutoDetectHiveTypesInternal(files.front(), context);
+	}
+}
+void MultiFileReaderOptions::VerifyHiveTypesArePartitions(const std::map<string, string> &partitions) const {
+	for (auto &hive_type : hive_types_schema) {
+		if (partitions.find(hive_type.first) == partitions.end()) {
+			throw InvalidInputException("Unknown hive_type: \"%s\" does not appear to be a partition", hive_type.first);
+		}
+	}
+}
+LogicalType MultiFileReaderOptions::GetHiveLogicalType(const string &hive_partition_column) const {
+	if (!hive_types_schema.empty()) {
+		auto it = hive_types_schema.find(hive_partition_column);
+		if (it != hive_types_schema.end()) {
+			return it->second;
+		}
+	}
+	return LogicalType::VARCHAR;
+}
+Value MultiFileReaderOptions::GetHivePartitionValue(const string &base, const string &entry,
+                                                    ClientContext &context) const {
+	Value value(base);
+	auto it = hive_types_schema.find(entry);
+	if (it == hive_types_schema.end()) {
+		return value;
+	}
+
+	// Handle nulls
+	if (base.empty() || StringUtil::CIEquals(base, "NULL")) {
+		return Value(it->second);
+	}
+
+	if (!value.TryCastAs(context, it->second)) {
+		throw InvalidInputException("Unable to cast '%s' (from hive partition column '%s') to: '%s'", value.ToString(),
+		                            StringUtil::Upper(it->first), it->second.ToString());
+	}
+	return value;
+}
+
 } // namespace duckdb