duckdb 0.7.2-dev314.0 → 0.7.2-dev3154.0

package/src/duckdb/src/function/table/read_csv.cpp

@@ -3,6 +3,7 @@
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/main/database.hpp"
 #include "duckdb/common/string_util.hpp"
+#include "duckdb/common/enum_util.hpp"
 #include "duckdb/common/hive_partitioning.hpp"
 #include "duckdb/common/union_by_name.hpp"
 #include "duckdb/main/config.hpp"
@@ -11,6 +12,8 @@
 #include "duckdb/parser/tableref/table_function_ref.hpp"
 #include "duckdb/planner/operator/logical_get.hpp"
 #include "duckdb/main/extension_helper.hpp"
+#include "duckdb/common/multi_file_reader.hpp"
+#include "duckdb/main/client_data.hpp"
 
 #include <limits>
 
@@ -22,68 +25,57 @@ unique_ptr<CSVFileHandle> ReadCSV::OpenCSV(const string &file_path, FileCompress
 	auto opener = FileSystem::GetFileOpener(context);
 	auto file_handle =
 	    fs.OpenFile(file_path.c_str(), FileFlags::FILE_FLAGS_READ, FileLockType::NO_LOCK, compression, opener);
-	return make_unique<CSVFileHandle>(std::move(file_handle));
-}
-
-void ReadCSVData::InitializeFiles(ClientContext &context, const vector<string> &patterns) {
-	auto &fs = FileSystem::GetFileSystem(context);
-	for (auto &file_pattern : patterns) {
-		auto found_files = fs.GlobFiles(file_pattern, context);
-		files.insert(files.end(), found_files.begin(), found_files.end());
-	}
-	if (files.empty()) {
-		throw IOException("CSV reader needs at least one file to read");
+	if (file_handle->CanSeek()) {
+		file_handle->Reset();
 	}
+	return make_uniq<CSVFileHandle>(std::move(file_handle), false);
 }
 
 void ReadCSVData::FinalizeRead(ClientContext &context) {
 	BaseCSVData::Finalize();
-	auto &config = DBConfig::GetConfig(context);
-	single_threaded = !config.options.experimental_parallel_csv_reader;
-	if (options.has_parallel) {
-		// Override the option set in the config
-		single_threaded = !options.use_parallel;
-	}
+	// Here we identify if we can run this CSV file on parallel or not.
 	bool null_or_empty = options.delimiter.empty() || options.escape.empty() || options.quote.empty() ||
 	                     options.delimiter[0] == '\0' || options.escape[0] == '\0' || options.quote[0] == '\0';
 	bool complex_options = options.delimiter.size() > 1 || options.escape.size() > 1 || options.quote.size() > 1;
-	if (null_or_empty || complex_options || options.new_line == NewLineIdentifier::MIX) {
+	bool not_supported_options = options.null_padding;
+
+	if (!options.run_parallel || null_or_empty || not_supported_options || complex_options ||
+	    options.new_line == NewLineIdentifier::MIX) {
 		// not supported for parallel CSV reading
 		single_threaded = true;
 	}
 }
 
-static unique_ptr<FunctionData> ReadCSVBind(ClientContext &context, TableFunctionBindInput &input,
-                                            vector<LogicalType> &return_types, vector<string> &names) {
-	auto &config = DBConfig::GetConfig(context);
-	if (!config.options.enable_external_access) {
-		throw PermissionException("Scanning CSV files is disabled through configuration");
+uint8_t GetCandidateSpecificity(const LogicalType &candidate_type) {
+	//! Const ht with accepted auto_types and their weights in specificity
+	const duckdb::unordered_map<uint8_t, uint8_t> auto_type_candidates_specificity {
+	    {(uint8_t)LogicalTypeId::VARCHAR, 0},  {(uint8_t)LogicalTypeId::TIMESTAMP, 1},
+	    {(uint8_t)LogicalTypeId::DATE, 2},     {(uint8_t)LogicalTypeId::TIME, 3},
+	    {(uint8_t)LogicalTypeId::DOUBLE, 4},   {(uint8_t)LogicalTypeId::FLOAT, 5},
+	    {(uint8_t)LogicalTypeId::BIGINT, 6},   {(uint8_t)LogicalTypeId::INTEGER, 7},
+	    {(uint8_t)LogicalTypeId::SMALLINT, 8}, {(uint8_t)LogicalTypeId::TINYINT, 9},
+	    {(uint8_t)LogicalTypeId::BOOLEAN, 10}, {(uint8_t)LogicalTypeId::SQLNULL, 11}};
+
+	auto id = (uint8_t)candidate_type.id();
+	auto it = auto_type_candidates_specificity.find(id);
+	if (it == auto_type_candidates_specificity.end()) {
+		throw BinderException("Auto Type Candidate of type %s is not accepted as a valid input",
+		                      EnumUtil::ToString(candidate_type.id()));
 	}
+	return it->second;
+}
 
-	auto result = make_unique<ReadCSVData>();
+static unique_ptr<FunctionData> ReadCSVBind(ClientContext &context, TableFunctionBindInput &input,
+                                            vector<LogicalType> &return_types, vector<string> &names) {
+	auto result = make_uniq<ReadCSVData>();
 	auto &options = result->options;
-
-	vector<string> patterns;
-	if (input.inputs[0].IsNull()) {
-		throw ParserException("CSV reader cannot take NULL as parameter");
-	}
-	if (input.inputs[0].type().id() == LogicalTypeId::LIST) {
-		// list of globs
-		for (auto &val : ListValue::GetChildren(input.inputs[0])) {
-			if (val.IsNull()) {
-				throw ParserException("CSV reader cannot take NULL input as parameter");
-			}
-			patterns.push_back(StringValue::Get(val));
-		}
-	} else {
-		// single glob pattern
-		patterns.push_back(StringValue::Get(input.inputs[0]));
-	}
-
-	result->InitializeFiles(context, patterns);
+	result->files = MultiFileReader::GetFileList(context, input.inputs[0], "CSV");
 
 	bool explicitly_set_columns = false;
 	for (auto &kv : input.named_parameters) {
+		if (MultiFileReader::ParseOption(kv.first, kv.second, options.file_options)) {
+			continue;
+		}
 		auto loption = StringUtil::Lower(kv.first);
 		if (loption == "columns") {
 			explicitly_set_columns = true;
@@ -105,6 +97,32 @@ static unique_ptr<FunctionData> ReadCSVBind(ClientContext &context, TableFunctio
 			if (names.empty()) {
 				throw BinderException("read_csv requires at least a single column as input!");
 			}
+		} else if (loption == "auto_type_candidates") {
+			options.auto_type_candidates.clear();
+			map<uint8_t, LogicalType> candidate_types;
+			// We always have the extremes of Null and Varchar, so we can default to varchar if the
+			// sniffer is not able to confidently detect that column type
+			candidate_types[GetCandidateSpecificity(LogicalType::VARCHAR)] = LogicalType::VARCHAR;
+			candidate_types[GetCandidateSpecificity(LogicalType::SQLNULL)] = LogicalType::SQLNULL;
+
+			auto &child_type = kv.second.type();
+			if (child_type.id() != LogicalTypeId::LIST) {
+				throw BinderException("read_csv auto_types requires a list as input");
+			}
+			auto &list_children = ListValue::GetChildren(kv.second);
+			if (list_children.empty()) {
+				throw BinderException("auto_type_candidates requires at least one type");
+			}
+			for (auto &child : list_children) {
+				if (child.type().id() != LogicalTypeId::VARCHAR) {
+					throw BinderException("auto_type_candidates requires a type specification as string");
+				}
+				auto candidate_type = TransformStringToLogicalType(StringValue::Get(child), context);
+				candidate_types[GetCandidateSpecificity(candidate_type)] = candidate_type;
+			}
+			for (auto &candidate_type : candidate_types) {
+				options.auto_type_candidates.emplace_back(candidate_type.second);
+			}
 		} else if (loption == "column_names" || loption == "names") {
 			if (!options.name_list.empty()) {
 				throw BinderException("read_csv_auto column_names/names can only be supplied once");
@@ -160,14 +178,16 @@ static unique_ptr<FunctionData> ReadCSVBind(ClientContext &context, TableFunctio
 			options.all_varchar = BooleanValue::Get(kv.second);
 		} else if (loption == "normalize_names") {
 			options.normalize_names = BooleanValue::Get(kv.second);
-		} else if (loption == "filename") {
-			options.include_file_name = BooleanValue::Get(kv.second);
-		} else if (loption == "hive_partitioning") {
-			options.include_parsed_hive_partitions = BooleanValue::Get(kv.second);
+		} else if (loption == "parallel") {
+			options.run_parallel = BooleanValue::Get(kv.second);
 		} else {
 			options.SetReadOption(loption, kv.second, names);
 		}
 	}
+	if (options.file_options.auto_detect_hive_partitioning) {
+		options.file_options.hive_partitioning = MultiFileReaderOptions::AutoDetectHivePartitioning(result->files);
+	}
+
 	if (!options.auto_detect && return_types.empty()) {
 		throw BinderException("read_csv requires columns to be specified through the 'columns' option. Use "
 		                      "read_csv_auto or set read_csv(..., "
@@ -175,7 +195,7 @@ static unique_ptr<FunctionData> ReadCSVBind(ClientContext &context, TableFunctio
 	}
 	if (options.auto_detect) {
 		options.file_path = result->files[0];
-		auto initial_reader = make_unique<BufferedCSVReader>(context, options);
+		auto initial_reader = make_uniq<BufferedCSVReader>(context, options);
 		return_types.assign(initial_reader->return_types.begin(), initial_reader->return_types.end());
 		if (names.empty()) {
 			names.assign(initial_reader->names.begin(), initial_reader->names.end());
@@ -190,61 +210,45 @@ static unique_ptr<FunctionData> ReadCSVBind(ClientContext &context, TableFunctio
 			} else {
 				D_ASSERT(return_types.size() == names.size());
 			}
+			initial_reader->names = names;
 		}
 		options = initial_reader->options;
-		result->sql_types = initial_reader->return_types;
 		result->initial_reader = std::move(initial_reader);
 	} else {
-		result->sql_types = return_types;
 		D_ASSERT(return_types.size() == names.size());
 	}
-
-	// union_col_names will exclude filename and hivepartition
-	if (options.union_by_name) {
-		case_insensitive_map_t<idx_t> union_names_map;
-		vector<string> union_col_names;
-		vector<LogicalType> union_col_types;
-
-		auto dummy_readers = UnionByName<BufferedCSVReader, BufferedCSVReaderOptions>::UnionCols(
-		    context, result->files, union_col_types, union_col_names, union_names_map, options);
-
-		dummy_readers = UnionByName<BufferedCSVReader, BufferedCSVReaderOptions>::CreateUnionMap(
-		    std::move(dummy_readers), union_col_types, union_col_names, union_names_map);
-
-		std::move(dummy_readers.begin(), dummy_readers.end(), std::back_inserter(result->union_readers));
-		for (auto &reader : result->union_readers) {
-			reader->insert_cols_idx = reader->union_idx_map;
+	result->csv_types = return_types;
+	result->csv_names = names;
+
+	if (options.file_options.union_by_name) {
+		result->reader_bind =
+		    MultiFileReader::BindUnionReader<BufferedCSVReader>(context, return_types, names, *result, options);
+		if (result->union_readers.size() > 1) {
+			result->column_info.emplace_back(result->csv_names, result->csv_types);
+			for (idx_t i = 1; i < result->union_readers.size(); i++) {
+				result->column_info.emplace_back(result->union_readers[i]->names,
+				                                 result->union_readers[i]->return_types);
+			}
 		}
-
-		names.assign(union_col_names.begin(), union_col_names.end());
-		return_types.assign(union_col_types.begin(), union_col_types.end());
-		const idx_t first_file_index = 0;
-		result->initial_reader = std::move(result->union_readers[first_file_index]);
-		D_ASSERT(names.size() == return_types.size());
-
 		if (!options.sql_types_per_column.empty()) {
 			auto exception = BufferedCSVReader::ColumnTypesError(options.sql_types_per_column, names);
 			if (!exception.empty()) {
 				throw BinderException(exception);
 			}
 		}
+	} else {
+		result->reader_bind = MultiFileReader::BindOptions(options.file_options, result->files, return_types, names);
 	}
-
-	if (result->options.include_file_name) {
-		result->filename_col_idx = names.size();
-		return_types.emplace_back(LogicalType::VARCHAR);
-		names.emplace_back("filename");
-	}
-
-	if (result->options.include_parsed_hive_partitions) {
-		auto partitions = HivePartitioning::Parse(result->files[0]);
-		result->hive_partition_col_idx = names.size();
-		for (auto &part : partitions) {
-			return_types.emplace_back(LogicalType::VARCHAR);
-			names.emplace_back(part.first);
+	auto &fs = FileSystem::GetFileSystem(context);
+	for (auto &file : result->files) {
+		if (fs.IsPipe(file)) {
+			result->is_pipe = true;
+			result->single_threaded = true;
+			break;
 		}
 	}
-	result->options.names = names;
+	result->return_types = return_types;
+	result->return_names = names;
 	result->FinalizeRead(context);
 	return std::move(result);
 }
@@ -265,36 +269,54 @@ struct ParallelCSVGlobalState : public GlobalTableFunctionState {
 public:
 	ParallelCSVGlobalState(ClientContext &context, unique_ptr<CSVFileHandle> file_handle_p,
 	                       vector<string> &files_path_p, idx_t system_threads_p, idx_t buffer_size_p,
-	                       idx_t rows_to_skip, bool force_parallelism_p)
+	                       idx_t rows_to_skip, bool force_parallelism_p, vector<column_t> column_ids_p)
 	    : file_handle(std::move(file_handle_p)), system_threads(system_threads_p), buffer_size(buffer_size_p),
-	      force_parallelism(force_parallelism_p) {
+	      force_parallelism(force_parallelism_p), column_ids(std::move(column_ids_p)) {
+		file_handle->DisableReset();
 		current_file_path = files_path_p[0];
 		estimated_linenr = rows_to_skip;
 		file_size = file_handle->FileSize();
 		first_file_size = file_size;
 		bytes_read = 0;
-		if (buffer_size < file_size) {
+		if (buffer_size < file_size || file_size == 0) {
 			bytes_per_local_state = buffer_size / ParallelCSVGlobalState::MaxThreads();
 		} else {
 			bytes_per_local_state = file_size / MaxThreads();
 		}
-		current_buffer = make_shared<CSVBuffer>(context, buffer_size, *file_handle, current_csv_position);
-		next_buffer = current_buffer->Next(*file_handle, buffer_size, current_csv_position);
+		if (bytes_per_local_state == 0) {
+			// In practice, I think this won't happen, it only happens because we are mocking up test scenarios
+			// this boy needs to be at least one.
+			bytes_per_local_state = 1;
+		}
+		for (idx_t i = 0; i < rows_to_skip; i++) {
+			file_handle->ReadLine();
+		}
+		first_position = current_csv_position;
+		current_buffer = make_shared<CSVBuffer>(context, buffer_size, *file_handle, current_csv_position, file_number);
+		next_buffer = shared_ptr<CSVBuffer>(
+		    current_buffer->Next(*file_handle, buffer_size, current_csv_position, file_number).release());
 		running_threads = MaxThreads();
 	}
 	ParallelCSVGlobalState() {
+		running_threads = MaxThreads();
 	}
 
 	~ParallelCSVGlobalState() override {
 	}
 
+	//! How many bytes were read up to this point
+	atomic<idx_t> bytes_read;
+	//! Size of current file
+	idx_t file_size;
+
+public:
 	idx_t MaxThreads() const override;
-	//! Returns buffer and index that caller thread should read.
-	unique_ptr<CSVBufferRead> Next(ClientContext &context, ReadCSVData &bind_data);
+	//! Updates the CSV reader with the next buffer to read. Returns false if no more buffers are available.
+	bool Next(ClientContext &context, const ReadCSVData &bind_data, unique_ptr<ParallelCSVReader> &reader);
 	//! Verify if the CSV File was read correctly
 	void Verify();
 
-	void UpdateVerification(VerificationPositions positions);
+	void UpdateVerification(VerificationPositions positions, idx_t file_number);
 
 	void IncrementThread();
 
@@ -302,11 +324,6 @@ public:
 
 	bool Finished();
 
-	//! How many bytes were read up to this point
-	atomic<idx_t> bytes_read;
-	//! Size of current file
-	idx_t file_size;
-
 	double GetProgress(ReadCSVData &bind_data) const {
 		idx_t total_files = bind_data.files.size();
 
@@ -325,7 +342,6 @@ public:
 
 private:
 	//! File Handle for current file
-	unique_ptr<CSVFileHandle> prev_file_handle;
 	unique_ptr<CSVFileHandle> file_handle;
 	shared_ptr<CSVBuffer> current_buffer;
 	shared_ptr<CSVBuffer> next_buffer;
@@ -351,25 +367,30 @@ private:
 	//! Current batch index
 	idx_t batch_index = 0;
 	//! Forces parallelism for small CSV Files, should only be used for testing.
-	bool force_parallelism;
+	bool force_parallelism = false;
 	//! Current (Global) position of CSV
 	idx_t current_csv_position = 0;
+	//! First Position of First Buffer
+	idx_t first_position = 0;
+	//! Current File Number
+	idx_t file_number = 0;
 	idx_t max_tuple_end = 0;
 	//! the vector stores positions where threads ended the last line they read in the CSV File, and the set stores
 	//! positions where they started reading the first line.
-	vector<idx_t> tuple_end;
-	set<idx_t> tuple_start;
+	vector<vector<idx_t>> tuple_end;
+	vector<set<idx_t>> tuple_start;
 	idx_t running_threads = 0;
+	//! The column ids to read
+	vector<column_t> column_ids;
 };
 
 idx_t ParallelCSVGlobalState::MaxThreads() const {
 	if (force_parallelism) {
 		return system_threads;
 	}
-
 	idx_t one_mb = 1000000; // We initialize max one thread per Mb
 	idx_t threads_per_mb = first_file_size / one_mb + 1;
-	if (threads_per_mb < system_threads) {
+	if (threads_per_mb < system_threads || threads_per_mb == 1) {
 		return threads_per_mb;
 	}
 
@@ -395,31 +416,43 @@ bool ParallelCSVGlobalState::Finished() {
 void ParallelCSVGlobalState::Verify() {
 	// All threads are done, we run some magic sweet verification code
 	if (running_threads == 0) {
-		for (auto &last_pos : tuple_end) {
-			auto first_pos = tuple_start.find(last_pos);
-			if (first_pos == tuple_start.end()) {
-				// this might be necessary due to carriage returns outside buffer scopes.
-				first_pos = tuple_start.find(last_pos + 1);
+		D_ASSERT(tuple_end.size() == tuple_start.size());
+		for (idx_t i = 0; i < tuple_start.size(); i++) {
+			auto &current_tuple_end = tuple_end[i];
+			auto &current_tuple_start = tuple_start[i];
+			// figure out max value of last_pos
+			if (current_tuple_end.empty()) {
+				return;
 			}
-			if (first_pos == tuple_start.end() && last_pos != NumericLimits<uint64_t>::Maximum()) {
-				string error = "Not possible to read this CSV File with multithreading. Tuple: " + to_string(last_pos) +
-				               " does not have a match\n";
-				error += "End Lines: \n";
-				for (auto &end_line : tuple_end) {
-					error += to_string(end_line) + "\n";
+			auto max_value = *max_element(std::begin(current_tuple_end), std::end(current_tuple_end));
+			for (auto &last_pos : current_tuple_end) {
+				auto first_pos = current_tuple_start.find(last_pos);
+				if (first_pos == current_tuple_start.end()) {
+					// this might be necessary due to carriage returns outside buffer scopes.
+					first_pos = current_tuple_start.find(last_pos + 1);
 				}
-				error += "Start Lines: \n";
-				for (auto &start_line : tuple_start) {
-					error += to_string(start_line) + "\n";
+				if (first_pos == current_tuple_start.end() && last_pos != max_value) {
+					string error =
+					    "Not possible to read this CSV File with multithreading. Tuple: " + to_string(last_pos) +
+					    " does not have a match\n";
+					error += "End Lines: \n";
+					for (auto &end_line : current_tuple_end) {
+						error += to_string(end_line) + "\n";
+					}
+					error += "Start Lines: \n";
+					for (auto &start_line : current_tuple_start) {
+						error += to_string(start_line) + "\n";
+					}
+					throw InvalidInputException(
+					    "CSV File not supported for multithreading. Please run single-threaded CSV Reading");
 				}
-				throw InvalidInputException(
-				    "CSV File not supported for multithreading. Please run single-threaded CSV Reading");
 			}
 		}
 	}
 }
 
-unique_ptr<CSVBufferRead> ParallelCSVGlobalState::Next(ClientContext &context, ReadCSVData &bind_data) {
+bool ParallelCSVGlobalState::Next(ClientContext &context, const ReadCSVData &bind_data,
+                                  unique_ptr<ParallelCSVReader> &reader) {
 	lock_guard<mutex> parallel_lock(main_mutex);
 	if (!current_buffer) {
 		// This means we are done with the current file, we need to go to the next one (if exists).
@@ -427,19 +460,23 @@ unique_ptr<CSVBufferRead> ParallelCSVGlobalState::Next(ClientContext &context, R
 			current_file_path = bind_data.files[file_index++];
 			file_handle = ReadCSV::OpenCSV(current_file_path, bind_data.options.compression, context);
 			current_csv_position = 0;
-			current_buffer = make_shared<CSVBuffer>(context, buffer_size, *file_handle, current_csv_position);
-			next_buffer = current_buffer->Next(*file_handle, buffer_size, current_csv_position);
+			file_number++;
+			current_buffer =
+			    make_shared<CSVBuffer>(context, buffer_size, *file_handle, current_csv_position, file_number);
+			next_buffer = shared_ptr<CSVBuffer>(
+			    current_buffer->Next(*file_handle, buffer_size, current_csv_position, file_number).release());
 		} else {
 			// We are done scanning.
-			return nullptr;
+			reader.reset();
+			return false;
 		}
 	}
 	// set up the current buffer
-	auto result = make_unique<CSVBufferRead>(current_buffer, next_buffer, next_byte, next_byte + bytes_per_local_state,
-	                                         batch_index++, estimated_linenr);
+	auto result = make_uniq<CSVBufferRead>(current_buffer, next_buffer, next_byte, next_byte + bytes_per_local_state,
+	                                       batch_index++, estimated_linenr);
 	// move the byte index of the CSV reader to the next buffer
 	next_byte += bytes_per_local_state;
-	estimated_linenr += bytes_per_local_state / (bind_data.sql_types.size() * 5); // estimate 5 bytes per column
+	estimated_linenr += bytes_per_local_state / (bind_data.csv_types.size() * 5); // estimate 5 bytes per column
 	if (next_byte >= current_buffer->GetBufferSize()) {
 		// We replace the current buffer with the next buffer
 		next_byte = 0;
@@ -447,19 +484,54 @@ unique_ptr<CSVBufferRead> ParallelCSVGlobalState::Next(ClientContext &context, R
 		current_buffer = next_buffer;
 		if (next_buffer) {
 			// Next buffer gets the next-next buffer
-			next_buffer = next_buffer->Next(*file_handle, buffer_size, current_csv_position);
+			next_buffer = shared_ptr<CSVBuffer>(
+			    next_buffer->Next(*file_handle, buffer_size, current_csv_position, file_number).release());
+		}
+	}
+	if (!reader || reader->options.file_path != current_file_path) {
+		// we either don't have a reader, or the reader was created for a different file
+		// we need to create a new reader and instantiate it
+		if (file_index > 0 && file_index <= bind_data.union_readers.size() && bind_data.union_readers[file_index - 1]) {
+			// we are doing UNION BY NAME - fetch the options from the union reader for this file
+			auto &union_reader = *bind_data.union_readers[file_index - 1];
+			reader = make_uniq<ParallelCSVReader>(context, union_reader.options, std::move(result), first_position,
+			                                      union_reader.GetTypes());
+			reader->names = union_reader.GetNames();
+		} else if (file_index <= bind_data.column_info.size()) {
+			// Serialized Union By name
+			reader = make_uniq<ParallelCSVReader>(context, bind_data.options, std::move(result), first_position,
+			                                      bind_data.column_info[file_index - 1].types);
+			reader->names = bind_data.column_info[file_index - 1].names;
+		} else {
+			// regular file - use the standard options
+			reader = make_uniq<ParallelCSVReader>(context, bind_data.options, std::move(result), first_position,
+			                                      bind_data.csv_types);
+			reader->names = bind_data.csv_names;
 		}
+		reader->options.file_path = current_file_path;
+		MultiFileReader::InitializeReader(*reader, bind_data.options.file_options, bind_data.reader_bind,
+		                                  bind_data.return_types, bind_data.return_names, column_ids, nullptr,
+		                                  bind_data.files.front());
+	} else {
+		// update the current reader
+		reader->SetBufferRead(std::move(result));
 	}
-	return result;
+	return true;
 }
-void ParallelCSVGlobalState::UpdateVerification(VerificationPositions positions) {
+void ParallelCSVGlobalState::UpdateVerification(VerificationPositions positions, idx_t file_number_p) {
 	lock_guard<mutex> parallel_lock(main_mutex);
 	if (positions.beginning_of_first_line < positions.end_of_last_line) {
 		if (positions.end_of_last_line > max_tuple_end) {
 			max_tuple_end = positions.end_of_last_line;
 		}
-		tuple_start.insert(positions.beginning_of_first_line);
-		tuple_end.push_back(positions.end_of_last_line);
+		while (file_number_p >= tuple_start.size()) {
+			vector<idx_t> empty_tuple_end;
+			set<idx_t> empty_set;
+			tuple_start.emplace_back(empty_set);
+			tuple_end.emplace_back(empty_tuple_end);
+		}
+		tuple_start[file_number_p].insert(positions.beginning_of_first_line);
+		tuple_end[file_number_p].push_back(positions.end_of_last_line);
 	}
 }
 
@@ -468,17 +540,15 @@ static unique_ptr<GlobalTableFunctionState> ParallelCSVInitGlobal(ClientContext
 	auto &bind_data = (ReadCSVData &)*input.bind_data;
 	if (bind_data.files.empty()) {
 		// This can happen when a filename based filter pushdown has eliminated all possible files for this scan.
-		return make_unique<ParallelCSVGlobalState>();
+		return make_uniq<ParallelCSVGlobalState>();
 	}
 	unique_ptr<CSVFileHandle> file_handle;
 
 	bind_data.options.file_path = bind_data.files[0];
 	file_handle = ReadCSV::OpenCSV(bind_data.options.file_path, bind_data.options.compression, context);
-	idx_t rows_to_skip =
-	    bind_data.options.skip_rows + (bind_data.options.has_header && bind_data.options.header ? 1 : 0);
-	return make_unique<ParallelCSVGlobalState>(context, std::move(file_handle), bind_data.files,
-	                                           context.db->NumberOfThreads(), bind_data.options.buffer_size,
-	                                           rows_to_skip, ClientConfig::GetConfig(context).verify_parallelism);
+	return make_uniq<ParallelCSVGlobalState>(
+	    context, std::move(file_handle), bind_data.files, context.db->NumberOfThreads(), bind_data.options.buffer_size,
+	    bind_data.options.skip_rows, ClientConfig::GetConfig(context).verify_parallelism, input.column_ids);
 }
 
 //===--------------------------------------------------------------------===//
@@ -497,23 +567,21 @@ public:
 
 unique_ptr<LocalTableFunctionState> ParallelReadCSVInitLocal(ExecutionContext &context, TableFunctionInitInput &input,
                                                              GlobalTableFunctionState *global_state_p) {
-	auto &csv_data = (ReadCSVData &)*input.bind_data;
-	auto &global_state = (ParallelCSVGlobalState &)*global_state_p;
-	auto next_local_buffer = global_state.Next(context.client, csv_data);
+	auto &csv_data = input.bind_data->Cast<ReadCSVData>();
+	auto &global_state = global_state_p->Cast<ParallelCSVGlobalState>();
 	unique_ptr<ParallelCSVReader> csv_reader;
-	if (next_local_buffer) {
-		csv_reader = make_unique<ParallelCSVReader>(context.client, csv_data.options, std::move(next_local_buffer),
-		                                            csv_data.sql_types);
-	} else {
+	auto has_next = global_state.Next(context.client, csv_data, csv_reader);
+	if (!has_next) {
 		global_state.DecrementThread();
+		csv_reader.reset();
 	}
-	return make_unique<ParallelCSVLocalState>(std::move(csv_reader));
+	return make_uniq<ParallelCSVLocalState>(std::move(csv_reader));
 }
 
 static void ParallelReadCSVFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) {
-	auto &bind_data = (ReadCSVData &)*data_p.bind_data;
-	auto &csv_global_state = (ParallelCSVGlobalState &)*data_p.global_state;
-	auto &csv_local_state = (ParallelCSVLocalState &)*data_p.local_state;
+	auto &bind_data = data_p.bind_data->Cast<ReadCSVData>();
+	auto &csv_global_state = data_p.global_state->Cast<ParallelCSVGlobalState>();
+	auto &csv_local_state = data_p.local_state->Cast<ParallelCSVLocalState>();
 
 	if (!csv_local_state.csv_reader) {
 		// no csv_reader was set, this can happen when a filename-based filter has filtered out all possible files
@@ -522,21 +590,20 @@ static void ParallelReadCSVFunction(ClientContext &context, TableFunctionInput &
 
 	do {
 		if (output.size() != 0) {
+			MultiFileReader::FinalizeChunk(bind_data.reader_bind, csv_local_state.csv_reader->reader_data, output);
 			break;
 		}
 		if (csv_local_state.csv_reader->finished) {
 			auto verification_updates = csv_local_state.csv_reader->GetVerificationPositions();
-			if (!csv_local_state.csv_reader->buffer->next_buffer) {
-				// if it's the last line of the file we mark as the maximum
-				verification_updates.end_of_last_line = NumericLimits<uint64_t>::Maximum();
+			if (verification_updates.beginning_of_first_line != verification_updates.end_of_last_line) {
+				csv_global_state.UpdateVerification(verification_updates,
+				                                    csv_local_state.csv_reader->buffer->buffer->GetFileNumber());
 			}
-			csv_global_state.UpdateVerification(verification_updates);
-			auto next_chunk = csv_global_state.Next(context, bind_data);
-			if (!next_chunk) {
+			auto has_next = csv_global_state.Next(context, bind_data, csv_local_state.csv_reader);
+			if (!has_next) {
 				csv_global_state.DecrementThread();
 				break;
 			}
-			csv_local_state.csv_reader->SetBufferRead(std::move(next_chunk));
 		}
 		csv_local_state.csv_reader->ParseCSV(output);
 
@@ -544,15 +611,6 @@ static void ParallelReadCSVFunction(ClientContext &context, TableFunctionInput &
 	if (csv_global_state.Finished()) {
 		csv_global_state.Verify();
 	}
-	if (bind_data.options.union_by_name) {
-		throw InternalException("FIXME: union by name");
-	}
-	if (bind_data.options.include_file_name) {
-		throw InternalException("FIXME: output file name");
-	}
-	if (bind_data.options.include_parsed_hive_partitions) {
-		throw InternalException("FIXME: hive partitions");
-	}
 }
 
 //===--------------------------------------------------------------------===//
@@ -572,7 +630,11 @@ struct SingleThreadedCSVState : public GlobalTableFunctionState {
 	//! This goes from [0...next_file] * 100
 	atomic<idx_t> progress_in_files;
 	//! The set of SQL types
-	vector<LogicalType> sql_types;
+	vector<LogicalType> csv_types;
+	//! The set of SQL names to be read from the file
+	vector<string> csv_names;
+	//! The column ids to read
+	vector<column_t> column_ids;
 
 	idx_t MaxThreads() const override {
 		return total_files;
@@ -602,11 +664,18 @@ struct SingleThreadedCSVState : public GlobalTableFunctionState {
 		}
 		// reuse csv_readers was created during binding
 		unique_ptr<BufferedCSVReader> result;
-		if (options.union_by_name) {
+		if (file_index < bind_data.union_readers.size() && bind_data.union_readers[file_index]) {
 			result = std::move(bind_data.union_readers[file_index]);
 		} else {
+			auto union_by_name = options.file_options.union_by_name;
 			options.file_path = bind_data.files[file_index];
-			result = make_unique<BufferedCSVReader>(context, std::move(options), sql_types);
+			result = make_uniq<BufferedCSVReader>(context, std::move(options), csv_types);
+			if (!union_by_name) {
+				result->names = csv_names;
+			}
+			MultiFileReader::InitializeReader(*result, bind_data.options.file_options, bind_data.reader_bind,
+			                                  bind_data.return_types, bind_data.return_names, column_ids, nullptr,
+			                                  bind_data.files.front());
 		}
 		total_size = result->file_handle->FileSize();
 		return result;
@@ -633,29 +702,48 @@ public:
 static unique_ptr<GlobalTableFunctionState> SingleThreadedCSVInit(ClientContext &context,
                                                                   TableFunctionInitInput &input) {
 	auto &bind_data = (ReadCSVData &)*input.bind_data;
-	auto result = make_unique<SingleThreadedCSVState>(bind_data.files.size());
-	if (bind_data.initial_reader) {
-		result->initial_reader = std::move(bind_data.initial_reader);
-	} else if (bind_data.files.empty()) {
+	auto result = make_uniq<SingleThreadedCSVState>(bind_data.files.size());
+	if (bind_data.files.empty()) {
 		// This can happen when a filename based filter pushdown has eliminated all possible files for this scan.
 		return std::move(result);
 	} else {
 		bind_data.options.file_path = bind_data.files[0];
-		result->initial_reader = make_unique<BufferedCSVReader>(context, bind_data.options, bind_data.sql_types);
+		if (bind_data.initial_reader && bind_data.is_pipe) {
+			// If this is a pipe and an initial reader already exists due to read_csv_auto
+			// We must re-use it, since we can't restart the reader due for it being a pipe.
+			result->initial_reader = std::move(bind_data.initial_reader);
+		} else {
+			result->initial_reader = make_uniq<BufferedCSVReader>(context, bind_data.options, bind_data.csv_types);
+		}
+		if (!bind_data.options.file_options.union_by_name) {
+			result->initial_reader->names = bind_data.csv_names;
+		}
 		if (bind_data.options.auto_detect) {
 			bind_data.options = result->initial_reader->options;
 		}
 	}
-	if (!bind_data.options.union_by_name) {
+	MultiFileReader::InitializeReader(*result->initial_reader, bind_data.options.file_options, bind_data.reader_bind,
+	                                  bind_data.return_types, bind_data.return_names, input.column_ids, input.filters,
+	                                  bind_data.files.front());
+	for (auto &reader : bind_data.union_readers) {
+		if (!reader) {
+			continue;
+		}
+		MultiFileReader::InitializeReader(*reader, bind_data.options.file_options, bind_data.reader_bind,
+		                                  bind_data.return_types, bind_data.return_names, input.column_ids,
+		                                  input.filters, bind_data.files.front());
+	}
+	result->column_ids = input.column_ids;
+
+	if (!bind_data.options.file_options.union_by_name) {
 		// if we are reading multiple files - run auto-detect only on the first file
 		// UNLESS union by name is turned on - in that case we assume that different files have different schemas
 		// as such, we need to re-run the auto detection on each file
 		bind_data.options.auto_detect = false;
 	}
+	result->csv_types = bind_data.csv_types;
+	result->csv_names = bind_data.csv_names;
 	result->next_file = 1;
-	if (result->initial_reader) {
-		result->sql_types = result->initial_reader->return_types;
-	}
 	return std::move(result);
 }
 
@@ -664,7 +752,7 @@ unique_ptr<LocalTableFunctionState> SingleThreadedReadCSVInitLocal(ExecutionCont
                                                                    GlobalTableFunctionState *global_state_p) {
 	auto &bind_data = (ReadCSVData &)*input.bind_data;
 	auto &data = (SingleThreadedCSVState &)*global_state_p;
-	auto result = make_unique<SingleThreadedCSVLocalState>();
+	auto result = make_uniq<SingleThreadedCSVLocalState>();
 	result->csv_reader = data.GetCSVReader(context.client, bind_data, result->file_index, result->total_size);
 	return std::move(result);
 }
@@ -672,7 +760,7 @@ unique_ptr<LocalTableFunctionState> SingleThreadedReadCSVInitLocal(ExecutionCont
 static void SingleThreadedCSVFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) {
 	auto &bind_data = (ReadCSVData &)*data_p.bind_data;
 	auto &data = (SingleThreadedCSVState &)*data_p.global_state;
-	auto &lstate = (SingleThreadedCSVLocalState &)*data_p.local_state;
+	auto &lstate = data_p.local_state->Cast<SingleThreadedCSVLocalState>();
 	if (!lstate.csv_reader) {
 		// no csv_reader was set, this can happen when a filename-based filter has filtered out all possible files
 		return;
@@ -708,41 +796,10 @@ static void SingleThreadedCSVFunction(ClientContext &context, TableFunctionInput
 			}
 			lstate.bytes_read = 0;
 		} else {
+			MultiFileReader::FinalizeChunk(bind_data.reader_bind, lstate.csv_reader->reader_data, output);
 			break;
 		}
 	} while (true);
-
-	if (bind_data.options.union_by_name) {
-		UnionByName<BufferedCSVReader, BufferedCSVReaderOptions>::SetNullUnionCols(output,
-		                                                                           lstate.csv_reader->union_null_cols);
-	}
-	if (bind_data.options.include_file_name) {
-		auto &col = output.data[bind_data.filename_col_idx];
-		col.SetValue(0, Value(lstate.csv_reader->options.file_path));
-		col.SetVectorType(VectorType::CONSTANT_VECTOR);
-	}
-	if (bind_data.options.include_parsed_hive_partitions) {
-		auto partitions = HivePartitioning::Parse(lstate.csv_reader->options.file_path);
-
-		idx_t i = bind_data.hive_partition_col_idx;
-
-		if (partitions.size() != (bind_data.options.names.size() - bind_data.hive_partition_col_idx)) {
-			throw IOException("Hive partition count mismatch, expected " +
-			                  std::to_string(bind_data.options.names.size() - bind_data.hive_partition_col_idx) +
-			                  " hive partitions, got " + std::to_string(partitions.size()) + "\n");
-		}
-
-		for (auto &part : partitions) {
-			if (bind_data.options.names[i] != part.first) {
-				throw IOException("Hive partition names mismatch, expected '" + bind_data.options.names[i] +
-				                  "' but found '" + part.first + "' for file '" + lstate.csv_reader->options.file_path +
-				                  "'");
-			}
-			auto &col = output.data[i++];
-			col.SetValue(0, Value(part.second));
-			col.SetVectorType(VectorType::CONSTANT_VECTOR);
-		}
-	}
 }
 
 //===--------------------------------------------------------------------===//
@@ -780,10 +837,10 @@ static idx_t CSVReaderGetBatchIndex(ClientContext &context, const FunctionData *
                                     LocalTableFunctionState *local_state, GlobalTableFunctionState *global_state) {
 	auto &bind_data = (ReadCSVData &)*bind_data_p;
 	if (bind_data.single_threaded) {
-		auto &data = (SingleThreadedCSVLocalState &)*local_state;
+		auto &data = local_state->Cast<SingleThreadedCSVLocalState>();
 		return data.file_index;
 	}
-	auto &data = (ParallelCSVLocalState &)*local_state;
+	auto &data = local_state->Cast<ParallelCSVLocalState>();
 	return data.csv_reader->buffer->batch_index;
 }
 
@@ -795,6 +852,7 @@ static void ReadCSVAddNamedParameters(TableFunction &table_function) {
 	table_function.named_parameters["escape"] = LogicalType::VARCHAR;
 	table_function.named_parameters["nullstr"] = LogicalType::VARCHAR;
 	table_function.named_parameters["columns"] = LogicalType::ANY;
+	table_function.named_parameters["auto_type_candidates"] = LogicalType::ANY;
 	table_function.named_parameters["header"] = LogicalType::BOOLEAN;
 	table_function.named_parameters["auto_detect"] = LogicalType::BOOLEAN;
 	table_function.named_parameters["sample_size"] = LogicalType::BIGINT;
@@ -805,16 +863,21 @@ static void ReadCSVAddNamedParameters(TableFunction &table_function) {
 	table_function.named_parameters["timestampformat"] = LogicalType::VARCHAR;
 	table_function.named_parameters["normalize_names"] = LogicalType::BOOLEAN;
 	table_function.named_parameters["compression"] = LogicalType::VARCHAR;
-	table_function.named_parameters["filename"] = LogicalType::BOOLEAN;
-	table_function.named_parameters["hive_partitioning"] = LogicalType::BOOLEAN;
 	table_function.named_parameters["skip"] = LogicalType::BIGINT;
 	table_function.named_parameters["max_line_size"] = LogicalType::VARCHAR;
 	table_function.named_parameters["maximum_line_size"] = LogicalType::VARCHAR;
 	table_function.named_parameters["ignore_errors"] = LogicalType::BOOLEAN;
-	table_function.named_parameters["union_by_name"] = LogicalType::BOOLEAN;
 	table_function.named_parameters["buffer_size"] = LogicalType::UBIGINT;
 	table_function.named_parameters["decimal_separator"] = LogicalType::VARCHAR;
 	table_function.named_parameters["parallel"] = LogicalType::BOOLEAN;
+	table_function.named_parameters["null_padding"] = LogicalType::BOOLEAN;
+	table_function.named_parameters["allow_quoted_nulls"] = LogicalType::BOOLEAN;
+	table_function.named_parameters["column_types"] = LogicalType::ANY;
+	table_function.named_parameters["dtypes"] = LogicalType::ANY;
+	table_function.named_parameters["types"] = LogicalType::ANY;
+	table_function.named_parameters["names"] = LogicalType::LIST(LogicalType::VARCHAR);
+	table_function.named_parameters["column_names"] = LogicalType::LIST(LogicalType::VARCHAR);
+	MultiFileReader::AddParameters(table_function);
 }
 
 double CSVReaderProgress(ClientContext &context, const FunctionData *bind_data_p,
@@ -832,22 +895,10 @@ double CSVReaderProgress(ClientContext &context, const FunctionData *bind_data_p
 void CSVComplexFilterPushdown(ClientContext &context, LogicalGet &get, FunctionData *bind_data_p,
                               vector<unique_ptr<Expression>> &filters) {
 	auto data = (ReadCSVData *)bind_data_p;
-
-	if (data->options.include_parsed_hive_partitions || data->options.include_file_name) {
-		string first_file = data->files[0];
-
-		unordered_map<string, column_t> column_map;
-		for (idx_t i = 0; i < get.column_ids.size(); i++) {
-			column_map.insert({get.names[get.column_ids[i]], i});
-		}
-
-		HivePartitioning::ApplyFiltersToFileList(context, data->files, filters, column_map, get.table_index,
-		                                         data->options.include_parsed_hive_partitions,
-		                                         data->options.include_file_name);
-
-		if (data->files.empty() || data->files[0] != first_file) {
-			data->initial_reader.reset();
-		}
+	auto reset_reader =
+	    MultiFileReader::ComplexFilterPushdown(context, data->files, data->options.file_options, get, filters);
+	if (reset_reader) {
+		MultiFileReader::PruneReaders(*data);
 	}
 }
 
@@ -855,13 +906,13 @@ unique_ptr<NodeStatistics> CSVReaderCardinality(ClientContext &context, const Fu
 	auto &bind_data = (ReadCSVData &)*bind_data_p;
 	idx_t per_file_cardinality = 0;
 	if (bind_data.initial_reader && bind_data.initial_reader->file_handle) {
-		auto estimated_row_width = (bind_data.sql_types.size() * 5);
+		auto estimated_row_width = (bind_data.csv_types.size() * 5);
 		per_file_cardinality = bind_data.initial_reader->file_handle->FileSize() / estimated_row_width;
 	} else {
 		// determined through the scientific method as the average amount of rows in a CSV file
 		per_file_cardinality = 42;
 	}
-	return make_unique<NodeStatistics>(bind_data.files.size() * per_file_cardinality);
+	return make_uniq<NodeStatistics>(bind_data.files.size() * per_file_cardinality);
 }
 
 void BufferedCSVReaderOptions::Serialize(FieldWriter &writer) const {
@@ -879,9 +930,11 @@ void BufferedCSVReaderOptions::Serialize(FieldWriter &writer) const {
 	writer.WriteField<idx_t>(buffer_sample_size);
 	writer.WriteString(null_str);
 	writer.WriteField<FileCompressionType>(compression);
+	writer.WriteField<NewLineIdentifier>(new_line);
+	writer.WriteField<bool>(allow_quoted_nulls);
 	// read options
-	writer.WriteList<string>(names);
 	writer.WriteField<idx_t>(skip_rows);
+	writer.WriteField<bool>(skip_rows_set);
 	writer.WriteField<idx_t>(maximum_line_size);
 	writer.WriteField<bool>(normalize_names);
 	writer.WriteListNoReference<bool>(force_not_null);
@@ -890,9 +943,9 @@ void BufferedCSVReaderOptions::Serialize(FieldWriter &writer) const {
 	writer.WriteField<idx_t>(sample_chunks);
 	writer.WriteField<bool>(auto_detect);
 	writer.WriteString(file_path);
-	writer.WriteField<bool>(include_file_name);
-	writer.WriteField<bool>(include_parsed_hive_partitions);
 	writer.WriteString(decimal_separator);
+	writer.WriteField<bool>(null_padding);
+	writer.WriteSerializable(file_options);
 	// write options
 	writer.WriteListNoReference<bool>(force_quote);
 }
@@ -912,9 +965,11 @@ void BufferedCSVReaderOptions::Deserialize(FieldReader &reader) {
 	buffer_sample_size = reader.ReadRequired<idx_t>();
 	null_str = reader.ReadRequired<string>();
 	compression = reader.ReadRequired<FileCompressionType>();
+	new_line = reader.ReadRequired<NewLineIdentifier>();
+	allow_quoted_nulls = reader.ReadRequired<bool>();
 	// read options
-	names = reader.ReadRequiredList<string>();
 	skip_rows = reader.ReadRequired<idx_t>();
+	skip_rows_set = reader.ReadRequired<bool>();
 	maximum_line_size = reader.ReadRequired<idx_t>();
 	normalize_names = reader.ReadRequired<bool>();
 	force_not_null = reader.ReadRequiredList<bool>();
@@ -923,9 +978,9 @@ void BufferedCSVReaderOptions::Deserialize(FieldReader &reader) {
 	sample_chunks = reader.ReadRequired<idx_t>();
 	auto_detect = reader.ReadRequired<bool>();
 	file_path = reader.ReadRequired<string>();
-	include_file_name = reader.ReadRequired<bool>();
-	include_parsed_hive_partitions = reader.ReadRequired<bool>();
 	decimal_separator = reader.ReadRequired<string>();
+	null_padding = reader.ReadRequired<bool>();
+	file_options = reader.ReadRequiredSerializable<MultiFileReaderOptions, MultiFileReaderOptions>();
 	// write options
 	force_quote = reader.ReadRequiredList<bool>();
 }
@@ -933,67 +988,65 @@ void BufferedCSVReaderOptions::Deserialize(FieldReader &reader) {
 static void CSVReaderSerialize(FieldWriter &writer, const FunctionData *bind_data_p, const TableFunction &function) {
 	auto &bind_data = (ReadCSVData &)*bind_data_p;
 	writer.WriteList<string>(bind_data.files);
-	writer.WriteRegularSerializableList<LogicalType>(bind_data.sql_types);
+	writer.WriteRegularSerializableList<LogicalType>(bind_data.csv_types);
+	writer.WriteList<string>(bind_data.csv_names);
+	writer.WriteRegularSerializableList<LogicalType>(bind_data.return_types);
+	writer.WriteList<string>(bind_data.return_names);
 	writer.WriteField<idx_t>(bind_data.filename_col_idx);
 	writer.WriteField<idx_t>(bind_data.hive_partition_col_idx);
 	bind_data.options.Serialize(writer);
 	writer.WriteField<bool>(bind_data.single_threaded);
+	writer.WriteSerializable(bind_data.reader_bind);
+	writer.WriteField<uint32_t>(bind_data.column_info.size());
+	for (auto &col : bind_data.column_info) {
+		col.Serialize(writer);
+	}
 }
 
 static unique_ptr<FunctionData> CSVReaderDeserialize(ClientContext &context, FieldReader &reader,
                                                      TableFunction &function) {
-	auto result_data = make_unique<ReadCSVData>();
+	auto result_data = make_uniq<ReadCSVData>();
 	result_data->files = reader.ReadRequiredList<string>();
-	result_data->sql_types = reader.ReadRequiredSerializableList<LogicalType, LogicalType>();
+	result_data->csv_types = reader.ReadRequiredSerializableList<LogicalType, LogicalType>();
+	result_data->csv_names = reader.ReadRequiredList<string>();
+	result_data->return_types = reader.ReadRequiredSerializableList<LogicalType, LogicalType>();
+	result_data->return_names = reader.ReadRequiredList<string>();
 	result_data->filename_col_idx = reader.ReadRequired<idx_t>();
 	result_data->hive_partition_col_idx = reader.ReadRequired<idx_t>();
 	result_data->options.Deserialize(reader);
 	result_data->single_threaded = reader.ReadField<bool>(true);
+	result_data->reader_bind = reader.ReadRequiredSerializable<MultiFileReaderBindData, MultiFileReaderBindData>();
+	uint32_t file_number = reader.ReadRequired<uint32_t>();
+	for (idx_t i = 0; i < file_number; i++) {
+		result_data->column_info.emplace_back(ColumnInfo::Deserialize(reader));
+	}
 	return std::move(result_data);
 }
 
-TableFunction ReadCSVTableFunction::GetFunction(bool list_parameter) {
-	auto parameter = list_parameter ? LogicalType::LIST(LogicalType::VARCHAR) : LogicalType::VARCHAR;
-	TableFunction read_csv("read_csv", {parameter}, ReadCSVFunction, ReadCSVBind, ReadCSVInitGlobal, ReadCSVInitLocal);
+TableFunction ReadCSVTableFunction::GetFunction() {
+	TableFunction read_csv("read_csv", {LogicalType::VARCHAR}, ReadCSVFunction, ReadCSVBind, ReadCSVInitGlobal,
+	                       ReadCSVInitLocal);
 	read_csv.table_scan_progress = CSVReaderProgress;
 	read_csv.pushdown_complex_filter = CSVComplexFilterPushdown;
 	read_csv.serialize = CSVReaderSerialize;
 	read_csv.deserialize = CSVReaderDeserialize;
 	read_csv.get_batch_index = CSVReaderGetBatchIndex;
 	read_csv.cardinality = CSVReaderCardinality;
+	read_csv.projection_pushdown = true;
 	ReadCSVAddNamedParameters(read_csv);
 	return read_csv;
 }
 
-TableFunction ReadCSVTableFunction::GetAutoFunction(bool list_parameter) {
-	auto parameter = list_parameter ? LogicalType::LIST(LogicalType::VARCHAR) : LogicalType::VARCHAR;
-	TableFunction read_csv_auto("read_csv_auto", {parameter}, ReadCSVFunction, ReadCSVAutoBind, ReadCSVInitGlobal,
-	                            ReadCSVInitLocal);
-	read_csv_auto.table_scan_progress = CSVReaderProgress;
-	read_csv_auto.pushdown_complex_filter = CSVComplexFilterPushdown;
-	read_csv_auto.serialize = CSVReaderSerialize;
-	read_csv_auto.deserialize = CSVReaderDeserialize;
-	read_csv_auto.get_batch_index = CSVReaderGetBatchIndex;
-	read_csv_auto.cardinality = CSVReaderCardinality;
-	ReadCSVAddNamedParameters(read_csv_auto);
-	read_csv_auto.named_parameters["column_types"] = LogicalType::ANY;
-	read_csv_auto.named_parameters["dtypes"] = LogicalType::ANY;
-	read_csv_auto.named_parameters["types"] = LogicalType::ANY;
-	read_csv_auto.named_parameters["names"] = LogicalType::LIST(LogicalType::VARCHAR);
-	read_csv_auto.named_parameters["column_names"] = LogicalType::LIST(LogicalType::VARCHAR);
+TableFunction ReadCSVTableFunction::GetAutoFunction() {
+	auto read_csv_auto = ReadCSVTableFunction::GetFunction();
+	read_csv_auto.name = "read_csv_auto";
+	read_csv_auto.bind = ReadCSVAutoBind;
 	return read_csv_auto;
 }
 
 void ReadCSVTableFunction::RegisterFunction(BuiltinFunctions &set) {
-	TableFunctionSet read_csv("read_csv");
-	read_csv.AddFunction(ReadCSVTableFunction::GetFunction());
-	read_csv.AddFunction(ReadCSVTableFunction::GetFunction(true));
-	set.AddFunction(read_csv);
-
-	TableFunctionSet read_csv_auto("read_csv_auto");
-	read_csv_auto.AddFunction(ReadCSVTableFunction::GetAutoFunction());
-	read_csv_auto.AddFunction(ReadCSVTableFunction::GetAutoFunction(true));
-	set.AddFunction(read_csv_auto);
+	set.AddFunction(MultiFileReader::CreateFunctionSet(ReadCSVTableFunction::GetFunction()));
+	set.AddFunction(MultiFileReader::CreateFunctionSet(ReadCSVTableFunction::GetAutoFunction()));
 }
 
 unique_ptr<TableRef> ReadCSVReplacement(ClientContext &context, const string &table_name, ReplacementScanData *data) {
@@ -1008,10 +1061,10 @@ unique_ptr<TableRef> ReadCSVReplacement(ClientContext &context, const string &ta
 	    !StringUtil::EndsWith(lower_name, ".tsv") && !StringUtil::Contains(lower_name, ".tsv?")) {
 		return nullptr;
 	}
-	auto table_function = make_unique<TableFunctionRef>();
+	auto table_function = make_uniq<TableFunctionRef>();
 	vector<unique_ptr<ParsedExpression>> children;
-	children.push_back(make_unique<ConstantExpression>(Value(table_name)));
-	table_function->function = make_unique<FunctionExpression>("read_csv_auto", std::move(children));
+	children.push_back(make_uniq<ConstantExpression>(Value(table_name)));
+	table_function->function = make_uniq<FunctionExpression>("read_csv_auto", std::move(children));
 	return std::move(table_function);
 }