duckdb 1.1.2-dev4.0 → 1.1.2-dev6.0

package/src/duckdb/src/execution/operator/persistent/buffered_csv_reader.cpp

@@ -0,0 +1,1487 @@
+#include "duckdb/execution/operator/persistent/buffered_csv_reader.hpp"
+
+#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
+#include "duckdb/common/file_system.hpp"
+#include "duckdb/common/string_util.hpp"
+#include "duckdb/common/to_string.hpp"
+#include "duckdb/common/types/cast_helpers.hpp"
+#include "duckdb/common/vector_operations/unary_executor.hpp"
+#include "duckdb/common/vector_operations/vector_operations.hpp"
+#include "duckdb/function/scalar/strftime_format.hpp"
+#include "duckdb/main/database.hpp"
+#include "duckdb/parser/column_definition.hpp"
+#include "duckdb/storage/data_table.hpp"
+#include "utf8proc_wrapper.hpp"
+#include "utf8proc.hpp"
+#include "duckdb/parser/keyword_helper.hpp"
+#include "duckdb/main/error_manager.hpp"
+#include "duckdb/main/client_data.hpp"
+
+#include <algorithm>
+#include <cctype>
+#include <cstring>
+#include <fstream>
+
+namespace duckdb {
+
+BufferedCSVReader::BufferedCSVReader(ClientContext &context, BufferedCSVReaderOptions options_p,
+                                     const vector<LogicalType> &requested_types)
+    : BaseCSVReader(context, std::move(options_p), requested_types), buffer_size(0), position(0), start(0) {
+	file_handle = OpenCSV(options);
+	Initialize(requested_types);
+}
+
+BufferedCSVReader::BufferedCSVReader(ClientContext &context, string filename, BufferedCSVReaderOptions options_p,
+                                     const vector<LogicalType> &requested_types)
+    : BaseCSVReader(context, std::move(options_p), requested_types), buffer_size(0), position(0), start(0) {
+	options.file_path = std::move(filename);
+	file_handle = OpenCSV(options);
+	Initialize(requested_types);
+}
+
+enum class QuoteRule : uint8_t { QUOTES_RFC = 0, QUOTES_OTHER = 1, NO_QUOTES = 2 };
+
+static bool StartsWithNumericDate(string &separator, const string &value) {
+	auto begin = value.c_str();
+	auto end = begin + value.size();
+
+	//	StrpTimeFormat::Parse will skip whitespace, so we can too
+	auto field1 = std::find_if_not(begin, end, StringUtil::CharacterIsSpace);
+	if (field1 == end) {
+		return false;
+	}
+
+	//	first numeric field must start immediately
+	if (!StringUtil::CharacterIsDigit(*field1)) {
+		return false;
+	}
+	auto literal1 = std::find_if_not(field1, end, StringUtil::CharacterIsDigit);
+	if (literal1 == end) {
+		return false;
+	}
+
+	//	second numeric field must exist
+	auto field2 = std::find_if(literal1, end, StringUtil::CharacterIsDigit);
+	if (field2 == end) {
+		return false;
+	}
+	auto literal2 = std::find_if_not(field2, end, StringUtil::CharacterIsDigit);
+	if (literal2 == end) {
+		return false;
+	}
+
+	//	third numeric field must exist
+	auto field3 = std::find_if(literal2, end, StringUtil::CharacterIsDigit);
+	if (field3 == end) {
+		return false;
+	}
+
+	//	second literal must match first
+	if (((field3 - literal2) != (field2 - literal1)) || strncmp(literal1, literal2, (field2 - literal1)) != 0) {
+		return false;
+	}
+
+	//	copy the literal as the separator, escaping percent signs
+	separator.clear();
+	while (literal1 < field2) {
+		const auto literal_char = *literal1++;
+		if (literal_char == '%') {
+			separator.push_back(literal_char);
+		}
+		separator.push_back(literal_char);
+	}
+
+	return true;
+}
+
+string GenerateDateFormat(const string &separator, const char *format_template) {
+	string format_specifier = format_template;
+	auto amount_of_dashes = std::count(format_specifier.begin(), format_specifier.end(), '-');
+	if (!amount_of_dashes) {
+		return format_specifier;
+	}
+	string result;
+	result.reserve(format_specifier.size() - amount_of_dashes + (amount_of_dashes * separator.size()));
+	for (auto &character : format_specifier) {
+		if (character == '-') {
+			result += separator;
+		} else {
+			result += character;
+		}
+	}
+	return result;
+}
+
+TextSearchShiftArray::TextSearchShiftArray() {
+}
+
+TextSearchShiftArray::TextSearchShiftArray(string search_term) : length(search_term.size()) {
+	if (length > 255) {
+		throw InvalidInputException("Size of delimiter/quote/escape in CSV reader is limited to 255 bytes");
+	}
+	// initialize the shifts array
+	shifts = unique_ptr<uint8_t[]>(new uint8_t[length * 255]);
+	memset(shifts.get(), 0, length * 255 * sizeof(uint8_t));
+	// iterate over each of the characters in the array
+	for (idx_t main_idx = 0; main_idx < length; main_idx++) {
+		uint8_t current_char = (uint8_t)search_term[main_idx];
+		// now move over all the remaining positions
+		for (idx_t i = main_idx; i < length; i++) {
+			bool is_match = true;
+			// check if the prefix matches at this position
+			// if it does, we move to this position after encountering the current character
+			for (idx_t j = 0; j < main_idx; j++) {
+				if (search_term[i - main_idx + j] != search_term[j]) {
+					is_match = false;
+				}
+			}
+			if (!is_match) {
+				continue;
+			}
+			shifts[i * 255 + current_char] = main_idx + 1;
+		}
+	}
+}
+
+// Helper function to generate column names
+static string GenerateColumnName(const idx_t total_cols, const idx_t col_number, const string &prefix = "column") {
+	int max_digits = NumericHelper::UnsignedLength(total_cols - 1);
+	int digits = NumericHelper::UnsignedLength(col_number);
+	string leading_zeros = string(max_digits - digits, '0');
+	string value = to_string(col_number);
+	return string(prefix + leading_zeros + value);
+}
+
+// Helper function for UTF-8 aware space trimming
+static string TrimWhitespace(const string &col_name) {
+	utf8proc_int32_t codepoint;
+	auto str = reinterpret_cast<const utf8proc_uint8_t *>(col_name.c_str());
+	idx_t size = col_name.size();
+	// Find the first character that is not left trimmed
+	idx_t begin = 0;
+	while (begin < size) {
+		auto bytes = utf8proc_iterate(str + begin, size - begin, &codepoint);
+		D_ASSERT(bytes > 0);
+		if (utf8proc_category(codepoint) != UTF8PROC_CATEGORY_ZS) {
+			break;
+		}
+		begin += bytes;
+	}
+
+	// Find the last character that is not right trimmed
+	idx_t end;
+	end = begin;
+	for (auto next = begin; next < col_name.size();) {
+		auto bytes = utf8proc_iterate(str + next, size - next, &codepoint);
+		D_ASSERT(bytes > 0);
+		next += bytes;
+		if (utf8proc_category(codepoint) != UTF8PROC_CATEGORY_ZS) {
+			end = next;
+		}
+	}
+
+	// return the trimmed string
+	return col_name.substr(begin, end - begin);
+}
+
+static string NormalizeColumnName(const string &col_name) {
+	// normalize UTF8 characters to NFKD
+	auto nfkd = utf8proc_NFKD(reinterpret_cast<const utf8proc_uint8_t *>(col_name.c_str()), col_name.size());
+	const string col_name_nfkd = string(const_char_ptr_cast(nfkd), strlen(const_char_ptr_cast(nfkd)));
+	free(nfkd);
+
+	// only keep ASCII characters 0-9 a-z A-Z and replace spaces with regular whitespace
+	string col_name_ascii = "";
+	for (idx_t i = 0; i < col_name_nfkd.size(); i++) {
+		if (col_name_nfkd[i] == '_' || (col_name_nfkd[i] >= '0' && col_name_nfkd[i] <= '9') ||
+		    (col_name_nfkd[i] >= 'A' && col_name_nfkd[i] <= 'Z') ||
+		    (col_name_nfkd[i] >= 'a' && col_name_nfkd[i] <= 'z')) {
+			col_name_ascii += col_name_nfkd[i];
+		} else if (StringUtil::CharacterIsSpace(col_name_nfkd[i])) {
+			col_name_ascii += " ";
+		}
+	}
+
+	// trim whitespace and replace remaining whitespace by _
+	string col_name_trimmed = TrimWhitespace(col_name_ascii);
+	string col_name_cleaned = "";
+	bool in_whitespace = false;
+	for (idx_t i = 0; i < col_name_trimmed.size(); i++) {
+		if (col_name_trimmed[i] == ' ') {
+			if (!in_whitespace) {
+				col_name_cleaned += "_";
+				in_whitespace = true;
+			}
+		} else {
+			col_name_cleaned += col_name_trimmed[i];
+			in_whitespace = false;
+		}
+	}
+
+	// don't leave string empty; if not empty, make lowercase
+	if (col_name_cleaned.empty()) {
+		col_name_cleaned = "_";
+	} else {
+		col_name_cleaned = StringUtil::Lower(col_name_cleaned);
+	}
+
+	// prepend _ if name starts with a digit or is a reserved keyword
+	if (KeywordHelper::IsKeyword(col_name_cleaned) || (col_name_cleaned[0] >= '0' && col_name_cleaned[0] <= '9')) {
+		col_name_cleaned = "_" + col_name_cleaned;
+	}
+	return col_name_cleaned;
+}
+
+void BufferedCSVReader::Initialize(const vector<LogicalType> &requested_types) {
+	PrepareComplexParser();
+	if (options.auto_detect) {
+		return_types = SniffCSV(requested_types);
+		if (return_types.empty()) {
+			throw InvalidInputException("Failed to detect column types from CSV: is the file a valid CSV file?");
+		}
+		JumpToBeginning(options.skip_rows, options.header);
+	} else {
+		return_types = requested_types;
+		ResetBuffer();
+		SkipRowsAndReadHeader(options.skip_rows, options.header);
+	}
+	InitParseChunk(return_types.size());
+}
+
+void BufferedCSVReader::ResetBuffer() {
+	buffer.reset();
+	buffer_size = 0;
+	position = 0;
+	start = 0;
+	cached_buffers.clear();
+}
+
+void BufferedCSVReader::ResetStream() {
+	file_handle->Reset();
+	linenr = 0;
+	linenr_estimated = false;
+	bytes_per_line_avg = 0;
+	sample_chunk_idx = 0;
+	jumping_samples = false;
+}
+
+void BufferedCSVReader::JumpToBeginning(idx_t skip_rows = 0, bool skip_header = false) {
+	ResetBuffer();
+	ResetStream();
+	sample_chunk_idx = 0;
+	bytes_in_chunk = 0;
+	end_of_file_reached = false;
+	bom_checked = false;
+	SkipRowsAndReadHeader(skip_rows, skip_header);
+}
+
+void BufferedCSVReader::SkipRowsAndReadHeader(idx_t skip_rows, bool skip_header) {
+	for (idx_t i = 0; i < skip_rows; i++) {
+		// ignore skip rows
+		string read_line = file_handle->ReadLine();
+		linenr++;
+	}
+
+	if (skip_header) {
+		// ignore the first line as a header line
+		InitParseChunk(return_types.size());
+		ParseCSV(ParserMode::PARSING_HEADER);
+	}
+}
+
+void BufferedCSVReader::PrepareComplexParser() {
+	delimiter_search = TextSearchShiftArray(options.delimiter);
+	escape_search = TextSearchShiftArray(options.escape);
+	quote_search = TextSearchShiftArray(options.quote);
+}
+
+bool BufferedCSVReader::JumpToNextSample() {
+	// get bytes contained in the previously read chunk
+	idx_t remaining_bytes_in_buffer = buffer_size - start;
+	bytes_in_chunk -= remaining_bytes_in_buffer;
+	if (remaining_bytes_in_buffer == 0) {
+		return false;
+	}
+
+	// assess if it makes sense to jump, based on size of the first chunk relative to size of the entire file
+	if (sample_chunk_idx == 0) {
+		idx_t bytes_first_chunk = bytes_in_chunk;
+		double chunks_fit = (file_handle->FileSize() / (double)bytes_first_chunk);
+		jumping_samples = chunks_fit >= options.sample_chunks;
+
+		// jump back to the beginning
+		JumpToBeginning(options.skip_rows, options.header);
+		sample_chunk_idx++;
+		return true;
+	}
+
+	if (end_of_file_reached || sample_chunk_idx >= options.sample_chunks) {
+		return false;
+	}
+
+	// if we deal with any other sources than plaintext files, jumping_samples can be tricky. In that case
+	// we just read x continuous chunks from the stream TODO: make jumps possible for zipfiles.
+	if (!file_handle->OnDiskFile() || !jumping_samples) {
+		sample_chunk_idx++;
+		return true;
+	}
+
+	// update average bytes per line
+	double bytes_per_line = bytes_in_chunk / (double)options.sample_chunk_size;
+	bytes_per_line_avg = ((bytes_per_line_avg * (sample_chunk_idx)) + bytes_per_line) / (sample_chunk_idx + 1);
+
+	// if none of the previous conditions were met, we can jump
+	idx_t partition_size = (idx_t)round(file_handle->FileSize() / (double)options.sample_chunks);
+
+	// calculate offset to end of the current partition
+	int64_t offset = partition_size - bytes_in_chunk - remaining_bytes_in_buffer;
+	auto current_pos = file_handle->SeekPosition();
+
+	if (current_pos + offset < file_handle->FileSize()) {
+		// set position in stream and clear failure bits
+		file_handle->Seek(current_pos + offset);
+
+		// estimate linenr
+		linenr += (idx_t)round((offset + remaining_bytes_in_buffer) / bytes_per_line_avg);
+		linenr_estimated = true;
+	} else {
+		// seek backwards from the end in last chunk and hope to catch the end of the file
+		// TODO: actually it would be good to make sure that the end of file is being reached, because
+		// messy end-lines are quite common. For this case, however, we first need a skip_end detection anyways.
+		file_handle->Seek(file_handle->FileSize() - bytes_in_chunk);
+
+		// estimate linenr
+		linenr = (idx_t)round((file_handle->FileSize() - bytes_in_chunk) / bytes_per_line_avg);
+		linenr_estimated = true;
+	}
+
+	// reset buffers and parse chunk
+	ResetBuffer();
+
+	// seek beginning of next line
+	// FIXME: if this jump ends up in a quoted linebreak, we will have a problem
+	string read_line = file_handle->ReadLine();
+	linenr++;
+
+	sample_chunk_idx++;
+
+	return true;
+}
+
+void BufferedCSVReader::DetectDialect(const vector<LogicalType> &requested_types,
+                                      BufferedCSVReaderOptions &original_options,
+                                      vector<BufferedCSVReaderOptions> &info_candidates, idx_t &best_num_cols) {
+	// set up the candidates we consider for delimiter and quote rules based on user input
+	vector<string> delim_candidates;
+	vector<QuoteRule> quoterule_candidates;
+	vector<vector<string>> quote_candidates_map;
+	vector<vector<string>> escape_candidates_map = {{""}, {"\\"}, {""}};
+
+	if (options.has_delimiter) {
+		// user provided a delimiter: use that delimiter
+		delim_candidates = {options.delimiter};
+	} else {
+		// no delimiter provided: try standard/common delimiters
+		delim_candidates = {",", "|", ";", "\t"};
+	}
+	if (options.has_quote) {
+		// user provided quote: use that quote rule
+		quote_candidates_map = {{options.quote}, {options.quote}, {options.quote}};
+	} else {
+		// no quote rule provided: use standard/common quotes
+		quote_candidates_map = {{"\""}, {"\"", "'"}, {""}};
+	}
+	if (options.has_escape) {
+		// user provided escape: use that escape rule
+		if (options.escape.empty()) {
+			quoterule_candidates = {QuoteRule::QUOTES_RFC};
+		} else {
+			quoterule_candidates = {QuoteRule::QUOTES_OTHER};
+		}
+		escape_candidates_map[static_cast<uint8_t>(quoterule_candidates[0])] = {options.escape};
+	} else {
+		// no escape provided: try standard/common escapes
+		quoterule_candidates = {QuoteRule::QUOTES_RFC, QuoteRule::QUOTES_OTHER, QuoteRule::NO_QUOTES};
+	}
+
+	idx_t best_consistent_rows = 0;
+	idx_t prev_padding_count = 0;
+	for (auto quoterule : quoterule_candidates) {
+		const auto &quote_candidates = quote_candidates_map[static_cast<uint8_t>(quoterule)];
+		for (const auto &quote : quote_candidates) {
+			for (const auto &delim : delim_candidates) {
+				const auto &escape_candidates = escape_candidates_map[static_cast<uint8_t>(quoterule)];
+				for (const auto &escape : escape_candidates) {
+					BufferedCSVReaderOptions sniff_info = original_options;
+					sniff_info.delimiter = delim;
+					sniff_info.quote = quote;
+					sniff_info.escape = escape;
+
+					options = sniff_info;
+					PrepareComplexParser();
+
+					JumpToBeginning(original_options.skip_rows);
+					sniffed_column_counts.clear();
+					if (!TryParseCSV(ParserMode::SNIFFING_DIALECT)) {
+						continue;
+					}
+
+					idx_t start_row = original_options.skip_rows;
+					idx_t consistent_rows = 0;
+					idx_t num_cols = sniffed_column_counts.empty() ? 0 : sniffed_column_counts[0];
+					idx_t padding_count = 0;
+					bool allow_padding = original_options.null_padding;
+					for (idx_t row = 0; row < sniffed_column_counts.size(); row++) {
+						if (sniffed_column_counts[row] == num_cols) {
+							consistent_rows++;
+						} else if (num_cols < sniffed_column_counts[row] && !original_options.skip_rows_set) {
+							// we use the maximum amount of num_cols that we find
+							num_cols = sniffed_column_counts[row];
+							start_row = row + original_options.skip_rows;
+							consistent_rows = 1;
+							padding_count = 0;
+						} else if (num_cols >= sniffed_column_counts[row] && allow_padding) {
+							// we are missing some columns, we can parse this as long as we add padding
+							padding_count++;
+						}
+					}
+
+					// some logic
+					consistent_rows += padding_count;
+					bool more_values = (consistent_rows > best_consistent_rows && num_cols >= best_num_cols);
+					bool require_more_padding = padding_count > prev_padding_count;
+					bool require_less_padding = padding_count < prev_padding_count;
+					bool single_column_before = best_num_cols < 2 && num_cols > best_num_cols;
+					bool rows_consistent =
+					    start_row + consistent_rows - original_options.skip_rows == sniffed_column_counts.size();
+					bool more_than_one_row = (consistent_rows > 1);
+					bool more_than_one_column = (num_cols > 1);
+					bool start_good = !info_candidates.empty() && (start_row <= info_candidates.front().skip_rows);
+
+					if (!requested_types.empty() && requested_types.size() != num_cols) {
+						continue;
+					} else if (rows_consistent && (single_column_before || (more_values && !require_more_padding) ||
+					                               (more_than_one_column && require_less_padding))) {
+						sniff_info.skip_rows = start_row;
+						sniff_info.num_cols = num_cols;
+						sniff_info.new_line = options.new_line;
+						best_consistent_rows = consistent_rows;
+						best_num_cols = num_cols;
+						prev_padding_count = padding_count;
+
+						info_candidates.clear();
+						info_candidates.push_back(sniff_info);
+					} else if (more_than_one_row && more_than_one_column && start_good && rows_consistent &&
+					           !require_more_padding) {
+						bool same_quote_is_candidate = false;
+						for (auto &info_candidate : info_candidates) {
+							if (quote.compare(info_candidate.quote) == 0) {
+								same_quote_is_candidate = true;
+							}
+						}
+						if (!same_quote_is_candidate) {
+							sniff_info.skip_rows = start_row;
+							sniff_info.num_cols = num_cols;
+							sniff_info.new_line = options.new_line;
+							info_candidates.push_back(sniff_info);
+						}
+					}
+				}
+			}
+		}
+	}
+}
+
+void BufferedCSVReader::DetectCandidateTypes(const vector<LogicalType> &type_candidates,
+                                             const map<LogicalTypeId, vector<const char *>> &format_template_candidates,
+                                             const vector<BufferedCSVReaderOptions> &info_candidates,
+                                             BufferedCSVReaderOptions &original_options, idx_t best_num_cols,
+                                             vector<vector<LogicalType>> &best_sql_types_candidates,
+                                             std::map<LogicalTypeId, vector<string>> &best_format_candidates,
+                                             DataChunk &best_header_row) {
+	BufferedCSVReaderOptions best_options;
+	idx_t min_varchar_cols = best_num_cols + 1;
+
+	// check which info candidate leads to minimum amount of non-varchar columns...
+	for (const auto &t : format_template_candidates) {
+		best_format_candidates[t.first].clear();
+	}
+	for (auto &info_candidate : info_candidates) {
+		options = info_candidate;
+		vector<vector<LogicalType>> info_sql_types_candidates(options.num_cols, type_candidates);
+		std::map<LogicalTypeId, bool> has_format_candidates;
+		std::map<LogicalTypeId, vector<string>> format_candidates;
+		for (const auto &t : format_template_candidates) {
+			has_format_candidates[t.first] = false;
+			format_candidates[t.first].clear();
+		}
+
+		// set all return_types to VARCHAR so we can do datatype detection based on VARCHAR values
+		return_types.clear();
+		return_types.assign(options.num_cols, LogicalType::VARCHAR);
+
+		// jump to beginning and skip potential header
+		JumpToBeginning(options.skip_rows, true);
+		DataChunk header_row;
+		header_row.Initialize(allocator, return_types);
+		parse_chunk.Copy(header_row);
+
+		if (header_row.size() == 0) {
+			continue;
+		}
+
+		// init parse chunk and read csv with info candidate
+		InitParseChunk(return_types.size());
+		if (!TryParseCSV(ParserMode::SNIFFING_DATATYPES)) {
+			continue;
+		}
+		for (idx_t row_idx = 0; row_idx <= parse_chunk.size(); row_idx++) {
+			bool is_header_row = row_idx == 0;
+			idx_t row = row_idx - 1;
+			for (idx_t col = 0; col < parse_chunk.ColumnCount(); col++) {
+				auto &col_type_candidates = info_sql_types_candidates[col];
+				while (col_type_candidates.size() > 1) {
+					const auto &sql_type = col_type_candidates.back();
+					// try cast from string to sql_type
+					Value dummy_val;
+					if (is_header_row) {
+						VerifyUTF8(col, 0, header_row, -int64_t(parse_chunk.size()));
+						dummy_val = header_row.GetValue(col, 0);
+					} else {
+						VerifyUTF8(col, row, parse_chunk);
+						dummy_val = parse_chunk.GetValue(col, row);
+					}
+					// try formatting for date types if the user did not specify one and it starts with numeric values.
+					string separator;
+					if (has_format_candidates.count(sql_type.id()) && !original_options.has_format[sql_type.id()] &&
+					    !dummy_val.IsNull() && StartsWithNumericDate(separator, StringValue::Get(dummy_val))) {
+						// generate date format candidates the first time through
+						auto &type_format_candidates = format_candidates[sql_type.id()];
+						const auto had_format_candidates = has_format_candidates[sql_type.id()];
+						if (!has_format_candidates[sql_type.id()]) {
+							has_format_candidates[sql_type.id()] = true;
+							// order by preference
+							auto entry = format_template_candidates.find(sql_type.id());
+							if (entry != format_template_candidates.end()) {
+								const auto &format_template_list = entry->second;
+								for (const auto &t : format_template_list) {
+									const auto format_string = GenerateDateFormat(separator, t);
+									// don't parse ISO 8601
+									if (format_string.find("%Y-%m-%d") == string::npos) {
+										type_format_candidates.emplace_back(format_string);
+									}
+								}
+							}
+							//	initialise the first candidate
+							options.has_format[sql_type.id()] = true;
+							//	all formats are constructed to be valid
+							SetDateFormat(type_format_candidates.back(), sql_type.id());
+						}
+						// check all formats and keep the first one that works
+						StrpTimeFormat::ParseResult result;
+						auto save_format_candidates = type_format_candidates;
+						while (!type_format_candidates.empty()) {
+							//	avoid using exceptions for flow control...
+							auto &current_format = options.date_format[sql_type.id()];
+							if (current_format.Parse(StringValue::Get(dummy_val), result)) {
+								break;
+							}
+							//	doesn't work - move to the next one
+							type_format_candidates.pop_back();
+							options.has_format[sql_type.id()] = (!type_format_candidates.empty());
+							if (!type_format_candidates.empty()) {
+								SetDateFormat(type_format_candidates.back(), sql_type.id());
+							}
+						}
+						//	if none match, then this is not a value of type sql_type,
+						if (type_format_candidates.empty()) {
+							//	so restore the candidates that did work.
+							//	or throw them out if they were generated by this value.
+							if (had_format_candidates) {
+								type_format_candidates.swap(save_format_candidates);
+								if (!type_format_candidates.empty()) {
+									SetDateFormat(type_format_candidates.back(), sql_type.id());
+								}
+							} else {
+								has_format_candidates[sql_type.id()] = false;
+							}
+						}
+					}
+					// try cast from string to sql_type
+					if (TryCastValue(dummy_val, sql_type)) {
+						break;
+					} else {
+						col_type_candidates.pop_back();
+					}
+				}
+			}
+			// reset type detection, because first row could be header,
+			// but only do it if csv has more than one line (including header)
+			if (parse_chunk.size() > 0 && is_header_row) {
+				info_sql_types_candidates = vector<vector<LogicalType>>(options.num_cols, type_candidates);
+				for (auto &f : format_candidates) {
+					f.second.clear();
+				}
+				for (auto &h : has_format_candidates) {
+					h.second = false;
+				}
+			}
+		}
+
+		idx_t varchar_cols = 0;
+		for (idx_t col = 0; col < parse_chunk.ColumnCount(); col++) {
+			auto &col_type_candidates = info_sql_types_candidates[col];
+			// check number of varchar columns
+			const auto &col_type = col_type_candidates.back();
+			if (col_type == LogicalType::VARCHAR) {
+				varchar_cols++;
+			}
+		}
+
+		// it's good if the dialect creates more non-varchar columns, but only if we sacrifice < 30% of best_num_cols.
+		if (varchar_cols < min_varchar_cols && parse_chunk.ColumnCount() > (best_num_cols * 0.7)) {
+			// we have a new best_options candidate
+			best_options = info_candidate;
+			min_varchar_cols = varchar_cols;
+			best_sql_types_candidates = info_sql_types_candidates;
+			best_format_candidates = format_candidates;
+			best_header_row.Destroy();
+			auto header_row_types = header_row.GetTypes();
+			best_header_row.Initialize(allocator, header_row_types);
+			header_row.Copy(best_header_row);
+		}
+	}
+
+	options = best_options;
+	for (const auto &best : best_format_candidates) {
+		if (!best.second.empty()) {
+			SetDateFormat(best.second.back(), best.first);
+		}
+	}
+}
+
+void BufferedCSVReader::DetectHeader(const vector<vector<LogicalType>> &best_sql_types_candidates,
+                                     const DataChunk &best_header_row) {
+	// information for header detection
+	bool first_row_consistent = true;
+	bool first_row_nulls = false;
+
+	// check if header row is all null and/or consistent with detected column data types
+	first_row_nulls = true;
+	for (idx_t col = 0; col < best_sql_types_candidates.size(); col++) {
+		auto dummy_val = best_header_row.GetValue(col, 0);
+		if (!dummy_val.IsNull()) {
+			first_row_nulls = false;
+		}
+
+		// try cast to sql_type of column
+		const auto &sql_type = best_sql_types_candidates[col].back();
+		if (!TryCastValue(dummy_val, sql_type)) {
+			first_row_consistent = false;
+		}
+	}
+
+	// update parser info, and read, generate & set col_names based on previous findings
+	if (((!first_row_consistent || first_row_nulls) && !options.has_header) || (options.has_header && options.header)) {
+		options.header = true;
+		case_insensitive_map_t<idx_t> name_collision_count;
+		// get header names from CSV
+		for (idx_t col = 0; col < options.num_cols; col++) {
+			const auto &val = best_header_row.GetValue(col, 0);
+			string col_name = val.ToString();
+
+			// generate name if field is empty
+			if (col_name.empty() || val.IsNull()) {
+				col_name = GenerateColumnName(options.num_cols, col);
+			}
+
+			// normalize names or at least trim whitespace
+			if (options.normalize_names) {
+				col_name = NormalizeColumnName(col_name);
+			} else {
+				col_name = TrimWhitespace(col_name);
+			}
+
+			// avoid duplicate header names
+			const string col_name_raw = col_name;
+			while (name_collision_count.find(col_name) != name_collision_count.end()) {
+				name_collision_count[col_name] += 1;
+				col_name = col_name + "_" + to_string(name_collision_count[col_name]);
+			}
+
+			names.push_back(col_name);
+			name_collision_count[col_name] = 0;
+		}
+
+	} else {
+		options.header = false;
+		for (idx_t col = 0; col < options.num_cols; col++) {
+			string column_name = GenerateColumnName(options.num_cols, col);
+			names.push_back(column_name);
+		}
+	}
+	for (idx_t i = 0; i < MinValue<idx_t>(names.size(), options.name_list.size()); i++) {
+		names[i] = options.name_list[i];
+	}
+}
+
+vector<LogicalType> BufferedCSVReader::RefineTypeDetection(const vector<LogicalType> &type_candidates,
+                                                           const vector<LogicalType> &requested_types,
+                                                           vector<vector<LogicalType>> &best_sql_types_candidates,
+                                                           map<LogicalTypeId, vector<string>> &best_format_candidates) {
+	// for the type refine we set the SQL types to VARCHAR for all columns
+	return_types.clear();
+	return_types.assign(options.num_cols, LogicalType::VARCHAR);
+
+	vector<LogicalType> detected_types;
+
+	// if data types were provided, exit here if number of columns does not match
+	if (!requested_types.empty()) {
+		if (requested_types.size() != options.num_cols) {
+			throw InvalidInputException(
+			    "Error while determining column types: found %lld columns but expected %d. (%s)", options.num_cols,
+			    requested_types.size(), options.ToString());
+		} else {
+			detected_types = requested_types;
+		}
+	} else if (options.all_varchar) {
+		// return all types varchar
+		detected_types = return_types;
+	} else {
+		// jump through the rest of the file and continue to refine the sql type guess
+		while (JumpToNextSample()) {
+			InitParseChunk(return_types.size());
+			// if jump ends up a bad line, we just skip this chunk
+			if (!TryParseCSV(ParserMode::SNIFFING_DATATYPES)) {
+				continue;
+			}
+			for (idx_t col = 0; col < parse_chunk.ColumnCount(); col++) {
+				vector<LogicalType> &col_type_candidates = best_sql_types_candidates[col];
+				while (col_type_candidates.size() > 1) {
+					const auto &sql_type = col_type_candidates.back();
+					//	narrow down the date formats
+					if (best_format_candidates.count(sql_type.id())) {
+						auto &best_type_format_candidates = best_format_candidates[sql_type.id()];
+						auto save_format_candidates = best_type_format_candidates;
+						while (!best_type_format_candidates.empty()) {
+							if (TryCastVector(parse_chunk.data[col], parse_chunk.size(), sql_type)) {
+								break;
+							}
+							//	doesn't work - move to the next one
+							best_type_format_candidates.pop_back();
+							options.has_format[sql_type.id()] = (!best_type_format_candidates.empty());
+							if (!best_type_format_candidates.empty()) {
+								SetDateFormat(best_type_format_candidates.back(), sql_type.id());
+							}
+						}
+						//	if none match, then this is not a column of type sql_type,
+						if (best_type_format_candidates.empty()) {
+							//	so restore the candidates that did work.
+							best_type_format_candidates.swap(save_format_candidates);
+							if (!best_type_format_candidates.empty()) {
+								SetDateFormat(best_type_format_candidates.back(), sql_type.id());
+							}
+						}
+					}
+
+					if (TryCastVector(parse_chunk.data[col], parse_chunk.size(), sql_type)) {
+						break;
+					} else {
+						col_type_candidates.pop_back();
+					}
+				}
+			}
+		}
+
+		// set sql types
+		for (auto &best_sql_types_candidate : best_sql_types_candidates) {
+			LogicalType d_type = best_sql_types_candidate.back();
+			if (best_sql_types_candidate.size() == type_candidates.size()) {
+				d_type = LogicalType::VARCHAR;
+			}
+			detected_types.push_back(d_type);
+		}
+	}
+
+	return detected_types;
+}
+
+string BufferedCSVReader::ColumnTypesError(case_insensitive_map_t<idx_t> sql_types_per_column,
+                                           const vector<string> &names) {
+	for (idx_t i = 0; i < names.size(); i++) {
+		auto it = sql_types_per_column.find(names[i]);
+		if (it != sql_types_per_column.end()) {
+			sql_types_per_column.erase(names[i]);
+			continue;
+		}
+	}
+	if (sql_types_per_column.empty()) {
+		return string();
+	}
+	string exception = "COLUMN_TYPES error: Columns with names: ";
+	for (auto &col : sql_types_per_column) {
+		exception += "\"" + col.first + "\",";
+	}
+	exception.pop_back();
+	exception += " do not exist in the CSV File";
+	return exception;
+}
+
+vector<LogicalType> BufferedCSVReader::SniffCSV(const vector<LogicalType> &requested_types) {
+	for (auto &type : requested_types) {
+		// auto detect for blobs not supported: there may be invalid UTF-8 in the file
+		if (type.id() == LogicalTypeId::BLOB) {
+			return requested_types;
+		}
+	}
+
+	// #######
+	// ### dialect detection
+	// #######
+	BufferedCSVReaderOptions original_options = options;
+	vector<BufferedCSVReaderOptions> info_candidates;
+	idx_t best_num_cols = 0;
+
+	DetectDialect(requested_types, original_options, info_candidates, best_num_cols);
+
+	// if no dialect candidate was found, then file was most likely empty and we throw an exception
+	if (info_candidates.empty()) {
+		throw InvalidInputException(
+		    "Error in file \"%s\": CSV options could not be auto-detected. Consider setting parser options manually.",
+		    options.file_path);
+	}
+
+	// #######
+	// ### type detection (initial)
+	// #######
+
+	// format template candidates, ordered by descending specificity (~ from high to low)
+	std::map<LogicalTypeId, vector<const char *>> format_template_candidates = {
+	    {LogicalTypeId::DATE, {"%m-%d-%Y", "%m-%d-%y", "%d-%m-%Y", "%d-%m-%y", "%Y-%m-%d", "%y-%m-%d"}},
+	    {LogicalTypeId::TIMESTAMP,
+	     {"%Y-%m-%d %H:%M:%S.%f", "%m-%d-%Y %I:%M:%S %p", "%m-%d-%y %I:%M:%S %p", "%d-%m-%Y %H:%M:%S",
+	      "%d-%m-%y %H:%M:%S", "%Y-%m-%d %H:%M:%S", "%y-%m-%d %H:%M:%S"}},
+	};
+	vector<vector<LogicalType>> best_sql_types_candidates;
+	map<LogicalTypeId, vector<string>> best_format_candidates;
+	DataChunk best_header_row;
+	DetectCandidateTypes(options.auto_type_candidates, format_template_candidates, info_candidates, original_options,
+	                     best_num_cols, best_sql_types_candidates, best_format_candidates, best_header_row);
+
+	if (best_format_candidates.empty() || best_header_row.size() == 0) {
+		throw InvalidInputException(
+		    "Error in file \"%s\": CSV options could not be auto-detected. Consider setting parser options manually.",
+		    original_options.file_path);
+	}
+
+	// #######
+	// ### header detection
+	// #######
+	options.num_cols = best_num_cols;
+	DetectHeader(best_sql_types_candidates, best_header_row);
+	if (!options.sql_type_list.empty()) {
+		// user-defined types were supplied for certain columns
+		// override the types
+		if (!options.sql_types_per_column.empty()) {
+			// types supplied as name -> value map
+			idx_t found = 0;
+			for (idx_t i = 0; i < names.size(); i++) {
+				auto it = options.sql_types_per_column.find(names[i]);
+				if (it != options.sql_types_per_column.end()) {
+					best_sql_types_candidates[i] = {options.sql_type_list[it->second]};
+					found++;
+					continue;
+				}
+			}
+			if (!options.file_options.union_by_name && found < options.sql_types_per_column.size()) {
+				string exception = ColumnTypesError(options.sql_types_per_column, names);
+				if (!exception.empty()) {
+					throw BinderException(exception);
+				}
+			}
+		} else {
+			// types supplied as list
+			if (names.size() < options.sql_type_list.size()) {
+				throw BinderException("read_csv: %d types were provided, but CSV file only has %d columns",
+				                      options.sql_type_list.size(), names.size());
+			}
+			for (idx_t i = 0; i < options.sql_type_list.size(); i++) {
+				best_sql_types_candidates[i] = {options.sql_type_list[i]};
+			}
+		}
+	}
+
+	// #######
+	// ### type detection (refining)
+	// #######
+	return RefineTypeDetection(options.auto_type_candidates, requested_types, best_sql_types_candidates,
+	                           best_format_candidates);
+}
+
+bool BufferedCSVReader::TryParseComplexCSV(DataChunk &insert_chunk, string &error_message) {
+	// used for parsing algorithm
+	bool finished_chunk = false;
+	idx_t column = 0;
+	vector<idx_t> escape_positions;
+	bool has_quotes = false;
+	uint8_t delimiter_pos = 0, escape_pos = 0, quote_pos = 0;
+	idx_t offset = 0;
+	idx_t line_start = 0;
+	// read values into the buffer (if any)
+	if (position >= buffer_size) {
+		if (!ReadBuffer(start, line_start)) {
+			return true;
+		}
+	}
+	// start parsing the first value
+	start = position;
+	goto value_start;
+value_start:
+	/* state: value_start */
+	// this state parses the first characters of a value
+	offset = 0;
+	delimiter_pos = 0;
+	quote_pos = 0;
+	do {
+		idx_t count = 0;
+		for (; position < buffer_size; position++) {
+			quote_search.Match(quote_pos, buffer[position]);
+			delimiter_search.Match(delimiter_pos, buffer[position]);
+			count++;
+			if (delimiter_pos == options.delimiter.size()) {
+				// found a delimiter, add the value
+				offset = options.delimiter.size() - 1;
+				goto add_value;
+			} else if (StringUtil::CharacterIsNewline(buffer[position])) {
+				// found a newline, add the row
+				goto add_row;
+			}
+			if (count > quote_pos) {
+				// did not find a quote directly at the start of the value, stop looking for the quote now
+				goto normal;
+			}
+			if (quote_pos == options.quote.size()) {
+				// found a quote, go to quoted loop and skip the initial quote
+				start += options.quote.size();
+				goto in_quotes;
+			}
+		}
+	} while (ReadBuffer(start, line_start));
+	// file ends while scanning for quote/delimiter, go to final state
+	goto final_state;
+normal:
+	/* state: normal parsing state */
+	// this state parses the remainder of a non-quoted value until we reach a delimiter or newline
+	position++;
+	do {
+		for (; position < buffer_size; position++) {
+			delimiter_search.Match(delimiter_pos, buffer[position]);
+			if (delimiter_pos == options.delimiter.size()) {
+				offset = options.delimiter.size() - 1;
+				goto add_value;
+			} else if (StringUtil::CharacterIsNewline(buffer[position])) {
+				goto add_row;
+			}
+		}
+	} while (ReadBuffer(start, line_start));
+	goto final_state;
+add_value:
+	AddValue(string_t(buffer.get() + start, position - start - offset), column, escape_positions, has_quotes);
+	// increase position by 1 and move start to the new position
+	offset = 0;
+	has_quotes = false;
+	start = ++position;
+	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
+		// file ends right after delimiter, go to final state
+		goto final_state;
+	}
+	goto value_start;
+add_row : {
+	// check type of newline (\r or \n)
+	bool carriage_return = buffer[position] == '\r';
+	AddValue(string_t(buffer.get() + start, position - start - offset), column, escape_positions, has_quotes);
+	finished_chunk = AddRow(insert_chunk, column, error_message);
+
+	if (!error_message.empty()) {
+		return false;
+	}
+	// increase position by 1 and move start to the new position
+	offset = 0;
+	has_quotes = false;
+	position++;
+	SkipEmptyLines();
+	start = position;
+	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
+		// file ends right after newline, go to final state
+		goto final_state;
+	}
+	if (carriage_return) {
+		// \r newline, go to special state that parses an optional \n afterwards
+		goto carriage_return;
+	} else {
+		// \n newline, move to value start
+		if (finished_chunk) {
+			return true;
+		}
+		goto value_start;
+	}
+}
+in_quotes:
+	/* state: in_quotes */
+	// this state parses the remainder of a quoted value
+	quote_pos = 0;
+	escape_pos = 0;
+	has_quotes = true;
+	position++;
+	do {
+		for (; position < buffer_size; position++) {
+			quote_search.Match(quote_pos, buffer[position]);
+			escape_search.Match(escape_pos, buffer[position]);
+			if (quote_pos == options.quote.size()) {
+				goto unquote;
+			} else if (escape_pos == options.escape.size()) {
+				escape_positions.push_back(position - start - (options.escape.size() - 1));
+				goto handle_escape;
+			}
+		}
+	} while (ReadBuffer(start, line_start));
+	// still in quoted state at the end of the file, error:
+	error_message = StringUtil::Format("Error in file \"%s\" on line %s: unterminated quotes. (%s)", options.file_path,
+	                                   GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
+	return false;
+unquote:
+	/* state: unquote */
+	// this state handles the state directly after we unquote
+	// in this state we expect either another quote (entering the quoted state again, and escaping the quote)
+	// or a delimiter/newline, ending the current value and moving on to the next value
+	delimiter_pos = 0;
+	quote_pos = 0;
+	position++;
+	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
+		// file ends right after unquote, go to final state
+		offset = options.quote.size();
+		goto final_state;
+	}
+	if (StringUtil::CharacterIsNewline(buffer[position])) {
+		// quote followed by newline, add row
+		offset = options.quote.size();
+		goto add_row;
+	}
+	do {
+		idx_t count = 0;
+		for (; position < buffer_size; position++) {
+			quote_search.Match(quote_pos, buffer[position]);
+			delimiter_search.Match(delimiter_pos, buffer[position]);
+			count++;
+			if (count > delimiter_pos && count > quote_pos) {
+				error_message = StringUtil::Format(
+				    "Error in file \"%s\" on line %s: quote should be followed by end of value, end "
+				    "of row or another quote. (%s)",
+				    options.file_path, GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
+				return false;
+			}
+			if (delimiter_pos == options.delimiter.size()) {
+				// quote followed by delimiter, add value
+				offset = options.quote.size() + options.delimiter.size() - 1;
+				goto add_value;
+			} else if (quote_pos == options.quote.size() &&
+			           (options.escape.empty() || options.escape == options.quote)) {
+				// quote followed by quote, go back to quoted state and add to escape
+				escape_positions.push_back(position - start - (options.quote.size() - 1));
+				goto in_quotes;
+			}
+		}
+	} while (ReadBuffer(start, line_start));
+	error_message = StringUtil::Format(
+	    "Error in file \"%s\" on line %s: quote should be followed by end of value, end of row or another quote. (%s)",
+	    options.file_path, GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
+	return false;
+handle_escape:
+	escape_pos = 0;
+	quote_pos = 0;
+	position++;
+	do {
+		idx_t count = 0;
+		for (; position < buffer_size; position++) {
+			quote_search.Match(quote_pos, buffer[position]);
+			escape_search.Match(escape_pos, buffer[position]);
+			count++;
+			if (count > escape_pos && count > quote_pos) {
+				error_message = StringUtil::Format(
+				    "Error in file \"%s\" on line %s: neither QUOTE nor ESCAPE is proceeded by ESCAPE. (%s)",
+				    options.file_path, GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
+				return false;
+			}
+			if (quote_pos == options.quote.size() || escape_pos == options.escape.size()) {
+				// found quote or escape: move back to quoted state
+				goto in_quotes;
+			}
+		}
+	} while (ReadBuffer(start, line_start));
+	error_message =
+	    StringUtil::Format("Error in file \"%s\" on line %s: neither QUOTE nor ESCAPE is proceeded by ESCAPE. (%s)",
+	                       options.file_path, GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
+	return false;
+carriage_return:
+	/* state: carriage_return */
+	// this stage optionally skips a newline (\n) character, which allows \r\n to be interpreted as a single line
+	if (buffer[position] == '\n') {
+		// newline after carriage return: skip
+		start = ++position;
+		if (position >= buffer_size && !ReadBuffer(start, line_start)) {
+			// file ends right after newline, go to final state
+			goto final_state;
+		}
+	}
+	if (finished_chunk) {
+		return true;
+	}
+	goto value_start;
+final_state:
+	if (finished_chunk) {
+		return true;
+	}
+	if (column > 0 || position > start) {
+		// remaining values to be added to the chunk
+		AddValue(string_t(buffer.get() + start, position - start - offset), column, escape_positions, has_quotes);
+		finished_chunk = AddRow(insert_chunk, column, error_message);
+		SkipEmptyLines();
+		if (!error_message.empty()) {
+			return false;
+		}
+	}
+	// final stage, only reached after parsing the file is finished
+	// flush the parsed chunk and finalize parsing
+	if (mode == ParserMode::PARSING) {
+		Flush(insert_chunk);
+	}
+
+	end_of_file_reached = true;
+	return true;
+}
+
+void BufferedCSVReader::SkipEmptyLines() {
+	if (parse_chunk.data.size() == 1) {
+		// Empty lines are null data.
+		return;
+	}
+	for (; position < buffer_size; position++) {
+		if (!StringUtil::CharacterIsNewline(buffer[position])) {
+			return;
+		}
+	}
+}
+
+void UpdateMaxLineLength(ClientContext &context, idx_t line_length) {
+	if (!context.client_data->debug_set_max_line_length) {
+		return;
+	}
+	if (line_length < context.client_data->debug_max_line_length) {
+		return;
+	}
+	context.client_data->debug_max_line_length = line_length;
+}
+
+bool BufferedCSVReader::TryParseSimpleCSV(DataChunk &insert_chunk, string &error_message) {
+	// used for parsing algorithm
+	bool finished_chunk = false;
+	idx_t column = 0;
+	idx_t offset = 0;
+	bool has_quotes = false;
+	vector<idx_t> escape_positions;
+
+	idx_t line_start = position;
+	// read values into the buffer (if any)
+	if (position >= buffer_size) {
+		if (!ReadBuffer(start, line_start)) {
+			return true;
+		}
+	}
+
+	// start parsing the first value
+	goto value_start;
+value_start:
+	offset = 0;
+	/* state: value_start */
+	// this state parses the first character of a value
+	if (buffer[position] == options.quote[0]) {
+		// quote: actual value starts in the next position
+		// move to in_quotes state
+		start = position + 1;
+		goto in_quotes;
+	} else {
+		// no quote, move to normal parsing state
+		start = position;
+		goto normal;
+	}
+normal:
+	/* state: normal parsing state */
+	// this state parses the remainder of a non-quoted value until we reach a delimiter or newline
+	do {
+		for (; position < buffer_size; position++) {
+			if (buffer[position] == options.delimiter[0]) {
+				// delimiter: end the value and add it to the chunk
+				goto add_value;
+			} else if (StringUtil::CharacterIsNewline(buffer[position])) {
+				// newline: add row
+				goto add_row;
+			}
+		}
+	} while (ReadBuffer(start, line_start));
+	// file ends during normal scan: go to end state
+	goto final_state;
+add_value:
+	AddValue(string_t(buffer.get() + start, position - start - offset), column, escape_positions, has_quotes);
+	// increase position by 1 and move start to the new position
+	offset = 0;
+	has_quotes = false;
+	start = ++position;
+	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
+		// file ends right after delimiter, go to final state
+		goto final_state;
+	}
+	goto value_start;
+add_row : {
+	// check type of newline (\r or \n)
+	bool carriage_return = buffer[position] == '\r';
+	AddValue(string_t(buffer.get() + start, position - start - offset), column, escape_positions, has_quotes);
+	if (!error_message.empty()) {
+		return false;
+	}
+	finished_chunk = AddRow(insert_chunk, column, error_message);
+	UpdateMaxLineLength(context, position - line_start);
+	if (!error_message.empty()) {
+		return false;
+	}
+	// increase position by 1 and move start to the new position
+	offset = 0;
+	has_quotes = false;
+	position++;
+	start = position;
+	line_start = position;
+	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
+		// file ends right after delimiter, go to final state
+		goto final_state;
+	}
+	if (carriage_return) {
+		// \r newline, go to special state that parses an optional \n afterwards
+		goto carriage_return;
+	} else {
+		SetNewLineDelimiter();
+		SkipEmptyLines();
+
+		start = position;
+		line_start = position;
+		if (position >= buffer_size && !ReadBuffer(start, line_start)) {
+			// file ends right after delimiter, go to final state
+			goto final_state;
+		}
+		// \n newline, move to value start
+		if (finished_chunk) {
+			return true;
+		}
+		goto value_start;
+	}
+}
+in_quotes:
+	/* state: in_quotes */
+	// this state parses the remainder of a quoted value
+	has_quotes = true;
+	position++;
+	do {
+		for (; position < buffer_size; position++) {
+			if (buffer[position] == options.quote[0]) {
+				// quote: move to unquoted state
+				goto unquote;
+			} else if (buffer[position] == options.escape[0]) {
+				// escape: store the escaped position and move to handle_escape state
+				escape_positions.push_back(position - start);
+				goto handle_escape;
+			}
+		}
+	} while (ReadBuffer(start, line_start));
+	// still in quoted state at the end of the file, error:
+	throw InvalidInputException("Error in file \"%s\" on line %s: unterminated quotes. (%s)", options.file_path,
+	                            GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
+unquote:
+	/* state: unquote */
+	// this state handles the state directly after we unquote
+	// in this state we expect either another quote (entering the quoted state again, and escaping the quote)
+	// or a delimiter/newline, ending the current value and moving on to the next value
+	position++;
+	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
+		// file ends right after unquote, go to final state
+		offset = 1;
+		goto final_state;
+	}
+	if (buffer[position] == options.quote[0] && (options.escape.empty() || options.escape[0] == options.quote[0])) {
+		// escaped quote, return to quoted state and store escape position
+		escape_positions.push_back(position - start);
+		goto in_quotes;
+	} else if (buffer[position] == options.delimiter[0]) {
+		// delimiter, add value
+		offset = 1;
+		goto add_value;
+	} else if (StringUtil::CharacterIsNewline(buffer[position])) {
+		offset = 1;
+		goto add_row;
+	} else {
+		error_message = StringUtil::Format(
+		    "Error in file \"%s\" on line %s: quote should be followed by end of value, end of "
+		    "row or another quote. (%s)",
+		    options.file_path, GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
+		return false;
+	}
+handle_escape:
+	/* state: handle_escape */
+	// escape should be followed by a quote or another escape character
+	position++;
+	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
+		error_message = StringUtil::Format(
+		    "Error in file \"%s\" on line %s: neither QUOTE nor ESCAPE is proceeded by ESCAPE. (%s)", options.file_path,
+		    GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
+		return false;
+	}
+	if (buffer[position] != options.quote[0] && buffer[position] != options.escape[0]) {
+		error_message = StringUtil::Format(
+		    "Error in file \"%s\" on line %s: neither QUOTE nor ESCAPE is proceeded by ESCAPE. (%s)", options.file_path,
+		    GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
+		return false;
+	}
+	// escape was followed by quote or escape, go back to quoted state
+	goto in_quotes;
+carriage_return:
+	/* state: carriage_return */
+	// this stage optionally skips a newline (\n) character, which allows \r\n to be interpreted as a single line
+	if (buffer[position] == '\n') {
+		SetNewLineDelimiter(true, true);
+		// newline after carriage return: skip
+		// increase position by 1 and move start to the new position
+		start = ++position;
+		if (position >= buffer_size && !ReadBuffer(start, line_start)) {
+			// file ends right after delimiter, go to final state
+			goto final_state;
+		}
+	} else {
+		SetNewLineDelimiter(true, false);
+	}
+	if (finished_chunk) {
+		return true;
+	}
+	SkipEmptyLines();
+	start = position;
+	line_start = position;
+	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
+		// file ends right after delimiter, go to final state
+		goto final_state;
+	}
+
+	goto value_start;
+final_state:
+	if (finished_chunk) {
+		return true;
+	}
+
+	if (column > 0 || position > start) {
+		// remaining values to be added to the chunk
+		AddValue(string_t(buffer.get() + start, position - start - offset), column, escape_positions, has_quotes);
+		finished_chunk = AddRow(insert_chunk, column, error_message);
+		SkipEmptyLines();
+		UpdateMaxLineLength(context, position - line_start);
+		if (!error_message.empty()) {
+			return false;
+		}
+	}
+
+	// final stage, only reached after parsing the file is finished
+	// flush the parsed chunk and finalize parsing
+	if (mode == ParserMode::PARSING) {
+		Flush(insert_chunk);
+	}
+
+	end_of_file_reached = true;
+	return true;
+}
+
+bool BufferedCSVReader::ReadBuffer(idx_t &start, idx_t &line_start) {
+	if (start > buffer_size) {
+		return false;
+	}
+	auto old_buffer = std::move(buffer);
+
+	// the remaining part of the last buffer
+	idx_t remaining = buffer_size - start;
+
+	bool large_buffers = mode == ParserMode::PARSING && !file_handle->OnDiskFile() && file_handle->CanSeek();
+	idx_t buffer_read_size = large_buffers ? INITIAL_BUFFER_SIZE_LARGE : INITIAL_BUFFER_SIZE;
+
+	while (remaining > buffer_read_size) {
+		buffer_read_size *= 2;
+	}
+
+	// Check line length
+	if (remaining > options.maximum_line_size) {
+		throw InvalidInputException("Maximum line size of %llu bytes exceeded on line %s!", options.maximum_line_size,
+		                            GetLineNumberStr(linenr, linenr_estimated));
+	}
+
+	buffer = make_unsafe_uniq_array<char>(buffer_read_size + remaining + 1);
+	buffer_size = remaining + buffer_read_size;
+	if (remaining > 0) {
+		// remaining from last buffer: copy it here
+		memcpy(buffer.get(), old_buffer.get() + start, remaining);
+	}
+	idx_t read_count = file_handle->Read(buffer.get() + remaining, buffer_read_size);
+
+	bytes_in_chunk += read_count;
+	buffer_size = remaining + read_count;
+	buffer[buffer_size] = '\0';
+	if (old_buffer) {
+		cached_buffers.push_back(std::move(old_buffer));
+	}
+	start = 0;
+	position = remaining;
+	if (!bom_checked) {
+		bom_checked = true;
+		if (read_count >= 3 && buffer[0] == '\xEF' && buffer[1] == '\xBB' && buffer[2] == '\xBF') {
+			start += 3;
+			position += 3;
+		}
+	}
+	line_start = start;
+
+	return read_count > 0;
+}
+
+void BufferedCSVReader::ParseCSV(DataChunk &insert_chunk) {
+	string error_message;
+	if (!TryParseCSV(ParserMode::PARSING, insert_chunk, error_message)) {
+		throw InvalidInputException(error_message);
+	}
+}
+
+bool BufferedCSVReader::TryParseCSV(ParserMode mode) {
+	DataChunk dummy_chunk;
+	string error_message;
+	return TryParseCSV(mode, dummy_chunk, error_message);
+}
+
+void BufferedCSVReader::ParseCSV(ParserMode mode) {
+	DataChunk dummy_chunk;
+	string error_message;
+	if (!TryParseCSV(mode, dummy_chunk, error_message)) {
+		throw InvalidInputException(error_message);
+	}
+}
+
+bool BufferedCSVReader::TryParseCSV(ParserMode parser_mode, DataChunk &insert_chunk, string &error_message) {
+	mode = parser_mode;
+
+	if (options.quote.size() <= 1 && options.escape.size() <= 1 && options.delimiter.size() == 1) {
+		return TryParseSimpleCSV(insert_chunk, error_message);
+	} else {
+		return TryParseComplexCSV(insert_chunk, error_message);
+	}
+}
+
+} // namespace duckdb