duckdb 0.8.2-dev77.0 → 0.9.0

package/src/duckdb/src/execution/operator/persistent/buffered_csv_reader.cpp

@@ -1,1487 +0,0 @@
-#include "duckdb/execution/operator/persistent/buffered_csv_reader.hpp"
-
-#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
-#include "duckdb/common/file_system.hpp"
-#include "duckdb/common/string_util.hpp"
-#include "duckdb/common/to_string.hpp"
-#include "duckdb/common/types/cast_helpers.hpp"
-#include "duckdb/common/vector_operations/unary_executor.hpp"
-#include "duckdb/common/vector_operations/vector_operations.hpp"
-#include "duckdb/function/scalar/strftime_format.hpp"
-#include "duckdb/main/database.hpp"
-#include "duckdb/parser/column_definition.hpp"
-#include "duckdb/storage/data_table.hpp"
-#include "utf8proc_wrapper.hpp"
-#include "utf8proc.hpp"
-#include "duckdb/parser/keyword_helper.hpp"
-#include "duckdb/main/error_manager.hpp"
-#include "duckdb/main/client_data.hpp"
-
-#include <algorithm>
-#include <cctype>
-#include <cstring>
-#include <fstream>
-
-namespace duckdb {
-
-BufferedCSVReader::BufferedCSVReader(ClientContext &context, BufferedCSVReaderOptions options_p,
-                                     const vector<LogicalType> &requested_types)
-    : BaseCSVReader(context, std::move(options_p), requested_types), buffer_size(0), position(0), start(0) {
-	file_handle = OpenCSV(options);
-	Initialize(requested_types);
-}
-
-BufferedCSVReader::BufferedCSVReader(ClientContext &context, string filename, BufferedCSVReaderOptions options_p,
-                                     const vector<LogicalType> &requested_types)
-    : BaseCSVReader(context, std::move(options_p), requested_types), buffer_size(0), position(0), start(0) {
-	options.file_path = std::move(filename);
-	file_handle = OpenCSV(options);
-	Initialize(requested_types);
-}
-
-enum class QuoteRule : uint8_t { QUOTES_RFC = 0, QUOTES_OTHER = 1, NO_QUOTES = 2 };
-
-static bool StartsWithNumericDate(string &separator, const string &value) {
-	auto begin = value.c_str();
-	auto end = begin + value.size();
-
-	//	StrpTimeFormat::Parse will skip whitespace, so we can too
-	auto field1 = std::find_if_not(begin, end, StringUtil::CharacterIsSpace);
-	if (field1 == end) {
-		return false;
-	}
-
-	//	first numeric field must start immediately
-	if (!StringUtil::CharacterIsDigit(*field1)) {
-		return false;
-	}
-	auto literal1 = std::find_if_not(field1, end, StringUtil::CharacterIsDigit);
-	if (literal1 == end) {
-		return false;
-	}
-
-	//	second numeric field must exist
-	auto field2 = std::find_if(literal1, end, StringUtil::CharacterIsDigit);
-	if (field2 == end) {
-		return false;
-	}
-	auto literal2 = std::find_if_not(field2, end, StringUtil::CharacterIsDigit);
-	if (literal2 == end) {
-		return false;
-	}
-
-	//	third numeric field must exist
-	auto field3 = std::find_if(literal2, end, StringUtil::CharacterIsDigit);
-	if (field3 == end) {
-		return false;
-	}
-
-	//	second literal must match first
-	if (((field3 - literal2) != (field2 - literal1)) || strncmp(literal1, literal2, (field2 - literal1)) != 0) {
-		return false;
-	}
-
-	//	copy the literal as the separator, escaping percent signs
-	separator.clear();
-	while (literal1 < field2) {
-		const auto literal_char = *literal1++;
-		if (literal_char == '%') {
-			separator.push_back(literal_char);
-		}
-		separator.push_back(literal_char);
-	}
-
-	return true;
-}
-
-string GenerateDateFormat(const string &separator, const char *format_template) {
-	string format_specifier = format_template;
-	auto amount_of_dashes = std::count(format_specifier.begin(), format_specifier.end(), '-');
-	if (!amount_of_dashes) {
-		return format_specifier;
-	}
-	string result;
-	result.reserve(format_specifier.size() - amount_of_dashes + (amount_of_dashes * separator.size()));
-	for (auto &character : format_specifier) {
-		if (character == '-') {
-			result += separator;
-		} else {
-			result += character;
-		}
-	}
-	return result;
-}
-
-TextSearchShiftArray::TextSearchShiftArray() {
-}
-
-TextSearchShiftArray::TextSearchShiftArray(string search_term) : length(search_term.size()) {
-	if (length > 255) {
-		throw InvalidInputException("Size of delimiter/quote/escape in CSV reader is limited to 255 bytes");
-	}
-	// initialize the shifts array
-	shifts = unique_ptr<uint8_t[]>(new uint8_t[length * 255]);
-	memset(shifts.get(), 0, length * 255 * sizeof(uint8_t));
-	// iterate over each of the characters in the array
-	for (idx_t main_idx = 0; main_idx < length; main_idx++) {
-		uint8_t current_char = (uint8_t)search_term[main_idx];
-		// now move over all the remaining positions
-		for (idx_t i = main_idx; i < length; i++) {
-			bool is_match = true;
-			// check if the prefix matches at this position
-			// if it does, we move to this position after encountering the current character
-			for (idx_t j = 0; j < main_idx; j++) {
-				if (search_term[i - main_idx + j] != search_term[j]) {
-					is_match = false;
-				}
-			}
-			if (!is_match) {
-				continue;
-			}
-			shifts[i * 255 + current_char] = main_idx + 1;
-		}
-	}
-}
-
-// Helper function to generate column names
-static string GenerateColumnName(const idx_t total_cols, const idx_t col_number, const string &prefix = "column") {
-	int max_digits = NumericHelper::UnsignedLength(total_cols - 1);
-	int digits = NumericHelper::UnsignedLength(col_number);
-	string leading_zeros = string(max_digits - digits, '0');
-	string value = to_string(col_number);
-	return string(prefix + leading_zeros + value);
-}
-
-// Helper function for UTF-8 aware space trimming
-static string TrimWhitespace(const string &col_name) {
-	utf8proc_int32_t codepoint;
-	auto str = reinterpret_cast<const utf8proc_uint8_t *>(col_name.c_str());
-	idx_t size = col_name.size();
-	// Find the first character that is not left trimmed
-	idx_t begin = 0;
-	while (begin < size) {
-		auto bytes = utf8proc_iterate(str + begin, size - begin, &codepoint);
-		D_ASSERT(bytes > 0);
-		if (utf8proc_category(codepoint) != UTF8PROC_CATEGORY_ZS) {
-			break;
-		}
-		begin += bytes;
-	}
-
-	// Find the last character that is not right trimmed
-	idx_t end;
-	end = begin;
-	for (auto next = begin; next < col_name.size();) {
-		auto bytes = utf8proc_iterate(str + next, size - next, &codepoint);
-		D_ASSERT(bytes > 0);
-		next += bytes;
-		if (utf8proc_category(codepoint) != UTF8PROC_CATEGORY_ZS) {
-			end = next;
-		}
-	}
-
-	// return the trimmed string
-	return col_name.substr(begin, end - begin);
-}
-
-static string NormalizeColumnName(const string &col_name) {
-	// normalize UTF8 characters to NFKD
-	auto nfkd = utf8proc_NFKD(reinterpret_cast<const utf8proc_uint8_t *>(col_name.c_str()), col_name.size());
-	const string col_name_nfkd = string(const_char_ptr_cast(nfkd), strlen(const_char_ptr_cast(nfkd)));
-	free(nfkd);
-
-	// only keep ASCII characters 0-9 a-z A-Z and replace spaces with regular whitespace
-	string col_name_ascii = "";
-	for (idx_t i = 0; i < col_name_nfkd.size(); i++) {
-		if (col_name_nfkd[i] == '_' || (col_name_nfkd[i] >= '0' && col_name_nfkd[i] <= '9') ||
-		    (col_name_nfkd[i] >= 'A' && col_name_nfkd[i] <= 'Z') ||
-		    (col_name_nfkd[i] >= 'a' && col_name_nfkd[i] <= 'z')) {
-			col_name_ascii += col_name_nfkd[i];
-		} else if (StringUtil::CharacterIsSpace(col_name_nfkd[i])) {
-			col_name_ascii += " ";
-		}
-	}
-
-	// trim whitespace and replace remaining whitespace by _
-	string col_name_trimmed = TrimWhitespace(col_name_ascii);
-	string col_name_cleaned = "";
-	bool in_whitespace = false;
-	for (idx_t i = 0; i < col_name_trimmed.size(); i++) {
-		if (col_name_trimmed[i] == ' ') {
-			if (!in_whitespace) {
-				col_name_cleaned += "_";
-				in_whitespace = true;
-			}
-		} else {
-			col_name_cleaned += col_name_trimmed[i];
-			in_whitespace = false;
-		}
-	}
-
-	// don't leave string empty; if not empty, make lowercase
-	if (col_name_cleaned.empty()) {
-		col_name_cleaned = "_";
-	} else {
-		col_name_cleaned = StringUtil::Lower(col_name_cleaned);
-	}
-
-	// prepend _ if name starts with a digit or is a reserved keyword
-	if (KeywordHelper::IsKeyword(col_name_cleaned) || (col_name_cleaned[0] >= '0' && col_name_cleaned[0] <= '9')) {
-		col_name_cleaned = "_" + col_name_cleaned;
-	}
-	return col_name_cleaned;
-}
-
-void BufferedCSVReader::Initialize(const vector<LogicalType> &requested_types) {
-	PrepareComplexParser();
-	if (options.auto_detect) {
-		return_types = SniffCSV(requested_types);
-		if (return_types.empty()) {
-			throw InvalidInputException("Failed to detect column types from CSV: is the file a valid CSV file?");
-		}
-		JumpToBeginning(options.skip_rows, options.header);
-	} else {
-		return_types = requested_types;
-		ResetBuffer();
-		SkipRowsAndReadHeader(options.skip_rows, options.header);
-	}
-	InitParseChunk(return_types.size());
-}
-
-void BufferedCSVReader::ResetBuffer() {
-	buffer.reset();
-	buffer_size = 0;
-	position = 0;
-	start = 0;
-	cached_buffers.clear();
-}
-
-void BufferedCSVReader::ResetStream() {
-	file_handle->Reset();
-	linenr = 0;
-	linenr_estimated = false;
-	bytes_per_line_avg = 0;
-	sample_chunk_idx = 0;
-	jumping_samples = false;
-}
-
-void BufferedCSVReader::JumpToBeginning(idx_t skip_rows = 0, bool skip_header = false) {
-	ResetBuffer();
-	ResetStream();
-	sample_chunk_idx = 0;
-	bytes_in_chunk = 0;
-	end_of_file_reached = false;
-	bom_checked = false;
-	SkipRowsAndReadHeader(skip_rows, skip_header);
-}
-
-void BufferedCSVReader::SkipRowsAndReadHeader(idx_t skip_rows, bool skip_header) {
-	for (idx_t i = 0; i < skip_rows; i++) {
-		// ignore skip rows
-		string read_line = file_handle->ReadLine();
-		linenr++;
-	}
-
-	if (skip_header) {
-		// ignore the first line as a header line
-		InitParseChunk(return_types.size());
-		ParseCSV(ParserMode::PARSING_HEADER);
-	}
-}
-
-void BufferedCSVReader::PrepareComplexParser() {
-	delimiter_search = TextSearchShiftArray(options.delimiter);
-	escape_search = TextSearchShiftArray(options.escape);
-	quote_search = TextSearchShiftArray(options.quote);
-}
-
-bool BufferedCSVReader::JumpToNextSample() {
-	// get bytes contained in the previously read chunk
-	idx_t remaining_bytes_in_buffer = buffer_size - start;
-	bytes_in_chunk -= remaining_bytes_in_buffer;
-	if (remaining_bytes_in_buffer == 0) {
-		return false;
-	}
-
-	// assess if it makes sense to jump, based on size of the first chunk relative to size of the entire file
-	if (sample_chunk_idx == 0) {
-		idx_t bytes_first_chunk = bytes_in_chunk;
-		double chunks_fit = (file_handle->FileSize() / (double)bytes_first_chunk);
-		jumping_samples = chunks_fit >= options.sample_chunks;
-
-		// jump back to the beginning
-		JumpToBeginning(options.skip_rows, options.header);
-		sample_chunk_idx++;
-		return true;
-	}
-
-	if (end_of_file_reached || sample_chunk_idx >= options.sample_chunks) {
-		return false;
-	}
-
-	// if we deal with any other sources than plaintext files, jumping_samples can be tricky. In that case
-	// we just read x continuous chunks from the stream TODO: make jumps possible for zipfiles.
-	if (!file_handle->OnDiskFile() || !jumping_samples) {
-		sample_chunk_idx++;
-		return true;
-	}
-
-	// update average bytes per line
-	double bytes_per_line = bytes_in_chunk / (double)options.sample_chunk_size;
-	bytes_per_line_avg = ((bytes_per_line_avg * (sample_chunk_idx)) + bytes_per_line) / (sample_chunk_idx + 1);
-
-	// if none of the previous conditions were met, we can jump
-	idx_t partition_size = (idx_t)round(file_handle->FileSize() / (double)options.sample_chunks);
-
-	// calculate offset to end of the current partition
-	int64_t offset = partition_size - bytes_in_chunk - remaining_bytes_in_buffer;
-	auto current_pos = file_handle->SeekPosition();
-
-	if (current_pos + offset < file_handle->FileSize()) {
-		// set position in stream and clear failure bits
-		file_handle->Seek(current_pos + offset);
-
-		// estimate linenr
-		linenr += (idx_t)round((offset + remaining_bytes_in_buffer) / bytes_per_line_avg);
-		linenr_estimated = true;
-	} else {
-		// seek backwards from the end in last chunk and hope to catch the end of the file
-		// TODO: actually it would be good to make sure that the end of file is being reached, because
-		// messy end-lines are quite common. For this case, however, we first need a skip_end detection anyways.
-		file_handle->Seek(file_handle->FileSize() - bytes_in_chunk);
-
-		// estimate linenr
-		linenr = (idx_t)round((file_handle->FileSize() - bytes_in_chunk) / bytes_per_line_avg);
-		linenr_estimated = true;
-	}
-
-	// reset buffers and parse chunk
-	ResetBuffer();
-
-	// seek beginning of next line
-	// FIXME: if this jump ends up in a quoted linebreak, we will have a problem
-	string read_line = file_handle->ReadLine();
-	linenr++;
-
-	sample_chunk_idx++;
-
-	return true;
-}
-
-void BufferedCSVReader::DetectDialect(const vector<LogicalType> &requested_types,
-                                      BufferedCSVReaderOptions &original_options,
-                                      vector<BufferedCSVReaderOptions> &info_candidates, idx_t &best_num_cols) {
-	// set up the candidates we consider for delimiter and quote rules based on user input
-	vector<string> delim_candidates;
-	vector<QuoteRule> quoterule_candidates;
-	vector<vector<string>> quote_candidates_map;
-	vector<vector<string>> escape_candidates_map = {{""}, {"\\"}, {""}};
-
-	if (options.has_delimiter) {
-		// user provided a delimiter: use that delimiter
-		delim_candidates = {options.delimiter};
-	} else {
-		// no delimiter provided: try standard/common delimiters
-		delim_candidates = {",", "|", ";", "\t"};
-	}
-	if (options.has_quote) {
-		// user provided quote: use that quote rule
-		quote_candidates_map = {{options.quote}, {options.quote}, {options.quote}};
-	} else {
-		// no quote rule provided: use standard/common quotes
-		quote_candidates_map = {{"\""}, {"\"", "'"}, {""}};
-	}
-	if (options.has_escape) {
-		// user provided escape: use that escape rule
-		if (options.escape.empty()) {
-			quoterule_candidates = {QuoteRule::QUOTES_RFC};
-		} else {
-			quoterule_candidates = {QuoteRule::QUOTES_OTHER};
-		}
-		escape_candidates_map[static_cast<uint8_t>(quoterule_candidates[0])] = {options.escape};
-	} else {
-		// no escape provided: try standard/common escapes
-		quoterule_candidates = {QuoteRule::QUOTES_RFC, QuoteRule::QUOTES_OTHER, QuoteRule::NO_QUOTES};
-	}
-
-	idx_t best_consistent_rows = 0;
-	idx_t prev_padding_count = 0;
-	for (auto quoterule : quoterule_candidates) {
-		const auto &quote_candidates = quote_candidates_map[static_cast<uint8_t>(quoterule)];
-		for (const auto &quote : quote_candidates) {
-			for (const auto &delim : delim_candidates) {
-				const auto &escape_candidates = escape_candidates_map[static_cast<uint8_t>(quoterule)];
-				for (const auto &escape : escape_candidates) {
-					BufferedCSVReaderOptions sniff_info = original_options;
-					sniff_info.delimiter = delim;
-					sniff_info.quote = quote;
-					sniff_info.escape = escape;
-
-					options = sniff_info;
-					PrepareComplexParser();
-
-					JumpToBeginning(original_options.skip_rows);
-					sniffed_column_counts.clear();
-					if (!TryParseCSV(ParserMode::SNIFFING_DIALECT)) {
-						continue;
-					}
-
-					idx_t start_row = original_options.skip_rows;
-					idx_t consistent_rows = 0;
-					idx_t num_cols = sniffed_column_counts.empty() ? 0 : sniffed_column_counts[0];
-					idx_t padding_count = 0;
-					bool allow_padding = original_options.null_padding;
-					for (idx_t row = 0; row < sniffed_column_counts.size(); row++) {
-						if (sniffed_column_counts[row] == num_cols) {
-							consistent_rows++;
-						} else if (num_cols < sniffed_column_counts[row] && !original_options.skip_rows_set) {
-							// we use the maximum amount of num_cols that we find
-							num_cols = sniffed_column_counts[row];
-							start_row = row + original_options.skip_rows;
-							consistent_rows = 1;
-							padding_count = 0;
-						} else if (num_cols >= sniffed_column_counts[row] && allow_padding) {
-							// we are missing some columns, we can parse this as long as we add padding
-							padding_count++;
-						}
-					}
-
-					// some logic
-					consistent_rows += padding_count;
-					bool more_values = (consistent_rows > best_consistent_rows && num_cols >= best_num_cols);
-					bool require_more_padding = padding_count > prev_padding_count;
-					bool require_less_padding = padding_count < prev_padding_count;
-					bool single_column_before = best_num_cols < 2 && num_cols > best_num_cols;
-					bool rows_consistent =
-					    start_row + consistent_rows - original_options.skip_rows == sniffed_column_counts.size();
-					bool more_than_one_row = (consistent_rows > 1);
-					bool more_than_one_column = (num_cols > 1);
-					bool start_good = !info_candidates.empty() && (start_row <= info_candidates.front().skip_rows);
-
-					if (!requested_types.empty() && requested_types.size() != num_cols) {
-						continue;
-					} else if (rows_consistent && (single_column_before || (more_values && !require_more_padding) ||
-					                               (more_than_one_column && require_less_padding))) {
-						sniff_info.skip_rows = start_row;
-						sniff_info.num_cols = num_cols;
-						sniff_info.new_line = options.new_line;
-						best_consistent_rows = consistent_rows;
-						best_num_cols = num_cols;
-						prev_padding_count = padding_count;
-
-						info_candidates.clear();
-						info_candidates.push_back(sniff_info);
-					} else if (more_than_one_row && more_than_one_column && start_good && rows_consistent &&
-					           !require_more_padding) {
-						bool same_quote_is_candidate = false;
-						for (auto &info_candidate : info_candidates) {
-							if (quote.compare(info_candidate.quote) == 0) {
-								same_quote_is_candidate = true;
-							}
-						}
-						if (!same_quote_is_candidate) {
-							sniff_info.skip_rows = start_row;
-							sniff_info.num_cols = num_cols;
-							sniff_info.new_line = options.new_line;
-							info_candidates.push_back(sniff_info);
-						}
-					}
-				}
-			}
-		}
-	}
-}
-
-void BufferedCSVReader::DetectCandidateTypes(const vector<LogicalType> &type_candidates,
-                                             const map<LogicalTypeId, vector<const char *>> &format_template_candidates,
-                                             const vector<BufferedCSVReaderOptions> &info_candidates,
-                                             BufferedCSVReaderOptions &original_options, idx_t best_num_cols,
-                                             vector<vector<LogicalType>> &best_sql_types_candidates,
-                                             std::map<LogicalTypeId, vector<string>> &best_format_candidates,
-                                             DataChunk &best_header_row) {
-	BufferedCSVReaderOptions best_options;
-	idx_t min_varchar_cols = best_num_cols + 1;
-
-	// check which info candidate leads to minimum amount of non-varchar columns...
-	for (const auto &t : format_template_candidates) {
-		best_format_candidates[t.first].clear();
-	}
-	for (auto &info_candidate : info_candidates) {
-		options = info_candidate;
-		vector<vector<LogicalType>> info_sql_types_candidates(options.num_cols, type_candidates);
-		std::map<LogicalTypeId, bool> has_format_candidates;
-		std::map<LogicalTypeId, vector<string>> format_candidates;
-		for (const auto &t : format_template_candidates) {
-			has_format_candidates[t.first] = false;
-			format_candidates[t.first].clear();
-		}
-
-		// set all return_types to VARCHAR so we can do datatype detection based on VARCHAR values
-		return_types.clear();
-		return_types.assign(options.num_cols, LogicalType::VARCHAR);
-
-		// jump to beginning and skip potential header
-		JumpToBeginning(options.skip_rows, true);
-		DataChunk header_row;
-		header_row.Initialize(allocator, return_types);
-		parse_chunk.Copy(header_row);
-
-		if (header_row.size() == 0) {
-			continue;
-		}
-
-		// init parse chunk and read csv with info candidate
-		InitParseChunk(return_types.size());
-		if (!TryParseCSV(ParserMode::SNIFFING_DATATYPES)) {
-			continue;
-		}
-		for (idx_t row_idx = 0; row_idx <= parse_chunk.size(); row_idx++) {
-			bool is_header_row = row_idx == 0;
-			idx_t row = row_idx - 1;
-			for (idx_t col = 0; col < parse_chunk.ColumnCount(); col++) {
-				auto &col_type_candidates = info_sql_types_candidates[col];
-				while (col_type_candidates.size() > 1) {
-					const auto &sql_type = col_type_candidates.back();
-					// try cast from string to sql_type
-					Value dummy_val;
-					if (is_header_row) {
-						VerifyUTF8(col, 0, header_row, -int64_t(parse_chunk.size()));
-						dummy_val = header_row.GetValue(col, 0);
-					} else {
-						VerifyUTF8(col, row, parse_chunk);
-						dummy_val = parse_chunk.GetValue(col, row);
-					}
-					// try formatting for date types if the user did not specify one and it starts with numeric values.
-					string separator;
-					if (has_format_candidates.count(sql_type.id()) && !original_options.has_format[sql_type.id()] &&
-					    !dummy_val.IsNull() && StartsWithNumericDate(separator, StringValue::Get(dummy_val))) {
-						// generate date format candidates the first time through
-						auto &type_format_candidates = format_candidates[sql_type.id()];
-						const auto had_format_candidates = has_format_candidates[sql_type.id()];
-						if (!has_format_candidates[sql_type.id()]) {
-							has_format_candidates[sql_type.id()] = true;
-							// order by preference
-							auto entry = format_template_candidates.find(sql_type.id());
-							if (entry != format_template_candidates.end()) {
-								const auto &format_template_list = entry->second;
-								for (const auto &t : format_template_list) {
-									const auto format_string = GenerateDateFormat(separator, t);
-									// don't parse ISO 8601
-									if (format_string.find("%Y-%m-%d") == string::npos) {
-										type_format_candidates.emplace_back(format_string);
-									}
-								}
-							}
-							//	initialise the first candidate
-							options.has_format[sql_type.id()] = true;
-							//	all formats are constructed to be valid
-							SetDateFormat(type_format_candidates.back(), sql_type.id());
-						}
-						// check all formats and keep the first one that works
-						StrpTimeFormat::ParseResult result;
-						auto save_format_candidates = type_format_candidates;
-						while (!type_format_candidates.empty()) {
-							//	avoid using exceptions for flow control...
-							auto &current_format = options.date_format[sql_type.id()];
-							if (current_format.Parse(StringValue::Get(dummy_val), result)) {
-								break;
-							}
-							//	doesn't work - move to the next one
-							type_format_candidates.pop_back();
-							options.has_format[sql_type.id()] = (!type_format_candidates.empty());
-							if (!type_format_candidates.empty()) {
-								SetDateFormat(type_format_candidates.back(), sql_type.id());
-							}
-						}
-						//	if none match, then this is not a value of type sql_type,
-						if (type_format_candidates.empty()) {
-							//	so restore the candidates that did work.
-							//	or throw them out if they were generated by this value.
-							if (had_format_candidates) {
-								type_format_candidates.swap(save_format_candidates);
-								if (!type_format_candidates.empty()) {
-									SetDateFormat(type_format_candidates.back(), sql_type.id());
-								}
-							} else {
-								has_format_candidates[sql_type.id()] = false;
-							}
-						}
-					}
-					// try cast from string to sql_type
-					if (TryCastValue(dummy_val, sql_type)) {
-						break;
-					} else {
-						col_type_candidates.pop_back();
-					}
-				}
-			}
-			// reset type detection, because first row could be header,
-			// but only do it if csv has more than one line (including header)
-			if (parse_chunk.size() > 0 && is_header_row) {
-				info_sql_types_candidates = vector<vector<LogicalType>>(options.num_cols, type_candidates);
-				for (auto &f : format_candidates) {
-					f.second.clear();
-				}
-				for (auto &h : has_format_candidates) {
-					h.second = false;
-				}
-			}
-		}
-
-		idx_t varchar_cols = 0;
-		for (idx_t col = 0; col < parse_chunk.ColumnCount(); col++) {
-			auto &col_type_candidates = info_sql_types_candidates[col];
-			// check number of varchar columns
-			const auto &col_type = col_type_candidates.back();
-			if (col_type == LogicalType::VARCHAR) {
-				varchar_cols++;
-			}
-		}
-
-		// it's good if the dialect creates more non-varchar columns, but only if we sacrifice < 30% of best_num_cols.
-		if (varchar_cols < min_varchar_cols && parse_chunk.ColumnCount() > (best_num_cols * 0.7)) {
-			// we have a new best_options candidate
-			best_options = info_candidate;
-			min_varchar_cols = varchar_cols;
-			best_sql_types_candidates = info_sql_types_candidates;
-			best_format_candidates = format_candidates;
-			best_header_row.Destroy();
-			auto header_row_types = header_row.GetTypes();
-			best_header_row.Initialize(allocator, header_row_types);
-			header_row.Copy(best_header_row);
-		}
-	}
-
-	options = best_options;
-	for (const auto &best : best_format_candidates) {
-		if (!best.second.empty()) {
-			SetDateFormat(best.second.back(), best.first);
-		}
-	}
-}
-
-void BufferedCSVReader::DetectHeader(const vector<vector<LogicalType>> &best_sql_types_candidates,
-                                     const DataChunk &best_header_row) {
-	// information for header detection
-	bool first_row_consistent = true;
-	bool first_row_nulls = false;
-
-	// check if header row is all null and/or consistent with detected column data types
-	first_row_nulls = true;
-	for (idx_t col = 0; col < best_sql_types_candidates.size(); col++) {
-		auto dummy_val = best_header_row.GetValue(col, 0);
-		if (!dummy_val.IsNull()) {
-			first_row_nulls = false;
-		}
-
-		// try cast to sql_type of column
-		const auto &sql_type = best_sql_types_candidates[col].back();
-		if (!TryCastValue(dummy_val, sql_type)) {
-			first_row_consistent = false;
-		}
-	}
-
-	// update parser info, and read, generate & set col_names based on previous findings
-	if (((!first_row_consistent || first_row_nulls) && !options.has_header) || (options.has_header && options.header)) {
-		options.header = true;
-		case_insensitive_map_t<idx_t> name_collision_count;
-		// get header names from CSV
-		for (idx_t col = 0; col < options.num_cols; col++) {
-			const auto &val = best_header_row.GetValue(col, 0);
-			string col_name = val.ToString();
-
-			// generate name if field is empty
-			if (col_name.empty() || val.IsNull()) {
-				col_name = GenerateColumnName(options.num_cols, col);
-			}
-
-			// normalize names or at least trim whitespace
-			if (options.normalize_names) {
-				col_name = NormalizeColumnName(col_name);
-			} else {
-				col_name = TrimWhitespace(col_name);
-			}
-
-			// avoid duplicate header names
-			const string col_name_raw = col_name;
-			while (name_collision_count.find(col_name) != name_collision_count.end()) {
-				name_collision_count[col_name] += 1;
-				col_name = col_name + "_" + to_string(name_collision_count[col_name]);
-			}
-
-			names.push_back(col_name);
-			name_collision_count[col_name] = 0;
-		}
-
-	} else {
-		options.header = false;
-		for (idx_t col = 0; col < options.num_cols; col++) {
-			string column_name = GenerateColumnName(options.num_cols, col);
-			names.push_back(column_name);
-		}
-	}
-	for (idx_t i = 0; i < MinValue<idx_t>(names.size(), options.name_list.size()); i++) {
-		names[i] = options.name_list[i];
-	}
-}
-
-vector<LogicalType> BufferedCSVReader::RefineTypeDetection(const vector<LogicalType> &type_candidates,
-                                                           const vector<LogicalType> &requested_types,
-                                                           vector<vector<LogicalType>> &best_sql_types_candidates,
-                                                           map<LogicalTypeId, vector<string>> &best_format_candidates) {
-	// for the type refine we set the SQL types to VARCHAR for all columns
-	return_types.clear();
-	return_types.assign(options.num_cols, LogicalType::VARCHAR);
-
-	vector<LogicalType> detected_types;
-
-	// if data types were provided, exit here if number of columns does not match
-	if (!requested_types.empty()) {
-		if (requested_types.size() != options.num_cols) {
-			throw InvalidInputException(
-			    "Error while determining column types: found %lld columns but expected %d. (%s)", options.num_cols,
-			    requested_types.size(), options.ToString());
-		} else {
-			detected_types = requested_types;
-		}
-	} else if (options.all_varchar) {
-		// return all types varchar
-		detected_types = return_types;
-	} else {
-		// jump through the rest of the file and continue to refine the sql type guess
-		while (JumpToNextSample()) {
-			InitParseChunk(return_types.size());
-			// if jump ends up a bad line, we just skip this chunk
-			if (!TryParseCSV(ParserMode::SNIFFING_DATATYPES)) {
-				continue;
-			}
-			for (idx_t col = 0; col < parse_chunk.ColumnCount(); col++) {
-				vector<LogicalType> &col_type_candidates = best_sql_types_candidates[col];
-				while (col_type_candidates.size() > 1) {
-					const auto &sql_type = col_type_candidates.back();
-					//	narrow down the date formats
-					if (best_format_candidates.count(sql_type.id())) {
-						auto &best_type_format_candidates = best_format_candidates[sql_type.id()];
-						auto save_format_candidates = best_type_format_candidates;
-						while (!best_type_format_candidates.empty()) {
-							if (TryCastVector(parse_chunk.data[col], parse_chunk.size(), sql_type)) {
-								break;
-							}
-							//	doesn't work - move to the next one
-							best_type_format_candidates.pop_back();
-							options.has_format[sql_type.id()] = (!best_type_format_candidates.empty());
-							if (!best_type_format_candidates.empty()) {
-								SetDateFormat(best_type_format_candidates.back(), sql_type.id());
-							}
-						}
-						//	if none match, then this is not a column of type sql_type,
-						if (best_type_format_candidates.empty()) {
-							//	so restore the candidates that did work.
-							best_type_format_candidates.swap(save_format_candidates);
-							if (!best_type_format_candidates.empty()) {
-								SetDateFormat(best_type_format_candidates.back(), sql_type.id());
-							}
-						}
-					}
-
-					if (TryCastVector(parse_chunk.data[col], parse_chunk.size(), sql_type)) {
-						break;
-					} else {
-						col_type_candidates.pop_back();
-					}
-				}
-			}
-		}
-
-		// set sql types
-		for (auto &best_sql_types_candidate : best_sql_types_candidates) {
-			LogicalType d_type = best_sql_types_candidate.back();
-			if (best_sql_types_candidate.size() == type_candidates.size()) {
-				d_type = LogicalType::VARCHAR;
-			}
-			detected_types.push_back(d_type);
-		}
-	}
-
-	return detected_types;
-}
-
-string BufferedCSVReader::ColumnTypesError(case_insensitive_map_t<idx_t> sql_types_per_column,
-                                           const vector<string> &names) {
-	for (idx_t i = 0; i < names.size(); i++) {
-		auto it = sql_types_per_column.find(names[i]);
-		if (it != sql_types_per_column.end()) {
-			sql_types_per_column.erase(names[i]);
-			continue;
-		}
-	}
-	if (sql_types_per_column.empty()) {
-		return string();
-	}
-	string exception = "COLUMN_TYPES error: Columns with names: ";
-	for (auto &col : sql_types_per_column) {
-		exception += "\"" + col.first + "\",";
-	}
-	exception.pop_back();
-	exception += " do not exist in the CSV File";
-	return exception;
-}
-
-vector<LogicalType> BufferedCSVReader::SniffCSV(const vector<LogicalType> &requested_types) {
-	for (auto &type : requested_types) {
-		// auto detect for blobs not supported: there may be invalid UTF-8 in the file
-		if (type.id() == LogicalTypeId::BLOB) {
-			return requested_types;
-		}
-	}
-
-	// #######
-	// ### dialect detection
-	// #######
-	BufferedCSVReaderOptions original_options = options;
-	vector<BufferedCSVReaderOptions> info_candidates;
-	idx_t best_num_cols = 0;
-
-	DetectDialect(requested_types, original_options, info_candidates, best_num_cols);
-
-	// if no dialect candidate was found, then file was most likely empty and we throw an exception
-	if (info_candidates.empty()) {
-		throw InvalidInputException(
-		    "Error in file \"%s\": CSV options could not be auto-detected. Consider setting parser options manually.",
-		    options.file_path);
-	}
-
-	// #######
-	// ### type detection (initial)
-	// #######
-
-	// format template candidates, ordered by descending specificity (~ from high to low)
-	std::map<LogicalTypeId, vector<const char *>> format_template_candidates = {
-	    {LogicalTypeId::DATE, {"%m-%d-%Y", "%m-%d-%y", "%d-%m-%Y", "%d-%m-%y", "%Y-%m-%d", "%y-%m-%d"}},
-	    {LogicalTypeId::TIMESTAMP,
-	     {"%Y-%m-%d %H:%M:%S.%f", "%m-%d-%Y %I:%M:%S %p", "%m-%d-%y %I:%M:%S %p", "%d-%m-%Y %H:%M:%S",
-	      "%d-%m-%y %H:%M:%S", "%Y-%m-%d %H:%M:%S", "%y-%m-%d %H:%M:%S"}},
-	};
-	vector<vector<LogicalType>> best_sql_types_candidates;
-	map<LogicalTypeId, vector<string>> best_format_candidates;
-	DataChunk best_header_row;
-	DetectCandidateTypes(options.auto_type_candidates, format_template_candidates, info_candidates, original_options,
-	                     best_num_cols, best_sql_types_candidates, best_format_candidates, best_header_row);
-
-	if (best_format_candidates.empty() || best_header_row.size() == 0) {
-		throw InvalidInputException(
-		    "Error in file \"%s\": CSV options could not be auto-detected. Consider setting parser options manually.",
-		    original_options.file_path);
-	}
-
-	// #######
-	// ### header detection
-	// #######
-	options.num_cols = best_num_cols;
-	DetectHeader(best_sql_types_candidates, best_header_row);
-	if (!options.sql_type_list.empty()) {
-		// user-defined types were supplied for certain columns
-		// override the types
-		if (!options.sql_types_per_column.empty()) {
-			// types supplied as name -> value map
-			idx_t found = 0;
-			for (idx_t i = 0; i < names.size(); i++) {
-				auto it = options.sql_types_per_column.find(names[i]);
-				if (it != options.sql_types_per_column.end()) {
-					best_sql_types_candidates[i] = {options.sql_type_list[it->second]};
-					found++;
-					continue;
-				}
-			}
-			if (!options.file_options.union_by_name && found < options.sql_types_per_column.size()) {
-				string exception = ColumnTypesError(options.sql_types_per_column, names);
-				if (!exception.empty()) {
-					throw BinderException(exception);
-				}
-			}
-		} else {
-			// types supplied as list
-			if (names.size() < options.sql_type_list.size()) {
-				throw BinderException("read_csv: %d types were provided, but CSV file only has %d columns",
-				                      options.sql_type_list.size(), names.size());
-			}
-			for (idx_t i = 0; i < options.sql_type_list.size(); i++) {
-				best_sql_types_candidates[i] = {options.sql_type_list[i]};
-			}
-		}
-	}
-
-	// #######
-	// ### type detection (refining)
-	// #######
-	return RefineTypeDetection(options.auto_type_candidates, requested_types, best_sql_types_candidates,
-	                           best_format_candidates);
-}
-
-bool BufferedCSVReader::TryParseComplexCSV(DataChunk &insert_chunk, string &error_message) {
-	// used for parsing algorithm
-	bool finished_chunk = false;
-	idx_t column = 0;
-	vector<idx_t> escape_positions;
-	bool has_quotes = false;
-	uint8_t delimiter_pos = 0, escape_pos = 0, quote_pos = 0;
-	idx_t offset = 0;
-	idx_t line_start = 0;
-	// read values into the buffer (if any)
-	if (position >= buffer_size) {
-		if (!ReadBuffer(start, line_start)) {
-			return true;
-		}
-	}
-	// start parsing the first value
-	start = position;
-	goto value_start;
-value_start:
-	/* state: value_start */
-	// this state parses the first characters of a value
-	offset = 0;
-	delimiter_pos = 0;
-	quote_pos = 0;
-	do {
-		idx_t count = 0;
-		for (; position < buffer_size; position++) {
-			quote_search.Match(quote_pos, buffer[position]);
-			delimiter_search.Match(delimiter_pos, buffer[position]);
-			count++;
-			if (delimiter_pos == options.delimiter.size()) {
-				// found a delimiter, add the value
-				offset = options.delimiter.size() - 1;
-				goto add_value;
-			} else if (StringUtil::CharacterIsNewline(buffer[position])) {
-				// found a newline, add the row
-				goto add_row;
-			}
-			if (count > quote_pos) {
-				// did not find a quote directly at the start of the value, stop looking for the quote now
-				goto normal;
-			}
-			if (quote_pos == options.quote.size()) {
-				// found a quote, go to quoted loop and skip the initial quote
-				start += options.quote.size();
-				goto in_quotes;
-			}
-		}
-	} while (ReadBuffer(start, line_start));
-	// file ends while scanning for quote/delimiter, go to final state
-	goto final_state;
-normal:
-	/* state: normal parsing state */
-	// this state parses the remainder of a non-quoted value until we reach a delimiter or newline
-	position++;
-	do {
-		for (; position < buffer_size; position++) {
-			delimiter_search.Match(delimiter_pos, buffer[position]);
-			if (delimiter_pos == options.delimiter.size()) {
-				offset = options.delimiter.size() - 1;
-				goto add_value;
-			} else if (StringUtil::CharacterIsNewline(buffer[position])) {
-				goto add_row;
-			}
-		}
-	} while (ReadBuffer(start, line_start));
-	goto final_state;
-add_value:
-	AddValue(string_t(buffer.get() + start, position - start - offset), column, escape_positions, has_quotes);
-	// increase position by 1 and move start to the new position
-	offset = 0;
-	has_quotes = false;
-	start = ++position;
-	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
-		// file ends right after delimiter, go to final state
-		goto final_state;
-	}
-	goto value_start;
-add_row : {
-	// check type of newline (\r or \n)
-	bool carriage_return = buffer[position] == '\r';
-	AddValue(string_t(buffer.get() + start, position - start - offset), column, escape_positions, has_quotes);
-	finished_chunk = AddRow(insert_chunk, column, error_message);
-
-	if (!error_message.empty()) {
-		return false;
-	}
-	// increase position by 1 and move start to the new position
-	offset = 0;
-	has_quotes = false;
-	position++;
-	SkipEmptyLines();
-	start = position;
-	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
-		// file ends right after newline, go to final state
-		goto final_state;
-	}
-	if (carriage_return) {
-		// \r newline, go to special state that parses an optional \n afterwards
-		goto carriage_return;
-	} else {
-		// \n newline, move to value start
-		if (finished_chunk) {
-			return true;
-		}
-		goto value_start;
-	}
-}
-in_quotes:
-	/* state: in_quotes */
-	// this state parses the remainder of a quoted value
-	quote_pos = 0;
-	escape_pos = 0;
-	has_quotes = true;
-	position++;
-	do {
-		for (; position < buffer_size; position++) {
-			quote_search.Match(quote_pos, buffer[position]);
-			escape_search.Match(escape_pos, buffer[position]);
-			if (quote_pos == options.quote.size()) {
-				goto unquote;
-			} else if (escape_pos == options.escape.size()) {
-				escape_positions.push_back(position - start - (options.escape.size() - 1));
-				goto handle_escape;
-			}
-		}
-	} while (ReadBuffer(start, line_start));
-	// still in quoted state at the end of the file, error:
-	error_message = StringUtil::Format("Error in file \"%s\" on line %s: unterminated quotes. (%s)", options.file_path,
-	                                   GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
-	return false;
-unquote:
-	/* state: unquote */
-	// this state handles the state directly after we unquote
-	// in this state we expect either another quote (entering the quoted state again, and escaping the quote)
-	// or a delimiter/newline, ending the current value and moving on to the next value
-	delimiter_pos = 0;
-	quote_pos = 0;
-	position++;
-	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
-		// file ends right after unquote, go to final state
-		offset = options.quote.size();
-		goto final_state;
-	}
-	if (StringUtil::CharacterIsNewline(buffer[position])) {
-		// quote followed by newline, add row
-		offset = options.quote.size();
-		goto add_row;
-	}
-	do {
-		idx_t count = 0;
-		for (; position < buffer_size; position++) {
-			quote_search.Match(quote_pos, buffer[position]);
-			delimiter_search.Match(delimiter_pos, buffer[position]);
-			count++;
-			if (count > delimiter_pos && count > quote_pos) {
-				error_message = StringUtil::Format(
-				    "Error in file \"%s\" on line %s: quote should be followed by end of value, end "
-				    "of row or another quote. (%s)",
-				    options.file_path, GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
-				return false;
-			}
-			if (delimiter_pos == options.delimiter.size()) {
-				// quote followed by delimiter, add value
-				offset = options.quote.size() + options.delimiter.size() - 1;
-				goto add_value;
-			} else if (quote_pos == options.quote.size() &&
-			           (options.escape.empty() || options.escape == options.quote)) {
-				// quote followed by quote, go back to quoted state and add to escape
-				escape_positions.push_back(position - start - (options.quote.size() - 1));
-				goto in_quotes;
-			}
-		}
-	} while (ReadBuffer(start, line_start));
-	error_message = StringUtil::Format(
-	    "Error in file \"%s\" on line %s: quote should be followed by end of value, end of row or another quote. (%s)",
-	    options.file_path, GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
-	return false;
-handle_escape:
-	escape_pos = 0;
-	quote_pos = 0;
-	position++;
-	do {
-		idx_t count = 0;
-		for (; position < buffer_size; position++) {
-			quote_search.Match(quote_pos, buffer[position]);
-			escape_search.Match(escape_pos, buffer[position]);
-			count++;
-			if (count > escape_pos && count > quote_pos) {
-				error_message = StringUtil::Format(
-				    "Error in file \"%s\" on line %s: neither QUOTE nor ESCAPE is proceeded by ESCAPE. (%s)",
-				    options.file_path, GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
-				return false;
-			}
-			if (quote_pos == options.quote.size() || escape_pos == options.escape.size()) {
-				// found quote or escape: move back to quoted state
-				goto in_quotes;
-			}
-		}
-	} while (ReadBuffer(start, line_start));
-	error_message =
-	    StringUtil::Format("Error in file \"%s\" on line %s: neither QUOTE nor ESCAPE is proceeded by ESCAPE. (%s)",
-	                       options.file_path, GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
-	return false;
-carriage_return:
-	/* state: carriage_return */
-	// this stage optionally skips a newline (\n) character, which allows \r\n to be interpreted as a single line
-	if (buffer[position] == '\n') {
-		// newline after carriage return: skip
-		start = ++position;
-		if (position >= buffer_size && !ReadBuffer(start, line_start)) {
-			// file ends right after newline, go to final state
-			goto final_state;
-		}
-	}
-	if (finished_chunk) {
-		return true;
-	}
-	goto value_start;
-final_state:
-	if (finished_chunk) {
-		return true;
-	}
-	if (column > 0 || position > start) {
-		// remaining values to be added to the chunk
-		AddValue(string_t(buffer.get() + start, position - start - offset), column, escape_positions, has_quotes);
-		finished_chunk = AddRow(insert_chunk, column, error_message);
-		SkipEmptyLines();
-		if (!error_message.empty()) {
-			return false;
-		}
-	}
-	// final stage, only reached after parsing the file is finished
-	// flush the parsed chunk and finalize parsing
-	if (mode == ParserMode::PARSING) {
-		Flush(insert_chunk);
-	}
-
-	end_of_file_reached = true;
-	return true;
-}
-
-void BufferedCSVReader::SkipEmptyLines() {
-	if (parse_chunk.data.size() == 1) {
-		// Empty lines are null data.
-		return;
-	}
-	for (; position < buffer_size; position++) {
-		if (!StringUtil::CharacterIsNewline(buffer[position])) {
-			return;
-		}
-	}
-}
-
-void UpdateMaxLineLength(ClientContext &context, idx_t line_length) {
-	if (!context.client_data->debug_set_max_line_length) {
-		return;
-	}
-	if (line_length < context.client_data->debug_max_line_length) {
-		return;
-	}
-	context.client_data->debug_max_line_length = line_length;
-}
-
-bool BufferedCSVReader::TryParseSimpleCSV(DataChunk &insert_chunk, string &error_message) {
-	// used for parsing algorithm
-	bool finished_chunk = false;
-	idx_t column = 0;
-	idx_t offset = 0;
-	bool has_quotes = false;
-	vector<idx_t> escape_positions;
-
-	idx_t line_start = position;
-	// read values into the buffer (if any)
-	if (position >= buffer_size) {
-		if (!ReadBuffer(start, line_start)) {
-			return true;
-		}
-	}
-
-	// start parsing the first value
-	goto value_start;
-value_start:
-	offset = 0;
-	/* state: value_start */
-	// this state parses the first character of a value
-	if (buffer[position] == options.quote[0]) {
-		// quote: actual value starts in the next position
-		// move to in_quotes state
-		start = position + 1;
-		goto in_quotes;
-	} else {
-		// no quote, move to normal parsing state
-		start = position;
-		goto normal;
-	}
-normal:
-	/* state: normal parsing state */
-	// this state parses the remainder of a non-quoted value until we reach a delimiter or newline
-	do {
-		for (; position < buffer_size; position++) {
-			if (buffer[position] == options.delimiter[0]) {
-				// delimiter: end the value and add it to the chunk
-				goto add_value;
-			} else if (StringUtil::CharacterIsNewline(buffer[position])) {
-				// newline: add row
-				goto add_row;
-			}
-		}
-	} while (ReadBuffer(start, line_start));
-	// file ends during normal scan: go to end state
-	goto final_state;
-add_value:
-	AddValue(string_t(buffer.get() + start, position - start - offset), column, escape_positions, has_quotes);
-	// increase position by 1 and move start to the new position
-	offset = 0;
-	has_quotes = false;
-	start = ++position;
-	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
-		// file ends right after delimiter, go to final state
-		goto final_state;
-	}
-	goto value_start;
-add_row : {
-	// check type of newline (\r or \n)
-	bool carriage_return = buffer[position] == '\r';
-	AddValue(string_t(buffer.get() + start, position - start - offset), column, escape_positions, has_quotes);
-	if (!error_message.empty()) {
-		return false;
-	}
-	finished_chunk = AddRow(insert_chunk, column, error_message);
-	UpdateMaxLineLength(context, position - line_start);
-	if (!error_message.empty()) {
-		return false;
-	}
-	// increase position by 1 and move start to the new position
-	offset = 0;
-	has_quotes = false;
-	position++;
-	start = position;
-	line_start = position;
-	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
-		// file ends right after delimiter, go to final state
-		goto final_state;
-	}
-	if (carriage_return) {
-		// \r newline, go to special state that parses an optional \n afterwards
-		goto carriage_return;
-	} else {
-		SetNewLineDelimiter();
-		SkipEmptyLines();
-
-		start = position;
-		line_start = position;
-		if (position >= buffer_size && !ReadBuffer(start, line_start)) {
-			// file ends right after delimiter, go to final state
-			goto final_state;
-		}
-		// \n newline, move to value start
-		if (finished_chunk) {
-			return true;
-		}
-		goto value_start;
-	}
-}
-in_quotes:
-	/* state: in_quotes */
-	// this state parses the remainder of a quoted value
-	has_quotes = true;
-	position++;
-	do {
-		for (; position < buffer_size; position++) {
-			if (buffer[position] == options.quote[0]) {
-				// quote: move to unquoted state
-				goto unquote;
-			} else if (buffer[position] == options.escape[0]) {
-				// escape: store the escaped position and move to handle_escape state
-				escape_positions.push_back(position - start);
-				goto handle_escape;
-			}
-		}
-	} while (ReadBuffer(start, line_start));
-	// still in quoted state at the end of the file, error:
-	throw InvalidInputException("Error in file \"%s\" on line %s: unterminated quotes. (%s)", options.file_path,
-	                            GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
-unquote:
-	/* state: unquote */
-	// this state handles the state directly after we unquote
-	// in this state we expect either another quote (entering the quoted state again, and escaping the quote)
-	// or a delimiter/newline, ending the current value and moving on to the next value
-	position++;
-	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
-		// file ends right after unquote, go to final state
-		offset = 1;
-		goto final_state;
-	}
-	if (buffer[position] == options.quote[0] && (options.escape.empty() || options.escape[0] == options.quote[0])) {
-		// escaped quote, return to quoted state and store escape position
-		escape_positions.push_back(position - start);
-		goto in_quotes;
-	} else if (buffer[position] == options.delimiter[0]) {
-		// delimiter, add value
-		offset = 1;
-		goto add_value;
-	} else if (StringUtil::CharacterIsNewline(buffer[position])) {
-		offset = 1;
-		goto add_row;
-	} else {
-		error_message = StringUtil::Format(
-		    "Error in file \"%s\" on line %s: quote should be followed by end of value, end of "
-		    "row or another quote. (%s)",
-		    options.file_path, GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
-		return false;
-	}
-handle_escape:
-	/* state: handle_escape */
-	// escape should be followed by a quote or another escape character
-	position++;
-	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
-		error_message = StringUtil::Format(
-		    "Error in file \"%s\" on line %s: neither QUOTE nor ESCAPE is proceeded by ESCAPE. (%s)", options.file_path,
-		    GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
-		return false;
-	}
-	if (buffer[position] != options.quote[0] && buffer[position] != options.escape[0]) {
-		error_message = StringUtil::Format(
-		    "Error in file \"%s\" on line %s: neither QUOTE nor ESCAPE is proceeded by ESCAPE. (%s)", options.file_path,
-		    GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
-		return false;
-	}
-	// escape was followed by quote or escape, go back to quoted state
-	goto in_quotes;
-carriage_return:
-	/* state: carriage_return */
-	// this stage optionally skips a newline (\n) character, which allows \r\n to be interpreted as a single line
-	if (buffer[position] == '\n') {
-		SetNewLineDelimiter(true, true);
-		// newline after carriage return: skip
-		// increase position by 1 and move start to the new position
-		start = ++position;
-		if (position >= buffer_size && !ReadBuffer(start, line_start)) {
-			// file ends right after delimiter, go to final state
-			goto final_state;
-		}
-	} else {
-		SetNewLineDelimiter(true, false);
-	}
-	if (finished_chunk) {
-		return true;
-	}
-	SkipEmptyLines();
-	start = position;
-	line_start = position;
-	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
-		// file ends right after delimiter, go to final state
-		goto final_state;
-	}
-
-	goto value_start;
-final_state:
-	if (finished_chunk) {
-		return true;
-	}
-
-	if (column > 0 || position > start) {
-		// remaining values to be added to the chunk
-		AddValue(string_t(buffer.get() + start, position - start - offset), column, escape_positions, has_quotes);
-		finished_chunk = AddRow(insert_chunk, column, error_message);
-		SkipEmptyLines();
-		UpdateMaxLineLength(context, position - line_start);
-		if (!error_message.empty()) {
-			return false;
-		}
-	}
-
-	// final stage, only reached after parsing the file is finished
-	// flush the parsed chunk and finalize parsing
-	if (mode == ParserMode::PARSING) {
-		Flush(insert_chunk);
-	}
-
-	end_of_file_reached = true;
-	return true;
-}
-
-bool BufferedCSVReader::ReadBuffer(idx_t &start, idx_t &line_start) {
-	if (start > buffer_size) {
-		return false;
-	}
-	auto old_buffer = std::move(buffer);
-
-	// the remaining part of the last buffer
-	idx_t remaining = buffer_size - start;
-
-	bool large_buffers = mode == ParserMode::PARSING && !file_handle->OnDiskFile() && file_handle->CanSeek();
-	idx_t buffer_read_size = large_buffers ? INITIAL_BUFFER_SIZE_LARGE : INITIAL_BUFFER_SIZE;
-
-	while (remaining > buffer_read_size) {
-		buffer_read_size *= 2;
-	}
-
-	// Check line length
-	if (remaining > options.maximum_line_size) {
-		throw InvalidInputException("Maximum line size of %llu bytes exceeded on line %s!", options.maximum_line_size,
-		                            GetLineNumberStr(linenr, linenr_estimated));
-	}
-
-	buffer = make_unsafe_uniq_array<char>(buffer_read_size + remaining + 1);
-	buffer_size = remaining + buffer_read_size;
-	if (remaining > 0) {
-		// remaining from last buffer: copy it here
-		memcpy(buffer.get(), old_buffer.get() + start, remaining);
-	}
-	idx_t read_count = file_handle->Read(buffer.get() + remaining, buffer_read_size);
-
-	bytes_in_chunk += read_count;
-	buffer_size = remaining + read_count;
-	buffer[buffer_size] = '\0';
-	if (old_buffer) {
-		cached_buffers.push_back(std::move(old_buffer));
-	}
-	start = 0;
-	position = remaining;
-	if (!bom_checked) {
-		bom_checked = true;
-		if (read_count >= 3 && buffer[0] == '\xEF' && buffer[1] == '\xBB' && buffer[2] == '\xBF') {
-			start += 3;
-			position += 3;
-		}
-	}
-	line_start = start;
-
-	return read_count > 0;
-}
-
-void BufferedCSVReader::ParseCSV(DataChunk &insert_chunk) {
-	string error_message;
-	if (!TryParseCSV(ParserMode::PARSING, insert_chunk, error_message)) {
-		throw InvalidInputException(error_message);
-	}
-}
-
-bool BufferedCSVReader::TryParseCSV(ParserMode mode) {
-	DataChunk dummy_chunk;
-	string error_message;
-	return TryParseCSV(mode, dummy_chunk, error_message);
-}
-
-void BufferedCSVReader::ParseCSV(ParserMode mode) {
-	DataChunk dummy_chunk;
-	string error_message;
-	if (!TryParseCSV(mode, dummy_chunk, error_message)) {
-		throw InvalidInputException(error_message);
-	}
-}
-
-bool BufferedCSVReader::TryParseCSV(ParserMode parser_mode, DataChunk &insert_chunk, string &error_message) {
-	mode = parser_mode;
-
-	if (options.quote.size() <= 1 && options.escape.size() <= 1 && options.delimiter.size() == 1) {
-		return TryParseSimpleCSV(insert_chunk, error_message);
-	} else {
-		return TryParseComplexCSV(insert_chunk, error_message);
-	}
-}
-
-} // namespace duckdb