duckdb 1.0.1-dev22.0 → 1.0.1-dev27.0

package/src/duckdb/src/execution/operator/csv_scanner/sniffer/dialect_detection.cpp

@@ -1,9 +1,12 @@
+#include "duckdb/common/shared_ptr.hpp"
 #include "duckdb/execution/operator/csv_scanner/csv_sniffer.hpp"
 #include "duckdb/main/client_data.hpp"
-#include "duckdb/common/shared_ptr.hpp"
+#include "duckdb/execution/operator/csv_scanner/csv_reader_options.hpp"
 
 namespace duckdb {
 
+constexpr idx_t CSVReaderOptions::sniff_size;
+
 bool IsQuoteDefault(char quote) {
 	if (quote == '\"' || quote == '\'' || quote == '\0') {
 		return true;
@@ -11,56 +14,129 @@ bool IsQuoteDefault(char quote) {
 	return false;
 }
 
-void CSVSniffer::GenerateCandidateDetectionSearchSpace(vector<char> &delim_candidates,
-                                                       vector<QuoteRule> &quoterule_candidates,
-                                                       unordered_map<uint8_t, vector<char>> &quote_candidates_map,
-                                                       unordered_map<uint8_t, vector<char>> &escape_candidates_map) {
-	if (options.dialect_options.state_machine_options.delimiter.IsSetByUser()) {
+vector<char> DialectCandidates::GetDefaultDelimiter() {
+	return {',', '|', ';', '\t'};
+}
+
+vector<vector<char>> DialectCandidates::GetDefaultQuote() {
+	return {{'\"'}, {'\"', '\''}, {'\0'}};
+}
+
+vector<QuoteRule> DialectCandidates::GetDefaultQuoteRule() {
+	return {QuoteRule::QUOTES_RFC, QuoteRule::QUOTES_OTHER, QuoteRule::NO_QUOTES};
+}
+
+vector<vector<char>> DialectCandidates::GetDefaultEscape() {
+	return {{'\"', '\0', '\''}, {'\\'}, {'\0'}};
+}
+
+vector<char> DialectCandidates::GetDefaultComment() {
+	return {'#', '\0'};
+}
+
+string DialectCandidates::Print() {
+	std::ostringstream search_space;
+
+	search_space << "Delimiter Candidates: ";
+	for (idx_t i = 0; i < delim_candidates.size(); i++) {
+		search_space << "\'" << delim_candidates[i] << "\'";
+		if (i < delim_candidates.size() - 1) {
+			search_space << ", ";
+		}
+	}
+	search_space << "\n";
+	search_space << "Quote/Escape Candidates: ";
+	for (uint8_t i = 0; i < static_cast<uint8_t>(quoterule_candidates.size()); i++) {
+		auto quote_candidate = quote_candidates_map[i];
+		auto escape_candidate = escape_candidates_map[i];
+		for (idx_t j = 0; j < quote_candidate.size(); j++) {
+			for (idx_t k = 0; k < escape_candidate.size(); k++) {
+				search_space << "[\'" << quote_candidate[j] << "\',\'" << escape_candidate[k] << "\']";
+				if (k < escape_candidate.size() - 1) {
+					search_space << ",";
+				}
+			}
+			if (j < quote_candidate.size() - 1) {
+				search_space << ",";
+			}
+		}
+		if (i < quoterule_candidates.size() - 1) {
+			search_space << ",";
+		}
+	}
+	search_space << "\n";
+
+	search_space << "Comment Candidates: ";
+	for (idx_t i = 0; i < comment_candidates.size(); i++) {
+		search_space << "\'" << comment_candidates[i] << "\'";
+		if (i < comment_candidates.size() - 1) {
+			search_space << ", ";
+		}
+	}
+	search_space << "\n";
+
+	return search_space.str();
+}
+
+DialectCandidates::DialectCandidates(const CSVStateMachineOptions &options) {
+	// assert that quotes escapes and rules have equal size
+	auto default_quote = GetDefaultQuote();
+	auto default_escape = GetDefaultEscape();
+	auto default_quote_rule = GetDefaultQuoteRule();
+	auto default_delimiter = GetDefaultDelimiter();
+	auto default_comment = GetDefaultComment();
+
+	D_ASSERT(default_quote.size() == default_quote_rule.size() && default_quote_rule.size() == default_escape.size());
+	// fill the escapes
+	for (idx_t i = 0; i < default_quote_rule.size(); i++) {
+		escape_candidates_map[static_cast<uint8_t>(default_quote_rule[i])] = default_escape[i];
+	}
+
+	if (options.delimiter.IsSetByUser()) {
 		// user provided a delimiter: use that delimiter
-		delim_candidates = {options.dialect_options.state_machine_options.delimiter.GetValue()};
+		delim_candidates = {options.delimiter.GetValue()};
 	} else {
 		// no delimiter provided: try standard/common delimiters
-		delim_candidates = {',', '|', ';', '\t'};
+		delim_candidates = default_delimiter;
+	}
+	if (options.comment.IsSetByUser()) {
+		// user provided comment character: use that as a comment
+		comment_candidates = {options.comment.GetValue()};
+	} else {
+		// no comment provided: try standard/common comments
+		comment_candidates = default_comment;
 	}
-	if (options.dialect_options.state_machine_options.quote.IsSetByUser()) {
+	if (options.quote.IsSetByUser()) {
 		// user provided quote: use that quote rule
-		quote_candidates_map[(uint8_t)QuoteRule::QUOTES_RFC] = {
-		    options.dialect_options.state_machine_options.quote.GetValue()};
-		quote_candidates_map[(uint8_t)QuoteRule::QUOTES_OTHER] = {
-		    options.dialect_options.state_machine_options.quote.GetValue()};
-		quote_candidates_map[(uint8_t)QuoteRule::NO_QUOTES] = {
-		    options.dialect_options.state_machine_options.quote.GetValue()};
+		for (auto &quote_rule : default_quote_rule) {
+			quote_candidates_map[static_cast<uint8_t>(quote_rule)] = {options.quote.GetValue()};
+		}
 		// also add it as a escape rule
-		if (!IsQuoteDefault(options.dialect_options.state_machine_options.quote.GetValue())) {
-			escape_candidates_map[(uint8_t)QuoteRule::QUOTES_RFC].emplace_back(
-			    options.dialect_options.state_machine_options.quote.GetValue());
+		if (!IsQuoteDefault(options.quote.GetValue())) {
+			escape_candidates_map[static_cast<uint8_t>(QuoteRule::QUOTES_RFC)].emplace_back(options.quote.GetValue());
 		}
 	} else {
 		// no quote rule provided: use standard/common quotes
-		quote_candidates_map[(uint8_t)QuoteRule::QUOTES_RFC] = {'\"'};
-		quote_candidates_map[(uint8_t)QuoteRule::QUOTES_OTHER] = {'\"', '\''};
-		quote_candidates_map[(uint8_t)QuoteRule::NO_QUOTES] = {'\0'};
+		for (idx_t i = 0; i < default_quote_rule.size(); i++) {
+			quote_candidates_map[static_cast<uint8_t>(default_quote_rule[i])] = {default_quote[i]};
+		}
 	}
-	if (options.dialect_options.state_machine_options.escape.IsSetByUser()) {
+	if (options.escape.IsSetByUser()) {
 		// user provided escape: use that escape rule
-		if (options.dialect_options.state_machine_options.escape == '\0') {
+		if (options.escape == '\0') {
 			quoterule_candidates = {QuoteRule::QUOTES_RFC};
 		} else {
 			quoterule_candidates = {QuoteRule::QUOTES_OTHER};
 		}
-		escape_candidates_map[(uint8_t)quoterule_candidates[0]] = {
-		    options.dialect_options.state_machine_options.escape.GetValue()};
+		escape_candidates_map[static_cast<uint8_t>(quoterule_candidates[0])] = {options.escape.GetValue()};
 	} else {
 		// no escape provided: try standard/common escapes
-		quoterule_candidates = {QuoteRule::QUOTES_RFC, QuoteRule::QUOTES_OTHER, QuoteRule::NO_QUOTES};
+		quoterule_candidates = default_quote_rule;
 	}
 }
 
 void CSVSniffer::GenerateStateMachineSearchSpace(vector<unique_ptr<ColumnCountScanner>> &column_count_scanners,
-                                                 const vector<char> &delimiter_candidates,
-                                                 const vector<QuoteRule> &quoterule_candidates,
-                                                 const unordered_map<uint8_t, vector<char>> &quote_candidates_map,
-                                                 const unordered_map<uint8_t, vector<char>> &escape_candidates_map) {
+                                                 const DialectCandidates &dialect_candidates) {
 	// Generate state machines for all option combinations
 	NewLineIdentifier new_line_id;
 	if (options.dialect_options.state_machine_options.new_line.IsSetByUser()) {
@@ -68,74 +144,159 @@ void CSVSniffer::GenerateStateMachineSearchSpace(vector<unique_ptr<ColumnCountSc
 	} else {
 		new_line_id = DetectNewLineDelimiter(*buffer_manager);
 	}
-	for (const auto quoterule : quoterule_candidates) {
-		const auto &quote_candidates = quote_candidates_map.at((uint8_t)quoterule);
+	CSVIterator first_iterator;
+	bool iterator_set = false;
+	for (const auto quoterule : dialect_candidates.quoterule_candidates) {
+		const auto &quote_candidates = dialect_candidates.quote_candidates_map.at(static_cast<uint8_t>(quoterule));
 		for (const auto &quote : quote_candidates) {
-			for (const auto &delimiter : delimiter_candidates) {
-				const auto &escape_candidates = escape_candidates_map.at((uint8_t)quoterule);
+			for (const auto &delimiter : dialect_candidates.delim_candidates) {
+				const auto &escape_candidates =
+				    dialect_candidates.escape_candidates_map.at(static_cast<uint8_t>(quoterule));
 				for (const auto &escape : escape_candidates) {
-					D_ASSERT(buffer_manager);
-					CSVStateMachineOptions state_machine_options(delimiter, quote, escape, new_line_id);
-					auto sniffing_state_machine =
-					    make_uniq<CSVStateMachine>(options, state_machine_options, state_machine_cache);
-					column_count_scanners.emplace_back(make_uniq<ColumnCountScanner>(
-					    buffer_manager, std::move(sniffing_state_machine), detection_error_handler));
+					for (const auto &comment : dialect_candidates.comment_candidates) {
+						D_ASSERT(buffer_manager);
+						CSVStateMachineOptions state_machine_options(delimiter, quote, escape, comment, new_line_id);
+						auto sniffing_state_machine =
+						    make_shared_ptr<CSVStateMachine>(options, state_machine_options, state_machine_cache);
+						if (options.dialect_options.skip_rows.IsSetByUser()) {
+							if (!iterator_set) {
+								first_iterator = BaseScanner::SkipCSVRows(buffer_manager, sniffing_state_machine,
+								                                          options.dialect_options.skip_rows.GetValue());
+								iterator_set = true;
+							}
+							column_count_scanners.emplace_back(make_uniq<ColumnCountScanner>(
+							    buffer_manager, std::move(sniffing_state_machine), detection_error_handler,
+							    CSVReaderOptions::sniff_size, first_iterator));
+							continue;
+						}
+						column_count_scanners.emplace_back(
+						    make_uniq<ColumnCountScanner>(buffer_manager, std::move(sniffing_state_machine),
+						                                  detection_error_handler, CSVReaderOptions::sniff_size));
+					}
 				}
 			}
 		}
 	}
 }
 
+// Returns true if a comment is acceptable
+bool AreCommentsAcceptable(const ColumnCountResult &result, idx_t num_cols, bool comment_set_by_user) {
+	// For a comment to be acceptable, we want 3/5th's majority of unmatches in the columns
+	constexpr double min_majority = 0.6;
+	// detected comments, are all lines that started with a comment character.
+	double detected_comments = 0;
+	// If at least one comment is a full line comment
+	bool has_full_line_comment = false;
+	// valid comments are all lines where the number of columns does not fit our expected number of columns.
+	double valid_comments = 0;
+	for (idx_t i = 0; i < result.result_position; i++) {
+		if (result.column_counts[i].is_comment || result.column_counts[i].is_mid_comment) {
+			detected_comments++;
+			if (result.column_counts[i].number_of_columns != num_cols && result.column_counts[i].is_comment) {
+				has_full_line_comment = true;
+				valid_comments++;
+			}
+			if (result.column_counts[i].number_of_columns == num_cols && result.column_counts[i].is_mid_comment) {
+				valid_comments++;
+			}
+		}
+	}
+	// If we do not encounter at least one full line comment, we do not consider this comment option.
+	if (valid_comments == 0 || (!has_full_line_comment && !comment_set_by_user)) {
+		// this is only valid if our comment character is \0
+		if (result.state_machine.state_machine_options.comment.GetValue() == '\0') {
+			return true;
+		}
+		return false;
+	}
+
+	return valid_comments / detected_comments >= min_majority;
+}
+
 void CSVSniffer::AnalyzeDialectCandidate(unique_ptr<ColumnCountScanner> scanner, idx_t &rows_read,
-                                         idx_t &best_consistent_rows, idx_t &prev_padding_count) {
+                                         idx_t &best_consistent_rows, idx_t &prev_padding_count,
+                                         idx_t &min_ignored_rows) {
 	// The sniffed_column_counts variable keeps track of the number of columns found for each row
 	auto &sniffed_column_counts = scanner->ParseChunk();
 	idx_t dirty_notes = 0;
+	idx_t dirty_notes_minus_comments = 0;
 	if (sniffed_column_counts.error) {
 		// This candidate has an error (i.e., over maximum line size or never unquoting quoted values)
 		return;
 	}
 	idx_t consistent_rows = 0;
-	idx_t num_cols = sniffed_column_counts.result_position == 0 ? 1 : sniffed_column_counts[0];
+	idx_t num_cols = sniffed_column_counts.result_position == 0 ? 1 : sniffed_column_counts[0].number_of_columns;
 	idx_t padding_count = 0;
+	idx_t comment_rows = 0;
+	idx_t ignored_rows = 0;
 	bool allow_padding = options.null_padding;
+	bool first_valid = false;
 	if (sniffed_column_counts.result_position > rows_read) {
 		rows_read = sniffed_column_counts.result_position;
 	}
 	if (set_columns.IsCandidateUnacceptable(num_cols, options.null_padding, options.ignore_errors.GetValue(),
-	                                        sniffed_column_counts.last_value_always_empty)) {
+	                                        sniffed_column_counts[0].last_value_always_empty)) {
 		// Not acceptable
 		return;
 	}
+	idx_t header_idx = 0;
 	for (idx_t row = 0; row < sniffed_column_counts.result_position; row++) {
-		if (set_columns.IsCandidateUnacceptable(sniffed_column_counts[row], options.null_padding,
+		if (set_columns.IsCandidateUnacceptable(sniffed_column_counts[row].number_of_columns, options.null_padding,
 		                                        options.ignore_errors.GetValue(),
-		                                        sniffed_column_counts.last_value_always_empty)) {
+		                                        sniffed_column_counts[row].last_value_always_empty)) {
 			// Not acceptable
 			return;
 		}
-		if (sniffed_column_counts[row] == num_cols || (options.ignore_errors.GetValue() && !options.null_padding)) {
+		if (sniffed_column_counts[row].is_comment) {
+			comment_rows++;
+		} else if (sniffed_column_counts[row].last_value_always_empty &&
+		           sniffed_column_counts[row].number_of_columns ==
+		               sniffed_column_counts[header_idx].number_of_columns + 1) {
+			// we allow for the first row to miss one column IF last_value_always_empty is true
+			// This is so we can sniff files that have an extra delimiter on the data part.
+			// e.g., C1|C2\n1|2|\n3|4|
 			consistent_rows++;
-		} else if (num_cols < sniffed_column_counts[row] && !options.dialect_options.skip_rows.IsSetByUser() &&
+		} else if (num_cols < sniffed_column_counts[row].number_of_columns &&
+		           (!options.dialect_options.skip_rows.IsSetByUser() || comment_rows > 0) &&
 		           (!set_columns.IsSet() || options.null_padding)) {
 			// all rows up to this point will need padding
+			if (!first_valid) {
+				first_valid = true;
+				sniffed_column_counts.state_machine.dialect_options.rows_until_header = row;
+			}
 			padding_count = 0;
 			// we use the maximum amount of num_cols that we find
-			num_cols = sniffed_column_counts[row];
+			num_cols = sniffed_column_counts[row].number_of_columns;
 			dirty_notes = row;
+			// sniffed_column_counts.state_machine.dialect_options.rows_until_header = dirty_notes;
+			dirty_notes_minus_comments = dirty_notes - comment_rows;
+			header_idx = row;
 			consistent_rows = 1;
-
-		} else if (num_cols >= sniffed_column_counts[row]) {
+		} else if (sniffed_column_counts[row].number_of_columns == num_cols ||
+		           (options.ignore_errors.GetValue() && !options.null_padding)) {
+			if (!first_valid) {
+				first_valid = true;
+				sniffed_column_counts.state_machine.dialect_options.rows_until_header = row;
+			}
+			if (sniffed_column_counts[row].number_of_columns != num_cols) {
+				ignored_rows++;
+			}
+			consistent_rows++;
+		} else if (num_cols >= sniffed_column_counts[row].number_of_columns) {
 			// we are missing some columns, we can parse this as long as we add padding
 			padding_count++;
 		}
 	}
 
+	if (sniffed_column_counts.state_machine.options.dialect_options.skip_rows.IsSetByUser()) {
+		sniffed_column_counts.state_machine.dialect_options.rows_until_header +=
+		    sniffed_column_counts.state_machine.options.dialect_options.skip_rows.GetValue();
+	}
 	// Calculate the total number of consistent rows after adding padding.
 	consistent_rows += padding_count;
 
 	// Whether there are more values (rows) available that are consistent, exceeding the current best.
-	bool more_values = (consistent_rows > best_consistent_rows && num_cols >= max_columns_found);
+	bool more_values = consistent_rows > best_consistent_rows && num_cols >= max_columns_found;
 
 	// If additional padding is required when compared to the previous padding count.
 	bool require_more_padding = padding_count > prev_padding_count;
@@ -148,21 +309,25 @@ void CSVSniffer::AnalyzeDialectCandidate(unique_ptr<ColumnCountScanner> scanner,
 
 	// If the number of rows is consistent with the calculated value after accounting for skipped rows and the
 	// start row.
-	bool rows_consistent = consistent_rows + (dirty_notes - options.dialect_options.skip_rows.GetValue()) ==
-	                       sniffed_column_counts.result_position - options.dialect_options.skip_rows.GetValue();
+	bool rows_consistent =
+	    consistent_rows + (dirty_notes_minus_comments - options.dialect_options.skip_rows.GetValue()) + comment_rows ==
+	    sniffed_column_counts.result_position - options.dialect_options.skip_rows.GetValue();
 	// If there are more than one consistent row.
-	bool more_than_one_row = (consistent_rows > 1);
+	bool more_than_one_row = consistent_rows > 1;
 
 	// If there are more than one column.
-	bool more_than_one_column = (num_cols > 1);
+	bool more_than_one_column = num_cols > 1;
 
 	// If the start position is valid.
 	bool start_good = !candidates.empty() &&
-	                  (dirty_notes <= candidates.front()->GetStateMachine().dialect_options.skip_rows.GetValue());
+	                  dirty_notes <= candidates.front()->GetStateMachine().dialect_options.skip_rows.GetValue();
 
 	// If padding happened but it is not allowed.
 	bool invalid_padding = !allow_padding && padding_count > 0;
 
+	bool comments_are_acceptable = AreCommentsAcceptable(
+	    sniffed_column_counts, num_cols, options.dialect_options.state_machine_options.comment.IsSetByUser());
+
 	// If rows are consistent and no invalid padding happens, this is the best suitable candidate if one of the
 	// following is valid:
 	// - There's a single column before.
@@ -171,7 +336,7 @@ void CSVSniffer::AnalyzeDialectCandidate(unique_ptr<ColumnCountScanner> scanner,
 	if (rows_consistent &&
 	    (single_column_before || (more_values && !require_more_padding) ||
 	     (more_than_one_column && require_less_padding)) &&
-	    !invalid_padding) {
+	    !invalid_padding && comments_are_acceptable) {
 		if (!candidates.empty() && set_columns.IsSet() && max_columns_found == candidates.size()) {
 			// We have a candidate that fits our requirements better
 			return;
@@ -182,18 +347,23 @@ void CSVSniffer::AnalyzeDialectCandidate(unique_ptr<ColumnCountScanner> scanner,
 			// Give preference to quoted boys.
 			return;
 		}
+		if (max_columns_found == num_cols && ignored_rows > min_ignored_rows) {
+			return;
+		}
 
 		best_consistent_rows = consistent_rows;
 		max_columns_found = num_cols;
 		prev_padding_count = padding_count;
+		min_ignored_rows = ignored_rows;
+
 		if (options.dialect_options.skip_rows.IsSetByUser()) {
 			// If skip rows is set by user, and we found dirty notes, we only accept it if either null_padding or
-			// ignore_errors is set
-			if (dirty_notes != 0 && !options.null_padding && !options.ignore_errors.GetValue()) {
+			// ignore_errors is set we have comments
+			if (dirty_notes != 0 && !options.null_padding && !options.ignore_errors.GetValue() && comment_rows == 0) {
 				return;
 			}
 			sniffing_state_machine.dialect_options.skip_rows = options.dialect_options.skip_rows.GetValue();
-		} else if (!options.null_padding && !options.ignore_errors.GetValue()) {
+		} else if (!options.null_padding) {
 			sniffing_state_machine.dialect_options.skip_rows = dirty_notes;
 		}
 
@@ -206,7 +376,7 @@ void CSVSniffer::AnalyzeDialectCandidate(unique_ptr<ColumnCountScanner> scanner,
 	// no additional padding is required, and there is no invalid padding, and there is not yet a candidate
 	// with the same quote, we add this state_machine as a suitable candidate.
 	if (more_than_one_row && more_than_one_column && start_good && rows_consistent && !require_more_padding &&
-	    !invalid_padding && num_cols == max_columns_found) {
+	    !invalid_padding && num_cols == max_columns_found && comments_are_acceptable) {
 		auto &sniffing_state_machine = scanner->GetStateMachine();
 
 		bool same_quote_is_candidate = false;
@@ -224,7 +394,7 @@ void CSVSniffer::AnalyzeDialectCandidate(unique_ptr<ColumnCountScanner> scanner,
 					return;
 				}
 				sniffing_state_machine.dialect_options.skip_rows = options.dialect_options.skip_rows.GetValue();
-			} else if (!options.null_padding && !options.ignore_errors.GetValue()) {
+			} else if (!options.null_padding) {
 				sniffing_state_machine.dialect_options.skip_rows = dirty_notes;
 			}
 
@@ -238,14 +408,13 @@ bool CSVSniffer::RefineCandidateNextChunk(ColumnCountScanner &candidate) {
 	auto &sniffed_column_counts = candidate.ParseChunk();
 	for (idx_t i = 0; i < sniffed_column_counts.result_position; i++) {
 		if (set_columns.IsSet()) {
-			return !set_columns.IsCandidateUnacceptable(sniffed_column_counts[i], options.null_padding,
-			                                            options.ignore_errors.GetValue(),
-			                                            sniffed_column_counts.last_value_always_empty);
-		} else {
-			if (max_columns_found != sniffed_column_counts[i] &&
-			    (!options.null_padding && !options.ignore_errors.GetValue())) {
-				return false;
-			}
+			return !set_columns.IsCandidateUnacceptable(sniffed_column_counts[i].number_of_columns,
+			                                            options.null_padding, options.ignore_errors.GetValue(),
+			                                            sniffed_column_counts[i].last_value_always_empty);
+		}
+		if (max_columns_found != sniffed_column_counts[i].number_of_columns &&
+		    (!options.null_padding && !options.ignore_errors.GetValue() && !sniffed_column_counts[i].is_comment)) {
+			return false;
 		}
 	}
 	return true;
@@ -281,9 +450,8 @@ void CSVSniffer::RefineCandidates() {
 	// that have actually quoted values, otherwise we will choose quotes = \0
 	candidates.clear();
 	if (!successful_candidates.empty()) {
-		unique_ptr<ColumnCountScanner> cc_best_candidate;
 		for (idx_t i = 0; i < successful_candidates.size(); i++) {
-			cc_best_candidate = std::move(successful_candidates[i]);
+			unique_ptr<ColumnCountScanner> cc_best_candidate = std::move(successful_candidates[i]);
 			if (cc_best_candidate->state_machine->state_machine_options.quote != '\0' &&
 			    cc_best_candidate->ever_quoted) {
 				candidates.clear();
@@ -292,9 +460,7 @@ void CSVSniffer::RefineCandidates() {
 			}
 			candidates.push_back(std::move(cc_best_candidate));
 		}
-		return;
 	}
-	return;
 }
 
 NewLineIdentifier CSVSniffer::DetectNewLineDelimiter(CSVBufferManager &buffer_manager) {
@@ -316,25 +482,10 @@ NewLineIdentifier CSVSniffer::DetectNewLineDelimiter(CSVBufferManager &buffer_ma
 	if (carriage_return && n) {
 		return NewLineIdentifier::CARRY_ON;
 	}
-	return NewLineIdentifier::SINGLE;
-}
-
-void CSVSniffer::SkipLines(vector<unique_ptr<ColumnCountScanner>> &csv_state_machines) {
-	if (csv_state_machines.empty()) {
-		return;
-	}
-	auto &first_scanner = *csv_state_machines[0];
-	// We figure out the iterator position for the first scanner
-	if (options.dialect_options.skip_rows.IsSetByUser()) {
-		first_scanner.SkipCSVRows(options.dialect_options.skip_rows.GetValue());
-	}
-	// The iterator position is the same regardless of the scanner configuration, hence we apply the same iterator
-	// To the remaining scanners
-	const auto first_iterator = first_scanner.GetIterator();
-	for (idx_t i = 1; i < csv_state_machines.size(); i++) {
-		auto &cur_scanner = *csv_state_machines[i];
-		cur_scanner.SetIterator(first_iterator);
+	if (carriage_return) {
+		return NewLineIdentifier::SINGLE_R;
 	}
+	return NewLineIdentifier::SINGLE_N;
 }
 
 // Dialect Detection consists of five steps:
@@ -344,43 +495,30 @@ void CSVSniffer::SkipLines(vector<unique_ptr<ColumnCountScanner>> &csv_state_mac
 // 4. Analyze the remaining chunks of the file and find the best dialect candidate
 void CSVSniffer::DetectDialect() {
 	// Variables for Dialect Detection
-	// Candidates for the delimiter
-	vector<char> delim_candidates;
-	// Quote-Rule Candidates
-	vector<QuoteRule> quoterule_candidates;
-	// Candidates for the quote option
-	unordered_map<uint8_t, vector<char>> quote_candidates_map;
-	// Candidates for the escape option
-	unordered_map<uint8_t, vector<char>> escape_candidates_map;
-	escape_candidates_map[(uint8_t)QuoteRule::QUOTES_RFC] = {'\"', '\'', '\0'};
-	escape_candidates_map[(uint8_t)QuoteRule::QUOTES_OTHER] = {'\\'};
-	escape_candidates_map[(uint8_t)QuoteRule::NO_QUOTES] = {'\0'};
+	DialectCandidates dialect_candidates(options.dialect_options.state_machine_options);
 	// Number of rows read
 	idx_t rows_read = 0;
 	// Best Number of consistent rows (i.e., presenting all columns)
 	idx_t best_consistent_rows = 0;
 	// If padding was necessary (i.e., rows are missing some columns, how many)
 	idx_t prev_padding_count = 0;
+	// Min number of ignores rows
+	idx_t best_ignored_rows = 0;
 	// Vector of CSV State Machines
 	vector<unique_ptr<ColumnCountScanner>> csv_state_machines;
-
-	// Step 1: Generate search space
-	GenerateCandidateDetectionSearchSpace(delim_candidates, quoterule_candidates, quote_candidates_map,
-	                                      escape_candidates_map);
-	// Step 2: Generate state machines
-	GenerateStateMachineSearchSpace(csv_state_machines, delim_candidates, quoterule_candidates, quote_candidates_map,
-	                                escape_candidates_map);
-	SkipLines(csv_state_machines);
-	// Step 3: Analyze all candidates on the first chunk
+	// Step 1: Generate state machines
+	GenerateStateMachineSearchSpace(csv_state_machines, dialect_candidates);
+	// Step 2: Analyze all candidates on the first chunk
 	for (auto &state_machine : csv_state_machines) {
-		AnalyzeDialectCandidate(std::move(state_machine), rows_read, best_consistent_rows, prev_padding_count);
+		AnalyzeDialectCandidate(std::move(state_machine), rows_read, best_consistent_rows, prev_padding_count,
+		                        best_ignored_rows);
 	}
-	// Step 4: Loop over candidates and find if they can still produce good results for the remaining chunks
+	// Step 3: Loop over candidates and find if they can still produce good results for the remaining chunks
 	RefineCandidates();
 
 	// if no dialect candidate was found, we throw an exception
 	if (candidates.empty()) {
-		auto error = CSVError::SniffingError(options.file_path);
+		auto error = CSVError::SniffingError(options, dialect_candidates.Print());
 		error_handler->Error(error);
 	}
 }