duckdb 1.1.4-dev2.0 → 1.1.4-dev9.0

package/src/duckdb/src/execution/operator/csv_scanner/state_machine/csv_state_machine_cache.cpp

@@ -22,15 +22,22 @@ void CSVStateMachineCache::Insert(const CSVStateMachineOptions &state_machine_op
 	auto &transition_array = state_machine_cache[state_machine_options];
 
 	for (uint32_t i = 0; i < StateMachine::NUM_STATES; i++) {
-		CSVState cur_state = static_cast<CSVState>(i);
+		const auto cur_state = static_cast<CSVState>(i);
 		switch (cur_state) {
+		case CSVState::MAYBE_QUOTED:
 		case CSVState::QUOTED:
 		case CSVState::QUOTED_NEW_LINE:
 		case CSVState::ESCAPE:
 			InitializeTransitionArray(transition_array, cur_state, CSVState::QUOTED);
 			break;
 		case CSVState::UNQUOTED:
-			InitializeTransitionArray(transition_array, cur_state, CSVState::INVALID);
+			if (state_machine_options.rfc_4180.GetValue()) {
+				// If we have an unquoted state, following rfc 4180, our base state is invalid
+				InitializeTransitionArray(transition_array, cur_state, CSVState::INVALID);
+			} else {
+				// This will allow us to accept unescaped quotes
+				InitializeTransitionArray(transition_array, cur_state, CSVState::UNQUOTED);
+			}
 			break;
 		case CSVState::COMMENT:
 			InitializeTransitionArray(transition_array, cur_state, CSVState::COMMENT);
@@ -41,49 +48,118 @@ void CSVStateMachineCache::Insert(const CSVStateMachineOptions &state_machine_op
 		}
 	}
 
-	uint8_t delimiter = static_cast<uint8_t>(state_machine_options.delimiter.GetValue());
-	uint8_t quote = static_cast<uint8_t>(state_machine_options.quote.GetValue());
-	uint8_t escape = static_cast<uint8_t>(state_machine_options.escape.GetValue());
-	uint8_t comment = static_cast<uint8_t>(state_machine_options.comment.GetValue());
+	const auto delimiter_value = state_machine_options.delimiter.GetValue();
+	const auto delimiter_first_byte = static_cast<uint8_t>(delimiter_value[0]);
+	const auto quote = static_cast<uint8_t>(state_machine_options.quote.GetValue());
+	const auto escape = static_cast<uint8_t>(state_machine_options.escape.GetValue());
+	const auto comment = static_cast<uint8_t>(state_machine_options.comment.GetValue());
+
+	const auto new_line_id = state_machine_options.new_line.GetValue();
 
-	auto new_line_id = state_machine_options.new_line.GetValue();
+	const bool multi_byte_delimiter = delimiter_value.size() != 1;
 
+	const bool enable_unquoted_escape = state_machine_options.rfc_4180.GetValue() == false &&
+	                                    state_machine_options.quote != state_machine_options.escape &&
+	                                    state_machine_options.escape != '\0';
 	// Now set values depending on configuration
 	// 1) Standard/Invalid State
-	vector<uint8_t> std_inv {static_cast<uint8_t>(CSVState::STANDARD), static_cast<uint8_t>(CSVState::INVALID)};
-	for (auto &state : std_inv) {
-		transition_array[delimiter][state] = CSVState::DELIMITER;
-		transition_array[static_cast<uint8_t>('\n')][state] = CSVState::RECORD_SEPARATOR;
+	const vector<uint8_t> std_inv {static_cast<uint8_t>(CSVState::STANDARD), static_cast<uint8_t>(CSVState::INVALID),
+	                               static_cast<uint8_t>(CSVState::STANDARD_NEWLINE)};
+	for (const auto &state : std_inv) {
+		if (multi_byte_delimiter) {
+			transition_array[delimiter_first_byte][state] = CSVState::DELIMITER_FIRST_BYTE;
+		} else {
+			transition_array[delimiter_first_byte][state] = CSVState::DELIMITER;
+		}
 		if (new_line_id == NewLineIdentifier::CARRY_ON) {
 			transition_array[static_cast<uint8_t>('\r')][state] = CSVState::CARRIAGE_RETURN;
+			if (state == static_cast<uint8_t>(CSVState::STANDARD_NEWLINE)) {
+				transition_array[static_cast<uint8_t>('\n')][state] = CSVState::STANDARD;
+			} else if (!state_machine_options.rfc_4180.GetValue()) {
+				transition_array[static_cast<uint8_t>('\n')][state] = CSVState::RECORD_SEPARATOR;
+			} else {
+				transition_array[static_cast<uint8_t>('\n')][state] = CSVState::INVALID;
+			}
 		} else {
 			transition_array[static_cast<uint8_t>('\r')][state] = CSVState::RECORD_SEPARATOR;
+			transition_array[static_cast<uint8_t>('\n')][state] = CSVState::RECORD_SEPARATOR;
 		}
 		if (comment != '\0') {
 			transition_array[comment][state] = CSVState::COMMENT;
 		}
+		if (enable_unquoted_escape) {
+			transition_array[escape][state] = CSVState::UNQUOTED_ESCAPE;
+		}
 	}
 	// 2) Field Separator State
-	transition_array[delimiter][static_cast<uint8_t>(CSVState::DELIMITER)] = CSVState::DELIMITER;
-	transition_array[static_cast<uint8_t>('\n')][static_cast<uint8_t>(CSVState::DELIMITER)] =
-	    CSVState::RECORD_SEPARATOR;
-	if (new_line_id == NewLineIdentifier::CARRY_ON) {
-		transition_array[static_cast<uint8_t>('\r')][static_cast<uint8_t>(CSVState::DELIMITER)] =
-		    CSVState::CARRIAGE_RETURN;
-	} else {
-		transition_array[static_cast<uint8_t>('\r')][static_cast<uint8_t>(CSVState::DELIMITER)] =
-		    CSVState::RECORD_SEPARATOR;
+	if (quote != '\0') {
+		transition_array[quote][static_cast<uint8_t>(CSVState::DELIMITER)] = CSVState::QUOTED;
 	}
-	transition_array[quote][static_cast<uint8_t>(CSVState::DELIMITER)] = CSVState::QUOTED;
-	if (delimiter != ' ') {
+	if (delimiter_first_byte != ' ') {
 		transition_array[' '][static_cast<uint8_t>(CSVState::DELIMITER)] = CSVState::EMPTY_SPACE;
 	}
-	if (comment != '\0') {
-		transition_array[comment][static_cast<uint8_t>(CSVState::DELIMITER)] = CSVState::COMMENT;
+
+	const vector<uint8_t> delimiter_states {
+	    static_cast<uint8_t>(CSVState::DELIMITER), static_cast<uint8_t>(CSVState::DELIMITER_FIRST_BYTE),
+	    static_cast<uint8_t>(CSVState::DELIMITER_SECOND_BYTE), static_cast<uint8_t>(CSVState::DELIMITER_THIRD_BYTE)};
+
+	// These are the same transitions for all delimiter states
+	for (auto &state : delimiter_states) {
+		if (multi_byte_delimiter) {
+			transition_array[delimiter_first_byte][state] = CSVState::DELIMITER_FIRST_BYTE;
+		} else {
+			transition_array[delimiter_first_byte][state] = CSVState::DELIMITER;
+		}
+		transition_array[static_cast<uint8_t>('\n')][state] = CSVState::RECORD_SEPARATOR;
+		if (new_line_id == NewLineIdentifier::CARRY_ON) {
+			transition_array[static_cast<uint8_t>('\r')][state] = CSVState::CARRIAGE_RETURN;
+		} else {
+			transition_array[static_cast<uint8_t>('\r')][state] = CSVState::RECORD_SEPARATOR;
+		}
+		if (comment != '\0') {
+			transition_array[comment][static_cast<uint8_t>(CSVState::DELIMITER)] = CSVState::COMMENT;
+		}
+	}
+	// Deal other multi-byte delimiters
+	if (delimiter_value.size() == 2) {
+		transition_array[static_cast<uint8_t>(delimiter_value[1])]
+		                [static_cast<uint8_t>(CSVState::DELIMITER_FIRST_BYTE)] = CSVState::DELIMITER;
+	} else if (delimiter_value.size() == 3) {
+		if (delimiter_value[0] == delimiter_value[1]) {
+			transition_array[static_cast<uint8_t>(delimiter_value[1])]
+			                [static_cast<uint8_t>(CSVState::DELIMITER_SECOND_BYTE)] = CSVState::DELIMITER_SECOND_BYTE;
+		}
+		transition_array[static_cast<uint8_t>(delimiter_value[1])]
+		                [static_cast<uint8_t>(CSVState::DELIMITER_FIRST_BYTE)] = CSVState::DELIMITER_SECOND_BYTE;
+		transition_array[static_cast<uint8_t>(delimiter_value[2])]
+		                [static_cast<uint8_t>(CSVState::DELIMITER_SECOND_BYTE)] = CSVState::DELIMITER;
+	} else if (delimiter_value.size() == 4) {
+		if (delimiter_value[0] == delimiter_value[2]) {
+			transition_array[static_cast<uint8_t>(delimiter_value[1])]
+			                [static_cast<uint8_t>(CSVState::DELIMITER_THIRD_BYTE)] = CSVState::DELIMITER_SECOND_BYTE;
+		}
+		if (delimiter_value[0] == delimiter_value[1] && delimiter_value[1] == delimiter_value[2]) {
+			transition_array[static_cast<uint8_t>(delimiter_value[1])]
+			                [static_cast<uint8_t>(CSVState::DELIMITER_THIRD_BYTE)] = CSVState::DELIMITER_THIRD_BYTE;
+		}
+		transition_array[static_cast<uint8_t>(delimiter_value[1])]
+		                [static_cast<uint8_t>(CSVState::DELIMITER_FIRST_BYTE)] = CSVState::DELIMITER_SECOND_BYTE;
+		transition_array[static_cast<uint8_t>(delimiter_value[2])]
+		                [static_cast<uint8_t>(CSVState::DELIMITER_SECOND_BYTE)] = CSVState::DELIMITER_THIRD_BYTE;
+		transition_array[static_cast<uint8_t>(delimiter_value[3])]
+		                [static_cast<uint8_t>(CSVState::DELIMITER_THIRD_BYTE)] = CSVState::DELIMITER;
+	}
+	if (enable_unquoted_escape) {
+		transition_array[escape][static_cast<uint8_t>(CSVState::DELIMITER)] = CSVState::UNQUOTED_ESCAPE;
 	}
 
 	// 3) Record Separator State
-	transition_array[delimiter][static_cast<uint8_t>(CSVState::RECORD_SEPARATOR)] = CSVState::DELIMITER;
+	if (multi_byte_delimiter) {
+		transition_array[delimiter_first_byte][static_cast<uint8_t>(CSVState::RECORD_SEPARATOR)] =
+		    CSVState::DELIMITER_FIRST_BYTE;
+	} else {
+		transition_array[delimiter_first_byte][static_cast<uint8_t>(CSVState::RECORD_SEPARATOR)] = CSVState::DELIMITER;
+	}
 	transition_array[static_cast<uint8_t>('\n')][static_cast<uint8_t>(CSVState::RECORD_SEPARATOR)] =
 	    CSVState::RECORD_SEPARATOR;
 	if (new_line_id == NewLineIdentifier::CARRY_ON) {
@@ -93,33 +169,44 @@ void CSVStateMachineCache::Insert(const CSVStateMachineOptions &state_machine_op
 		transition_array[static_cast<uint8_t>('\r')][static_cast<uint8_t>(CSVState::RECORD_SEPARATOR)] =
 		    CSVState::RECORD_SEPARATOR;
 	}
-	transition_array[quote][static_cast<uint8_t>(CSVState::RECORD_SEPARATOR)] = CSVState::QUOTED;
-	if (delimiter != ' ') {
+	if (quote != '\0') {
+		transition_array[quote][static_cast<uint8_t>(CSVState::RECORD_SEPARATOR)] = CSVState::QUOTED;
+	}
+	if (delimiter_first_byte != ' ') {
 		transition_array[' '][static_cast<uint8_t>(CSVState::RECORD_SEPARATOR)] = CSVState::EMPTY_SPACE;
 	}
 	if (comment != '\0') {
 		transition_array[comment][static_cast<uint8_t>(CSVState::RECORD_SEPARATOR)] = CSVState::COMMENT;
 	}
+	if (enable_unquoted_escape) {
+		transition_array[escape][static_cast<uint8_t>(CSVState::RECORD_SEPARATOR)] = CSVState::UNQUOTED_ESCAPE;
+	}
 
 	// 4) Carriage Return State
 	transition_array[static_cast<uint8_t>('\n')][static_cast<uint8_t>(CSVState::CARRIAGE_RETURN)] =
 	    CSVState::RECORD_SEPARATOR;
 	transition_array[static_cast<uint8_t>('\r')][static_cast<uint8_t>(CSVState::CARRIAGE_RETURN)] =
 	    CSVState::CARRIAGE_RETURN;
-	transition_array[quote][static_cast<uint8_t>(CSVState::CARRIAGE_RETURN)] = CSVState::QUOTED;
-	if (delimiter != ' ') {
+	if (quote != '\0') {
+		transition_array[quote][static_cast<uint8_t>(CSVState::CARRIAGE_RETURN)] = CSVState::QUOTED;
+	}
+	if (delimiter_first_byte != ' ') {
 		transition_array[' '][static_cast<uint8_t>(CSVState::CARRIAGE_RETURN)] = CSVState::EMPTY_SPACE;
 	}
 	if (comment != '\0') {
 		transition_array[comment][static_cast<uint8_t>(CSVState::CARRIAGE_RETURN)] = CSVState::COMMENT;
 	}
+	if (enable_unquoted_escape) {
+		transition_array[escape][static_cast<uint8_t>(CSVState::CARRIAGE_RETURN)] = CSVState::UNQUOTED_ESCAPE;
+	}
 
 	// 5) Quoted State
 	transition_array[quote][static_cast<uint8_t>(CSVState::QUOTED)] = CSVState::UNQUOTED;
 	transition_array['\n'][static_cast<uint8_t>(CSVState::QUOTED)] = CSVState::QUOTED_NEW_LINE;
 	transition_array['\r'][static_cast<uint8_t>(CSVState::QUOTED)] = CSVState::QUOTED_NEW_LINE;
 
-	if (state_machine_options.quote != state_machine_options.escape) {
+	if (state_machine_options.quote != state_machine_options.escape &&
+	    state_machine_options.escape.GetValue() != '\0') {
 		transition_array[escape][static_cast<uint8_t>(CSVState::QUOTED)] = CSVState::ESCAPE;
 	}
 	// 6) Unquoted State
@@ -131,19 +218,38 @@ void CSVStateMachineCache::Insert(const CSVStateMachineOptions &state_machine_op
 		transition_array[static_cast<uint8_t>('\r')][static_cast<uint8_t>(CSVState::UNQUOTED)] =
 		    CSVState::RECORD_SEPARATOR;
 	}
-	transition_array[delimiter][static_cast<uint8_t>(CSVState::UNQUOTED)] = CSVState::DELIMITER;
+	if (multi_byte_delimiter) {
+		transition_array[delimiter_first_byte][static_cast<uint8_t>(CSVState::UNQUOTED)] =
+		    CSVState::DELIMITER_FIRST_BYTE;
+	} else {
+		transition_array[delimiter_first_byte][static_cast<uint8_t>(CSVState::UNQUOTED)] = CSVState::DELIMITER;
+	}
 	if (state_machine_options.quote == state_machine_options.escape) {
-		transition_array[escape][static_cast<uint8_t>(CSVState::UNQUOTED)] = CSVState::QUOTED;
+		transition_array[quote][static_cast<uint8_t>(CSVState::UNQUOTED)] = CSVState::QUOTED;
+	}
+	if (state_machine_options.rfc_4180 == false) {
+		if (escape == '\0') {
+			// If escape is defined, it limits a bit how relaxed quotes can be in a reliable way.
+			transition_array[quote][static_cast<uint8_t>(CSVState::UNQUOTED)] = CSVState::MAYBE_QUOTED;
+		} else {
+			transition_array[quote][static_cast<uint8_t>(CSVState::UNQUOTED)] = CSVState::QUOTED;
+		}
 	}
 	if (comment != '\0') {
 		transition_array[comment][static_cast<uint8_t>(CSVState::UNQUOTED)] = CSVState::COMMENT;
 	}
-	// 7) Escaped State
-	transition_array[quote][static_cast<uint8_t>(CSVState::ESCAPE)] = CSVState::QUOTED;
-	transition_array[escape][static_cast<uint8_t>(CSVState::ESCAPE)] = CSVState::QUOTED;
+	if (delimiter_first_byte != ' ' && quote != ' ' && escape != ' ' && comment != ' ') {
+		// If space is not a special character, we can safely ignore it in an unquoted state
+		transition_array[' '][static_cast<uint8_t>(CSVState::UNQUOTED)] = CSVState::UNQUOTED;
+	}
 
 	// 8) Not Set
-	transition_array[delimiter][static_cast<uint8_t>(CSVState::NOT_SET)] = CSVState::DELIMITER;
+	if (multi_byte_delimiter) {
+		transition_array[delimiter_first_byte][static_cast<uint8_t>(CSVState::NOT_SET)] =
+		    CSVState::DELIMITER_FIRST_BYTE;
+	} else {
+		transition_array[delimiter_first_byte][static_cast<uint8_t>(CSVState::NOT_SET)] = CSVState::DELIMITER;
+	}
 	transition_array[static_cast<uint8_t>('\n')][static_cast<uint8_t>(CSVState::NOT_SET)] = CSVState::RECORD_SEPARATOR;
 	if (new_line_id == NewLineIdentifier::CARRY_ON) {
 		transition_array[static_cast<uint8_t>('\r')][static_cast<uint8_t>(CSVState::NOT_SET)] =
@@ -152,21 +258,33 @@ void CSVStateMachineCache::Insert(const CSVStateMachineOptions &state_machine_op
 		transition_array[static_cast<uint8_t>('\r')][static_cast<uint8_t>(CSVState::NOT_SET)] =
 		    CSVState::RECORD_SEPARATOR;
 	}
-	transition_array[quote][static_cast<uint8_t>(CSVState::NOT_SET)] = CSVState::QUOTED;
-	if (delimiter != ' ') {
+	if (quote != '\0') {
+		transition_array[quote][static_cast<uint8_t>(CSVState::NOT_SET)] = CSVState::QUOTED;
+	}
+	if (delimiter_first_byte != ' ') {
 		transition_array[' '][static_cast<uint8_t>(CSVState::NOT_SET)] = CSVState::EMPTY_SPACE;
 	}
 	if (comment != '\0') {
 		transition_array[comment][static_cast<uint8_t>(CSVState::NOT_SET)] = CSVState::COMMENT;
 	}
+	if (enable_unquoted_escape) {
+		transition_array[escape][static_cast<uint8_t>(CSVState::NOT_SET)] = CSVState::UNQUOTED_ESCAPE;
+	}
+
 	// 9) Quoted NewLine
 	transition_array[quote][static_cast<uint8_t>(CSVState::QUOTED_NEW_LINE)] = CSVState::UNQUOTED;
-	if (state_machine_options.quote != state_machine_options.escape) {
+	if (state_machine_options.quote != state_machine_options.escape &&
+	    state_machine_options.escape.GetValue() != '\0') {
 		transition_array[escape][static_cast<uint8_t>(CSVState::QUOTED_NEW_LINE)] = CSVState::ESCAPE;
 	}
 
 	// 10) Empty Value State (Not first value)
-	transition_array[delimiter][static_cast<uint8_t>(CSVState::EMPTY_SPACE)] = CSVState::DELIMITER;
+	if (multi_byte_delimiter) {
+		transition_array[delimiter_first_byte][static_cast<uint8_t>(CSVState::EMPTY_SPACE)] =
+		    CSVState::DELIMITER_FIRST_BYTE;
+	} else {
+		transition_array[delimiter_first_byte][static_cast<uint8_t>(CSVState::EMPTY_SPACE)] = CSVState::DELIMITER;
+	}
 	transition_array[static_cast<uint8_t>('\n')][static_cast<uint8_t>(CSVState::EMPTY_SPACE)] =
 	    CSVState::RECORD_SEPARATOR;
 	if (new_line_id == NewLineIdentifier::CARRY_ON) {
@@ -176,10 +294,15 @@ void CSVStateMachineCache::Insert(const CSVStateMachineOptions &state_machine_op
 		transition_array[static_cast<uint8_t>('\r')][static_cast<uint8_t>(CSVState::EMPTY_SPACE)] =
 		    CSVState::RECORD_SEPARATOR;
 	}
-	transition_array[quote][static_cast<uint8_t>(CSVState::EMPTY_SPACE)] = CSVState::QUOTED;
+	if (quote != '\0') {
+		transition_array[quote][static_cast<uint8_t>(CSVState::EMPTY_SPACE)] = CSVState::QUOTED;
+	}
 	if (comment != '\0') {
 		transition_array[comment][static_cast<uint8_t>(CSVState::EMPTY_SPACE)] = CSVState::COMMENT;
 	}
+	if (enable_unquoted_escape) {
+		transition_array[escape][static_cast<uint8_t>(CSVState::EMPTY_SPACE)] = CSVState::UNQUOTED_ESCAPE;
+	}
 
 	// 11) Comment State
 	transition_array[static_cast<uint8_t>('\n')][static_cast<uint8_t>(CSVState::COMMENT)] = CSVState::RECORD_SEPARATOR;
@@ -191,6 +314,52 @@ void CSVStateMachineCache::Insert(const CSVStateMachineOptions &state_machine_op
 		    CSVState::RECORD_SEPARATOR;
 	}
 
+	// 12) Unquoted Escape State
+	if (enable_unquoted_escape) {
+		// Any character can be escaped, so default to STANDARD
+		if (new_line_id == NewLineIdentifier::CARRY_ON) {
+			transition_array[static_cast<uint8_t>('\r')][static_cast<uint8_t>(CSVState::UNQUOTED_ESCAPE)] =
+			    CSVState::ESCAPED_RETURN;
+		}
+	}
+
+	// 13) Escaped Return State
+	if (enable_unquoted_escape) {
+		// The new state is STANDARD for \r + \n and \r + ordinary character.
+		// Other special characters need to be handled.
+		transition_array[delimiter_first_byte][static_cast<uint8_t>(CSVState::ESCAPED_RETURN)] = CSVState::DELIMITER;
+		if (new_line_id == NewLineIdentifier::CARRY_ON) {
+			transition_array[static_cast<uint8_t>('\r')][static_cast<uint8_t>(CSVState::ESCAPED_RETURN)] =
+			    CSVState::CARRIAGE_RETURN;
+		} else {
+			transition_array[static_cast<uint8_t>('\r')][static_cast<uint8_t>(CSVState::ESCAPED_RETURN)] =
+			    CSVState::RECORD_SEPARATOR;
+		}
+		if (comment != '\0') {
+			transition_array[comment][static_cast<uint8_t>(CSVState::ESCAPED_RETURN)] = CSVState::COMMENT;
+		}
+		transition_array[escape][static_cast<uint8_t>(CSVState::ESCAPED_RETURN)] = CSVState::UNQUOTED_ESCAPE;
+	}
+
+	// 14) Maybe quoted
+	transition_array[quote][static_cast<uint8_t>(CSVState::MAYBE_QUOTED)] = CSVState::MAYBE_QUOTED;
+
+	transition_array[static_cast<uint8_t>('\n')][static_cast<uint8_t>(CSVState::MAYBE_QUOTED)] =
+	    CSVState::RECORD_SEPARATOR;
+	if (new_line_id == NewLineIdentifier::CARRY_ON) {
+		transition_array[static_cast<uint8_t>('\r')][static_cast<uint8_t>(CSVState::MAYBE_QUOTED)] =
+		    CSVState::CARRIAGE_RETURN;
+	} else {
+		transition_array[static_cast<uint8_t>('\r')][static_cast<uint8_t>(CSVState::MAYBE_QUOTED)] =
+		    CSVState::RECORD_SEPARATOR;
+	}
+	if (multi_byte_delimiter) {
+		transition_array[delimiter_first_byte][static_cast<uint8_t>(CSVState::MAYBE_QUOTED)] =
+		    CSVState::DELIMITER_FIRST_BYTE;
+	} else {
+		transition_array[delimiter_first_byte][static_cast<uint8_t>(CSVState::MAYBE_QUOTED)] = CSVState::DELIMITER;
+	}
+
 	// Initialize characters we can skip during processing, for Standard and Quoted states
 	for (idx_t i = 0; i < StateMachine::NUM_TRANSITIONS; i++) {
 		transition_array.skip_standard[i] = true;
@@ -198,10 +367,13 @@ void CSVStateMachineCache::Insert(const CSVStateMachineOptions &state_machine_op
 		transition_array.skip_comment[i] = true;
 	}
 	// For standard states we only care for delimiters \r and \n
-	transition_array.skip_standard[delimiter] = false;
+	transition_array.skip_standard[delimiter_first_byte] = false;
 	transition_array.skip_standard[static_cast<uint8_t>('\n')] = false;
 	transition_array.skip_standard[static_cast<uint8_t>('\r')] = false;
 	transition_array.skip_standard[comment] = false;
+	if (enable_unquoted_escape) {
+		transition_array.skip_standard[escape] = false;
+	}
 
 	// For quoted we only care about quote, escape and for delimiters \r and \n
 	transition_array.skip_quoted[quote] = false;
@@ -212,7 +384,7 @@ void CSVStateMachineCache::Insert(const CSVStateMachineOptions &state_machine_op
 	transition_array.skip_comment[static_cast<uint8_t>('\r')] = false;
 	transition_array.skip_comment[static_cast<uint8_t>('\n')] = false;
 
-	transition_array.delimiter = delimiter;
+	transition_array.delimiter = delimiter_first_byte;
 	transition_array.new_line = static_cast<uint8_t>('\n');
 	transition_array.carriage_return = static_cast<uint8_t>('\r');
 	transition_array.quote = quote;
@@ -241,9 +413,11 @@ CSVStateMachineCache::CSVStateMachineCache() {
 				const auto &escape_candidates = default_escape[static_cast<uint8_t>(quote_rule)];
 				for (const auto &escape : escape_candidates) {
 					for (const auto &comment : default_comment) {
-						Insert({delimiter, quote, escape, comment, NewLineIdentifier::SINGLE_N});
-						Insert({delimiter, quote, escape, comment, NewLineIdentifier::SINGLE_R});
-						Insert({delimiter, quote, escape, comment, NewLineIdentifier::CARRY_ON});
+						for (const bool rfc_4180 : {true, false}) {
+							Insert({delimiter, quote, escape, comment, NewLineIdentifier::SINGLE_N, rfc_4180});
+							Insert({delimiter, quote, escape, comment, NewLineIdentifier::SINGLE_R, rfc_4180});
+							Insert({delimiter, quote, escape, comment, NewLineIdentifier::CARRY_ON, rfc_4180});
+						}
 					}
 				}
 			}

package/src/duckdb/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp

@@ -10,37 +10,38 @@ CSVUnionData::~CSVUnionData() {
 
 CSVFileScan::CSVFileScan(ClientContext &context, shared_ptr<CSVBufferManager> buffer_manager_p,
                          shared_ptr<CSVStateMachine> state_machine_p, const CSVReaderOptions &options_p,
-                         const ReadCSVData &bind_data, const vector<column_t> &column_ids, CSVSchema &file_schema)
+                         const ReadCSVData &bind_data, const vector<ColumnIndex> &column_ids, CSVSchema &file_schema)
     : file_path(options_p.file_path), file_idx(0), buffer_manager(std::move(buffer_manager_p)),
       state_machine(std::move(state_machine_p)), file_size(buffer_manager->file_handle->FileSize()),
       error_handler(make_shared_ptr<CSVErrorHandler>(options_p.ignore_errors.GetValue())),
       on_disk_file(buffer_manager->file_handle->OnDiskFile()), options(options_p) {
 
+	auto global_columns =
+	    MultiFileReaderColumnDefinition::ColumnsFromNamesAndTypes(bind_data.return_names, bind_data.return_types);
+
 	auto multi_file_reader = MultiFileReader::CreateDefault("CSV Scan");
 	if (bind_data.initial_reader.get()) {
 		auto &union_reader = *bind_data.initial_reader;
-		names = union_reader.GetNames();
+		SetNamesAndTypes(union_reader.GetNames(), union_reader.GetTypes());
 		options = union_reader.options;
-		types = union_reader.GetTypes();
-		multi_file_reader->InitializeReader(*this, options.file_options, bind_data.reader_bind, bind_data.return_types,
-		                                    bind_data.return_names, column_ids, nullptr, file_path, context, nullptr);
+		multi_file_reader->InitializeReader(*this, options.file_options, bind_data.reader_bind, global_columns,
+		                                    column_ids, nullptr, file_path, context, nullptr);
 		InitializeFileNamesTypes();
 		return;
 	}
 	if (!bind_data.column_info.empty()) {
 		// Serialized Union By name
-		names = bind_data.column_info[0].names;
-		types = bind_data.column_info[0].types;
-		multi_file_reader->InitializeReader(*this, options.file_options, bind_data.reader_bind, bind_data.return_types,
-		                                    bind_data.return_names, column_ids, nullptr, file_path, context, nullptr);
+		SetNamesAndTypes(bind_data.column_info[0].names, bind_data.column_info[0].types);
+		multi_file_reader->InitializeReader(*this, options.file_options, bind_data.reader_bind, global_columns,
+		                                    column_ids, nullptr, file_path, context, nullptr);
 		InitializeFileNamesTypes();
 		return;
 	}
-	names = bind_data.csv_names;
-	types = bind_data.csv_types;
+	SetNamesAndTypes(bind_data.csv_names, bind_data.csv_types);
+
 	file_schema.Initialize(names, types, file_path);
-	multi_file_reader->InitializeReader(*this, options.file_options, bind_data.reader_bind, bind_data.return_types,
-	                                    bind_data.return_names, column_ids, nullptr, file_path, context, nullptr);
+	multi_file_reader->InitializeReader(*this, options.file_options, bind_data.reader_bind, global_columns, column_ids,
+	                                    nullptr, file_path, context, nullptr);
 
 	InitializeFileNamesTypes();
 	SetStart();
@@ -59,12 +60,22 @@ void CSVFileScan::SetStart() {
 	start_iterator = skip_scanner.GetIterator();
 }
 
+void CSVFileScan::SetNamesAndTypes(const vector<string> &names_p, const vector<LogicalType> &types_p) {
+	names = names_p;
+	types = types_p;
+	columns = MultiFileReaderColumnDefinition::ColumnsFromNamesAndTypes(names, types);
+}
+
 CSVFileScan::CSVFileScan(ClientContext &context, const string &file_path_p, const CSVReaderOptions &options_p,
-                         const idx_t file_idx_p, const ReadCSVData &bind_data, const vector<column_t> &column_ids,
+                         idx_t file_idx_p, const ReadCSVData &bind_data, const vector<ColumnIndex> &column_ids,
                          CSVSchema &file_schema, bool per_file_single_threaded)
     : file_path(file_path_p), file_idx(file_idx_p),
       error_handler(make_shared_ptr<CSVErrorHandler>(options_p.ignore_errors.GetValue())), options(options_p) {
 	auto multi_file_reader = MultiFileReader::CreateDefault("CSV Scan");
+
+	auto global_columns =
+	    MultiFileReaderColumnDefinition::ColumnsFromNamesAndTypes(bind_data.return_names, bind_data.return_types);
+
 	if (file_idx == 0 && bind_data.initial_reader) {
 		auto &union_reader = *bind_data.initial_reader;
 		// Initialize Buffer Manager
@@ -72,12 +83,11 @@ CSVFileScan::CSVFileScan(ClientContext &context, const string &file_path_p, cons
 		// Initialize On Disk and Size of file
 		on_disk_file = union_reader.on_disk_file;
 		file_size = union_reader.file_size;
-		names = union_reader.GetNames();
 		options = union_reader.options;
-		types = union_reader.GetTypes();
+		SetNamesAndTypes(union_reader.GetNames(), union_reader.GetTypes());
 		state_machine = union_reader.state_machine;
-		multi_file_reader->InitializeReader(*this, options.file_options, bind_data.reader_bind, bind_data.return_types,
-		                                    bind_data.return_names, column_ids, nullptr, file_path, context, nullptr);
+		multi_file_reader->InitializeReader(*this, options.file_options, bind_data.reader_bind, global_columns,
+		                                    column_ids, nullptr, file_path, context, nullptr);
 
 		InitializeFileNamesTypes();
 		SetStart();
@@ -94,8 +104,7 @@ CSVFileScan::CSVFileScan(ClientContext &context, const string &file_path_p, cons
 
 	if (file_idx < bind_data.column_info.size()) {
 		// (Serialized) Union By name
-		names = bind_data.column_info[file_idx].names;
-		types = bind_data.column_info[file_idx].types;
+		SetNamesAndTypes(bind_data.column_info[file_idx].names, bind_data.column_info[file_idx].types);
 		if (file_idx < bind_data.union_readers.size()) {
 			// union readers - use cached options
 			D_ASSERT(names == bind_data.union_readers[file_idx]->names);
@@ -112,26 +121,24 @@ CSVFileScan::CSVFileScan(ClientContext &context, const string &file_path_p, cons
 		state_machine = make_shared_ptr<CSVStateMachine>(
 		    state_machine_cache.Get(options.dialect_options.state_machine_options), options);
 
-		multi_file_reader->InitializeReader(*this, options.file_options, bind_data.reader_bind, bind_data.return_types,
-		                                    bind_data.return_names, column_ids, nullptr, file_path, context, nullptr);
+		multi_file_reader->InitializeReader(*this, options.file_options, bind_data.reader_bind, global_columns,
+		                                    column_ids, nullptr, file_path, context, nullptr);
 		InitializeFileNamesTypes();
 		SetStart();
 		return;
 	}
 	// Sniff it!
-	names = bind_data.csv_names;
-	types = bind_data.csv_types;
+	SetNamesAndTypes(bind_data.csv_names, bind_data.csv_types);
 	if (options.auto_detect && bind_data.files.size() > 1) {
 		if (file_schema.Empty()) {
 			CSVSniffer sniffer(options, buffer_manager, state_machine_cache);
 			auto result = sniffer.SniffCSV();
-			file_schema.Initialize(result.names, result.return_types, options.file_path);
+			file_schema.Initialize(bind_data.csv_names, bind_data.csv_types, options.file_path);
 		} else if (file_idx > 0 && buffer_manager->file_handle->FileSize() > 0) {
 			options.file_path = file_path;
 			CSVSniffer sniffer(options, buffer_manager, state_machine_cache, false);
 			auto result = sniffer.AdaptiveSniff(file_schema);
-			names = result.names;
-			types = result.return_types;
+			SetNamesAndTypes(result.names, result.return_types);
 		}
 	}
 	if (options.dialect_options.num_cols == 0) {
@@ -143,8 +150,8 @@ CSVFileScan::CSVFileScan(ClientContext &context, const string &file_path_p, cons
 	}
 	state_machine = make_shared_ptr<CSVStateMachine>(
 	    state_machine_cache.Get(options.dialect_options.state_machine_options), options);
-	multi_file_reader->InitializeReader(*this, options.file_options, bind_data.reader_bind, bind_data.return_types,
-	                                    bind_data.return_names, column_ids, nullptr, file_path, context, nullptr);
+	multi_file_reader->InitializeReader(*this, options.file_options, bind_data.reader_bind, global_columns, column_ids,
+	                                    nullptr, file_path, context, nullptr);
 	InitializeFileNamesTypes();
 	SetStart();
 }
@@ -159,12 +166,12 @@ CSVFileScan::CSVFileScan(ClientContext &context, const string &file_name, const
 	// Sniff it (We only really care about dialect detection, if types or number of columns are different this will
 	// error out during scanning)
 	auto &state_machine_cache = CSVStateMachineCache::Get(context);
-	if (options.auto_detect && options.dialect_options.num_cols == 0) {
+	// We sniff file if it has not been sniffed yet and either auto-detect is on, or union by name is on
+	if ((options.auto_detect || options.file_options.union_by_name) && options.dialect_options.num_cols == 0) {
 		CSVSniffer sniffer(options, buffer_manager, state_machine_cache);
 		auto sniffer_result = sniffer.SniffCSV();
 		if (names.empty()) {
-			names = sniffer_result.names;
-			types = sniffer_result.return_types;
+			SetNamesAndTypes(sniffer_result.names, sniffer_result.return_types);
 		}
 	}
 	if (options.dialect_options.num_cols == 0) {
@@ -225,6 +232,10 @@ const vector<string> &CSVFileScan::GetNames() {
 const vector<LogicalType> &CSVFileScan::GetTypes() {
 	return types;
 }
+const vector<MultiFileReaderColumnDefinition> &CSVFileScan::GetColumns() {
+	D_ASSERT(types.size() == columns.size());
+	return columns;
+}
 
 void CSVFileScan::InitializeProjection() {
 	for (idx_t i = 0; i < options.dialect_options.num_cols; i++) {