duckdb 1.1.2-dev6.0 → 1.1.4-dev11.0

package/src/duckdb/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp

@@ -1,5 +1,4 @@
 #include "duckdb/execution/operator/csv_scanner/global_csv_state.hpp"
-
 #include "duckdb/execution/operator/csv_scanner/sniffer/csv_sniffer.hpp"
 #include "duckdb/execution/operator/csv_scanner/scanner_boundary.hpp"
 #include "duckdb/execution/operator/csv_scanner/skip_scanner.hpp"
@@ -11,7 +10,7 @@ namespace duckdb {
 
 CSVGlobalState::CSVGlobalState(ClientContext &context_p, const shared_ptr<CSVBufferManager> &buffer_manager,
                                const CSVReaderOptions &options, idx_t system_threads_p, const vector<string> &files,
-                               vector<column_t> column_ids_p, const ReadCSVData &bind_data_p)
+                               vector<ColumnIndex> column_ids_p, const ReadCSVData &bind_data_p)
     : context(context_p), system_threads(system_threads_p), column_ids(std::move(column_ids_p)),
       sniffer_mismatch_error(options.sniffer_user_mismatch_error), bind_data(bind_data_p) {
 
@@ -25,15 +24,21 @@ CSVGlobalState::CSVGlobalState(ClientContext &context_p, const shared_ptr<CSVBuf
 		// If not we need to construct it for the first file
 		file_scans.emplace_back(
 		    make_uniq<CSVFileScan>(context, files[0], options, 0U, bind_data, column_ids, file_schema, false));
-	};
+	}
+	idx_t cur_file_idx = 0;
+	while (file_scans.back()->start_iterator.done && file_scans.size() < files.size()) {
+		cur_file_idx++;
+		file_scans.emplace_back(make_uniq<CSVFileScan>(context, files[cur_file_idx], options, cur_file_idx, bind_data,
+		                                               column_ids, file_schema, false));
+	}
 	// There are situations where we only support single threaded scanning
 	bool many_csv_files = files.size() > 1 && files.size() > system_threads * 2;
 	single_threaded = many_csv_files || !options.parallel;
 	last_file_idx = 0;
 	scanner_idx = 0;
-	running_threads = MaxThreads();
+	running_threads = CSVGlobalState::MaxThreads();
 	current_boundary = file_scans.back()->start_iterator;
-	current_boundary.SetCurrentBoundaryToPosition(single_threaded);
+	current_boundary.SetCurrentBoundaryToPosition(single_threaded, options);
 	if (current_boundary.done && context.client_data->debug_set_max_line_length) {
 		context.client_data->debug_max_line_length = current_boundary.pos.buffer_pos;
 	}
@@ -54,7 +59,7 @@ double CSVGlobalState::GetProgress(const ReadCSVData &bind_data_p) const {
 	if (file_scans.front()->file_size == 0) {
 		percentage = 1.0;
 	} else {
-		// for compressed files, readed bytes may greater than files size.
+		// for compressed files, read bytes may greater than files size.
 		for (auto &file : file_scans) {
 			double file_progress;
 			if (!file->buffer_manager) {
@@ -68,14 +73,30 @@ double CSVGlobalState::GetProgress(const ReadCSVData &bind_data_p) const {
 				file_progress = static_cast<double>(file->bytes_read);
 			}
 			// This file is an uncompressed file, so we use the more price bytes_read from the scanner
-			percentage += (double(1) / double(total_files)) * std::min(1.0, file_progress / double(file->file_size));
+			percentage += (static_cast<double>(1) / static_cast<double>(total_files)) *
+			              std::min(1.0, file_progress / static_cast<double>(file->file_size));
 		}
 	}
 	return percentage * 100;
 }
 
 unique_ptr<StringValueScanner> CSVGlobalState::Next(optional_ptr<StringValueScanner> previous_scanner) {
+	if (previous_scanner) {
+		// We have to insert information for validation
+		lock_guard<mutex> parallel_lock(main_mutex);
+		validator.Insert(previous_scanner->csv_file_scan->file_idx, previous_scanner->scanner_idx,
+		                 previous_scanner->GetValidationLine());
+	}
 	if (single_threaded) {
+		{
+			lock_guard<mutex> parallel_lock(main_mutex);
+			if (previous_scanner) {
+				// Cleanup previous scanner.
+				previous_scanner->buffer_tracker.reset();
+				current_buffer_in_use.reset();
+				previous_scanner->csv_file_scan->Finish();
+			}
+		}
 		idx_t cur_idx;
 		bool empty_file = false;
 		do {
@@ -97,19 +118,16 @@ unique_ptr<StringValueScanner> CSVGlobalState::Next(optional_ptr<StringValueScan
 			auto file_scan = make_shared_ptr<CSVFileScan>(context, bind_data.files[cur_idx], bind_data.options, cur_idx,
 			                                              bind_data, column_ids, file_schema, true);
 			empty_file = file_scan->file_size == 0;
+
 			if (!empty_file) {
 				lock_guard<mutex> parallel_lock(main_mutex);
 				file_scans.emplace_back(std::move(file_scan));
 				auto current_file = file_scans.back();
 				current_boundary = current_file->start_iterator;
-				current_boundary.SetCurrentBoundaryToPosition(single_threaded);
+				current_boundary.SetCurrentBoundaryToPosition(single_threaded, bind_data.options);
 				current_buffer_in_use = make_shared_ptr<CSVBufferUsage>(*file_scans.back()->buffer_manager,
 				                                                        current_boundary.GetBufferIdx());
-				if (previous_scanner) {
-					previous_scanner->buffer_tracker.reset();
-					current_buffer_in_use.reset();
-					previous_scanner->csv_file_scan->Finish();
-				}
+
 				return make_uniq<StringValueScanner>(scanner_idx++, current_file->buffer_manager,
 				                                     current_file->state_machine, current_file->error_handler,
 				                                     current_file, false, current_boundary);
@@ -140,7 +158,7 @@ unique_ptr<StringValueScanner> CSVGlobalState::Next(optional_ptr<StringValueScan
 	csv_scanner->buffer_tracker = current_buffer_in_use;
 
 	// We then produce the next boundary
-	if (!current_boundary.Next(*current_file.buffer_manager)) {
+	if (!current_boundary.Next(*current_file.buffer_manager, bind_data.options)) {
 		// This means we are done scanning the current file
 		do {
 			auto current_file_idx = file_scans.back()->file_idx + 1;
@@ -151,7 +169,7 @@ unique_ptr<StringValueScanner> CSVGlobalState::Next(optional_ptr<StringValueScan
 				                                                     column_ids, file_schema, false));
 				// And re-start the boundary-iterator
 				current_boundary = file_scans.back()->start_iterator;
-				current_boundary.SetCurrentBoundaryToPosition(single_threaded);
+				current_boundary.SetCurrentBoundaryToPosition(single_threaded, bind_data.options);
 				current_buffer_in_use = make_shared_ptr<CSVBufferUsage>(*file_scans.back()->buffer_manager,
 				                                                        current_boundary.GetBufferIdx());
 			} else {
@@ -167,11 +185,11 @@ unique_ptr<StringValueScanner> CSVGlobalState::Next(optional_ptr<StringValueScan
 
 idx_t CSVGlobalState::MaxThreads() const {
 	// We initialize max one thread per our set bytes per thread limit
-	if (single_threaded) {
+	if (single_threaded || !file_scans.front()->on_disk_file) {
 		return system_threads;
 	}
-	idx_t total_threads = file_scans.front()->file_size / CSVIterator::BYTES_PER_THREAD + 1;
-
+	const idx_t bytes_per_thread = CSVIterator::BytesPerThread(file_scans.front()->options);
+	const idx_t total_threads = file_scans.front()->file_size / bytes_per_thread + 1;
 	if (total_threads < system_threads) {
 		return total_threads;
 	}
@@ -183,7 +201,13 @@ void CSVGlobalState::DecrementThread() {
 	D_ASSERT(running_threads > 0);
 	running_threads--;
 	if (running_threads == 0) {
-		for (auto &file : file_scans) {
+		const bool ignore_or_store_errors =
+		    bind_data.options.ignore_errors.GetValue() || bind_data.options.store_rejects.GetValue();
+		if (!single_threaded && !ignore_or_store_errors) {
+			// If we are running multithreaded and not ignoring errors, we must run the validator
+			validator.Verify();
+		}
+		for (const auto &file : file_scans) {
 			file->error_handler->ErrorIfNeeded();
 		}
 		FillRejectsTable();
@@ -193,39 +217,6 @@ void CSVGlobalState::DecrementThread() {
 	}
 }
 
-bool IsCSVErrorAcceptedReject(CSVErrorType type) {
-	switch (type) {
-	case CSVErrorType::CAST_ERROR:
-	case CSVErrorType::TOO_MANY_COLUMNS:
-	case CSVErrorType::TOO_FEW_COLUMNS:
-	case CSVErrorType::MAXIMUM_LINE_SIZE:
-	case CSVErrorType::UNTERMINATED_QUOTES:
-	case CSVErrorType::INVALID_UNICODE:
-		return true;
-	default:
-		return false;
-	}
-}
-
-string CSVErrorTypeToEnum(CSVErrorType type) {
-	switch (type) {
-	case CSVErrorType::CAST_ERROR:
-		return "CAST";
-	case CSVErrorType::TOO_FEW_COLUMNS:
-		return "MISSING COLUMNS";
-	case CSVErrorType::TOO_MANY_COLUMNS:
-		return "TOO MANY COLUMNS";
-	case CSVErrorType::MAXIMUM_LINE_SIZE:
-		return "LINE SIZE OVER MAXIMUM";
-	case CSVErrorType::UNTERMINATED_QUOTES:
-		return "UNQUOTED VALUE";
-	case CSVErrorType::INVALID_UNICODE:
-		return "INVALID UNICODE";
-	default:
-		throw InternalException("CSV Error is not valid to be stored in a Rejects Table");
-	}
-}
-
 void FillScanErrorTable(InternalAppender &scan_appender, idx_t scan_idx, idx_t file_idx, CSVFileScan &file) {
 	CSVReaderOptions &options = file.options;
 	// Add the row to the rejects table
@@ -248,12 +239,16 @@ void FillScanErrorTable(InternalAppender &scan_appender, idx_t scan_idx, idx_t f
 	scan_appender.Append(Value::UINTEGER(NumericCast<uint32_t>(options.dialect_options.skip_rows.GetValue())));
 	// 9. Has Header
 	scan_appender.Append(Value::BOOLEAN(options.dialect_options.header.GetValue()));
+
+	auto &types = file.GetTypes();
+	auto &names = file.GetNames();
+
 	// 10. List<Struct<Column-Name:Types>> {'col1': 'INTEGER', 'col2': 'VARCHAR'}
 	std::ostringstream columns;
 	columns << "{";
-	for (idx_t i = 0; i < file.types.size(); i++) {
-		columns << "'" << file.names[i] << "': '" << file.types[i].ToString() << "'";
-		if (i != file.types.size() - 1) {
+	for (idx_t i = 0; i < types.size(); i++) {
+		columns << "'" << names[i] << "': '" << types[i].ToString() << "'";
+		if (i != types.size() - 1) {
 			columns << ",";
 		}
 	}
@@ -285,7 +280,7 @@ void FillScanErrorTable(InternalAppender &scan_appender, idx_t scan_idx, idx_t f
 	scan_appender.EndRow();
 }
 
-void CSVGlobalState::FillRejectsTable() {
+void CSVGlobalState::FillRejectsTable() const {
 	auto &options = bind_data.options;
 
 	if (options.store_rejects.GetValue()) {
@@ -299,70 +294,10 @@ void CSVGlobalState::FillRejectsTable() {
 		InternalAppender scans_appender(context, scans_table);
 		idx_t scan_idx = context.transaction.GetActiveQuery();
 		for (auto &file : file_scans) {
-			idx_t file_idx = rejects->GetCurrentFileIndex(scan_idx);
+			const idx_t file_idx = rejects->GetCurrentFileIndex(scan_idx);
 			auto file_name = file->file_path;
-			auto &errors = file->error_handler->errors;
-			// We first insert the file into the file scans table
-			for (auto &error_vector : errors) {
-				for (auto &error : error_vector.second) {
-					if (!IsCSVErrorAcceptedReject(error.type)) {
-						continue;
-					}
-					// short circuit if we already have too many rejects
-					if (limit == 0 || rejects->count < limit) {
-						if (limit != 0 && rejects->count >= limit) {
-							break;
-						}
-						rejects->count++;
-						auto row_line = file->error_handler->GetLine(error.error_info);
-						auto col_idx = error.column_idx;
-						// Add the row to the rejects table
-						errors_appender.BeginRow();
-						// 1. Scan Id
-						errors_appender.Append(scan_idx);
-						// 2. File Id
-						errors_appender.Append(file_idx);
-						// 3. Row Line
-						errors_appender.Append(row_line);
-						// 4. Byte Position of the row error
-						errors_appender.Append(error.row_byte_position + 1);
-						// 5. Byte Position where error occurred
-						if (!error.byte_position.IsValid()) {
-							// This means this error comes from a flush, and we don't support this yet, so we give it
-							// a null
-							errors_appender.Append(Value());
-						} else {
-							errors_appender.Append(error.byte_position.GetIndex() + 1);
-						}
-						// 6. Column Index
-						if (error.type == CSVErrorType::MAXIMUM_LINE_SIZE) {
-							errors_appender.Append(Value());
-						} else {
-							errors_appender.Append(col_idx + 1);
-						}
-						// 7. Column Name (If Applicable)
-						switch (error.type) {
-						case CSVErrorType::TOO_MANY_COLUMNS:
-						case CSVErrorType::MAXIMUM_LINE_SIZE:
-							errors_appender.Append(Value());
-							break;
-						case CSVErrorType::TOO_FEW_COLUMNS:
-							D_ASSERT(bind_data.return_names.size() > col_idx + 1);
-							errors_appender.Append(string_t(bind_data.return_names[col_idx + 1]));
-							break;
-						default:
-							errors_appender.Append(string_t(bind_data.return_names[col_idx]));
-						}
-						// 8. Error Type
-						errors_appender.Append(string_t(CSVErrorTypeToEnum(error.type)));
-						// 9. Original CSV Line
-						errors_appender.Append(string_t(error.csv_row));
-						// 10. Full Error Message
-						errors_appender.Append(string_t(error.error_message));
-						errors_appender.EndRow();
-					}
-				}
-			}
+			file->error_handler->FillRejectsTable(errors_appender, file_idx, scan_idx, *file, *rejects, bind_data,
+			                                      limit);
 			if (rejects->count != 0) {
 				rejects->count = 0;
 				FillScanErrorTable(scans_appender, scan_idx, file_idx, *file);

package/src/duckdb/src/execution/operator/csv_scanner/util/csv_error.cpp

@@ -1,7 +1,10 @@
 #include "duckdb/execution/operator/csv_scanner/csv_error.hpp"
 #include "duckdb/common/exception/conversion_exception.hpp"
 #include "duckdb/common/string_util.hpp"
-
+#include "duckdb/function/table/read_csv.hpp"
+#include "duckdb/execution/operator/persistent/csv_rejects_table.hpp"
+#include "duckdb/execution/operator/csv_scanner/csv_file_scanner.hpp"
+#include "duckdb/main/appender.hpp"
 #include <sstream>
 
 namespace duckdb {
@@ -41,11 +44,11 @@ void CSVErrorHandler::ThrowError(const CSVError &csv_error) {
 	}
 }
 
-void CSVErrorHandler::Error(CSVError csv_error, bool force_error) {
+void CSVErrorHandler::Error(const CSVError &csv_error, bool force_error) {
 	lock_guard<mutex> parallel_lock(main_mutex);
 	if ((ignore_errors && !force_error) || (PrintLineNumber(csv_error) && !CanGetLine(csv_error.GetBoundaryIndex()))) {
 		// We store this error, we can't throw it now, or we are ignoring it
-		errors[csv_error.error_info].push_back(std::move(csv_error));
+		errors.push_back(csv_error);
 		return;
 	}
 	// Otherwise we can throw directly
@@ -58,10 +61,19 @@ void CSVErrorHandler::ErrorIfNeeded() {
 		// Nothing to error
 		return;
 	}
-	CSVError first_error = errors.begin()->second[0];
 
-	if (CanGetLine(first_error.error_info.boundary_idx)) {
-		ThrowError(first_error);
+	if (CanGetLine(errors[0].error_info.boundary_idx)) {
+		ThrowError(errors[0]);
+	}
+}
+
+void CSVErrorHandler::ErrorIfTypeExists(CSVErrorType error_type) {
+	lock_guard<mutex> parallel_lock(main_mutex);
+	for (auto &error : errors) {
+		if (error.type == error_type) {
+			// If it's a maximum line size error, we can do it now.
+			ThrowError(error);
+		}
 	}
 }
 
@@ -84,6 +96,136 @@ bool CSVErrorHandler::AnyErrors() {
 	return !errors.empty();
 }
 
+bool CSVErrorHandler::HasError(const CSVErrorType error_type) {
+	lock_guard<mutex> parallel_lock(main_mutex);
+	for (const auto &er : errors) {
+		if (er.type == error_type) {
+			return true;
+		}
+	}
+	return false;
+}
+
+idx_t CSVErrorHandler::GetSize() {
+	lock_guard<mutex> parallel_lock(main_mutex);
+	return errors.size();
+}
+
+bool IsCSVErrorAcceptedReject(CSVErrorType type) {
+	switch (type) {
+	case CSVErrorType::INVALID_STATE:
+	case CSVErrorType::CAST_ERROR:
+	case CSVErrorType::TOO_MANY_COLUMNS:
+	case CSVErrorType::TOO_FEW_COLUMNS:
+	case CSVErrorType::MAXIMUM_LINE_SIZE:
+	case CSVErrorType::UNTERMINATED_QUOTES:
+	case CSVErrorType::INVALID_UNICODE:
+		return true;
+	default:
+		return false;
+	}
+}
+string CSVErrorTypeToEnum(CSVErrorType type) {
+	switch (type) {
+	case CSVErrorType::CAST_ERROR:
+		return "CAST";
+	case CSVErrorType::TOO_FEW_COLUMNS:
+		return "MISSING COLUMNS";
+	case CSVErrorType::TOO_MANY_COLUMNS:
+		return "TOO MANY COLUMNS";
+	case CSVErrorType::MAXIMUM_LINE_SIZE:
+		return "LINE SIZE OVER MAXIMUM";
+	case CSVErrorType::UNTERMINATED_QUOTES:
+		return "UNQUOTED VALUE";
+	case CSVErrorType::INVALID_UNICODE:
+		return "INVALID UNICODE";
+	case CSVErrorType::INVALID_STATE:
+		return "INVALID STATE";
+	default:
+		throw InternalException("CSV Error is not valid to be stored in a Rejects Table");
+	}
+}
+
+void CSVErrorHandler::FillRejectsTable(InternalAppender &errors_appender, const idx_t file_idx, const idx_t scan_idx,
+                                       const CSVFileScan &file, CSVRejectsTable &rejects, const ReadCSVData &bind_data,
+                                       const idx_t limit) {
+	lock_guard<mutex> parallel_lock(main_mutex);
+	// We first insert the file into the file scans table
+	for (auto &error : file.error_handler->errors) {
+		if (!IsCSVErrorAcceptedReject(error.type)) {
+			continue;
+		}
+		// short circuit if we already have too many rejects
+		if (limit == 0 || rejects.count < limit) {
+			if (limit != 0 && rejects.count >= limit) {
+				break;
+			}
+			rejects.count++;
+			const auto row_line = file.error_handler->GetLineInternal(error.error_info);
+			const auto col_idx = error.column_idx;
+			// Add the row to the rejects table
+			errors_appender.BeginRow();
+			// 1. Scan ID
+			errors_appender.Append(scan_idx);
+			// 2. File ID
+			errors_appender.Append(file_idx);
+			// 3. Row Line
+			errors_appender.Append(row_line);
+			// 4. Byte Position of the row error
+			errors_appender.Append(error.row_byte_position + 1);
+			// 5. Byte Position where error occurred
+			if (!error.byte_position.IsValid()) {
+				// This means this error comes from a flush, and we don't support this yet, so we give it
+				// a null
+				errors_appender.Append(Value());
+			} else {
+				errors_appender.Append(error.byte_position.GetIndex() + 1);
+			}
+			// 6. Column Index
+			if (error.type == CSVErrorType::MAXIMUM_LINE_SIZE) {
+				errors_appender.Append(Value());
+			} else {
+				errors_appender.Append(col_idx + 1);
+			}
+			// 7. Column Name (If Applicable)
+			switch (error.type) {
+			case CSVErrorType::TOO_MANY_COLUMNS:
+			case CSVErrorType::MAXIMUM_LINE_SIZE:
+				errors_appender.Append(Value());
+				break;
+			case CSVErrorType::TOO_FEW_COLUMNS:
+				D_ASSERT(bind_data.return_names.size() > col_idx + 1);
+				errors_appender.Append(string_t(bind_data.return_names[col_idx + 1]));
+				break;
+			default:
+				errors_appender.Append(string_t(bind_data.return_names[col_idx]));
+			}
+			// 8. Error Type
+			errors_appender.Append(string_t(CSVErrorTypeToEnum(error.type)));
+			// 9. Original CSV Line
+			errors_appender.Append(string_t(error.csv_row));
+			// 10. Full Error Message
+			errors_appender.Append(string_t(error.error_message));
+			errors_appender.EndRow();
+		}
+	}
+}
+
+idx_t CSVErrorHandler::GetMaxLineLength() {
+	lock_guard<mutex> parallel_lock(main_mutex);
+	return max_line_length;
+}
+
+void CSVErrorHandler::DontPrintErrorLine() {
+	lock_guard<mutex> parallel_lock(main_mutex);
+	print_line = false;
+}
+
+void CSVErrorHandler::SetIgnoreErrors(bool ignore_errors_p) {
+	lock_guard<mutex> parallel_lock(main_mutex);
+	ignore_errors = ignore_errors_p;
+}
+
 CSVError::CSVError(string error_message_p, CSVErrorType type_p, LinesPerBoundary error_info_p)
     : error_message(std::move(error_message_p)), type(type_p), error_info(error_info_p) {
 }
@@ -110,7 +252,6 @@ CSVError CSVError::ColumnTypesError(case_insensitive_map_t<idx_t> sql_types_per_
 		auto it = sql_types_per_column.find(names[i]);
 		if (it != sql_types_per_column.end()) {
 			sql_types_per_column.erase(names[i]);
-			continue;
 		}
 	}
 	if (sql_types_per_column.empty()) {
@@ -129,8 +270,8 @@ void CSVError::RemoveNewLine(string &error) {
 	error = StringUtil::Split(error, "\n")[0];
 }
 
-CSVError CSVError::CastError(const CSVReaderOptions &options, string &column_name, string &cast_error, idx_t column_idx,
-                             string &csv_row, LinesPerBoundary error_info, idx_t row_byte_position,
+CSVError CSVError::CastError(const CSVReaderOptions &options, const string &column_name, string &cast_error,
+                             idx_t column_idx, string &csv_row, LinesPerBoundary error_info, idx_t row_byte_position,
                              optional_idx byte_position, LogicalTypeId type, const string &current_path) {
 	std::ostringstream error;
 	// Which column
@@ -160,22 +301,36 @@ CSVError CSVError::CastError(const CSVReaderOptions &options, string &column_nam
 	                how_to_fix_it.str(), current_path);
 }
 
-CSVError CSVError::LineSizeError(const CSVReaderOptions &options, idx_t actual_size, LinesPerBoundary error_info,
-                                 string &csv_row, idx_t byte_position, const string &current_path) {
+CSVError CSVError::LineSizeError(const CSVReaderOptions &options, LinesPerBoundary error_info, string &csv_row,
+                                 idx_t byte_position, const string &current_path) {
 	std::ostringstream error;
-	error << "Maximum line size of " << options.maximum_line_size << " bytes exceeded. ";
-	error << "Actual Size:" << actual_size << " bytes." << '\n';
+	error << "Maximum line size of " << options.maximum_line_size.GetValue() << " bytes exceeded. ";
+	error << "Actual Size:" << csv_row.size() << " bytes." << '\n';
 
 	std::ostringstream how_to_fix_it;
-	how_to_fix_it << "Possible Solution: Change the maximum length size, e.g., max_line_size=" << actual_size + 1
+	how_to_fix_it << "Possible Solution: Change the maximum length size, e.g., max_line_size=" << csv_row.size() + 2
 	              << "\n";
 
 	return CSVError(error.str(), MAXIMUM_LINE_SIZE, 0, csv_row, error_info, byte_position, byte_position, options,
 	                how_to_fix_it.str(), current_path);
 }
 
+CSVError CSVError::InvalidState(const CSVReaderOptions &options, idx_t current_column, LinesPerBoundary error_info,
+                                string &csv_row, idx_t row_byte_position, optional_idx byte_position,
+                                const string &current_path) {
+	std::ostringstream error;
+	error << "The CSV Parser state machine reached an invalid state.\nThis can happen when is not possible to parse "
+	         "your CSV File with the given options, or the CSV File is not RFC 4180 compliant ";
+
+	std::ostringstream how_to_fix_it;
+	how_to_fix_it << "Possible fixes:" << '\n';
+	how_to_fix_it << "* Enable scanning files that are not RFC 4180 compliant (rfc_4180=false)." << '\n';
+
+	return CSVError(error.str(), INVALID_STATE, current_column, csv_row, error_info, row_byte_position, byte_position,
+	                options, how_to_fix_it.str(), current_path);
+}
 CSVError CSVError::HeaderSniffingError(const CSVReaderOptions &options, const vector<HeaderValue> &best_header_row,
-                                       idx_t column_count, char delimiter) {
+                                       const idx_t column_count, const string &delimiter) {
 	std::ostringstream error;
 	// 1. Which file
 	error << "Error when sniffing file \"" << options.file_path << "\"." << '\n';
@@ -223,6 +378,7 @@ CSVError CSVError::HeaderSniffingError(const CSVReaderOptions &options, const ve
 	if (!options.null_padding) {
 		error << "* Enable null padding (null_padding=true) to pad missing columns with NULL values" << '\n';
 	}
+
 	return CSVError(error.str(), SNIFFING, {});
 }
 
@@ -281,7 +437,14 @@ CSVError CSVError::SniffingError(const CSVReaderOptions &options, const string &
 	}
 	error << "* Check you are using the correct file compression, otherwise set it (e.g., compression = \'zstd\')"
 	      << '\n';
+	error << "* Be sure that the maximum line size is set to an appropriate value, otherwise set it (e.g., "
+	         "max_line_size=10000000)"
+	      << "\n";
 
+	if (options.dialect_options.state_machine_options.rfc_4180.GetValue() != false ||
+	    !options.dialect_options.state_machine_options.rfc_4180.IsSetByUser()) {
+		error << "* Enable scanning files that are not RFC 4180 compliant (rfc_4180=false). " << '\n';
+	}
 	return CSVError(error.str(), SNIFFING, {});
 }
 
@@ -304,7 +467,7 @@ CSVError CSVError::UnterminatedQuotesError(const CSVReaderOptions &options, idx_
 	std::ostringstream how_to_fix_it;
 	how_to_fix_it << "Possible fixes:" << '\n';
 	how_to_fix_it << "* Enable ignore errors (ignore_errors=true) to skip this row" << '\n';
-	how_to_fix_it << "* Set quote do empty or to a different value (e.g., quote=\'\')" << '\n';
+	how_to_fix_it << "* Set quote to empty or to a different value (e.g., quote=\'\')" << '\n';
 	return CSVError(error.str(), UNTERMINATED_QUOTES, current_column, csv_row, error_info, row_byte_position,
 	                byte_position, options, how_to_fix_it.str(), current_path);
 }
@@ -324,12 +487,13 @@ CSVError CSVError::IncorrectColumnAmountError(const CSVReaderOptions &options, i
 	}
 	// How many columns were expected and how many were found
 	error << "Expected Number of Columns: " << options.dialect_options.num_cols << " Found: " << actual_columns + 1;
+	idx_t byte_pos = byte_position.GetIndex() == 0 ? 0 : byte_position.GetIndex() - 1;
 	if (actual_columns >= options.dialect_options.num_cols) {
-		return CSVError(error.str(), TOO_MANY_COLUMNS, actual_columns, csv_row, error_info, row_byte_position,
-		                byte_position.GetIndex() - 1, options, how_to_fix_it.str(), current_path);
+		return CSVError(error.str(), TOO_MANY_COLUMNS, actual_columns, csv_row, error_info, row_byte_position, byte_pos,
+		                options, how_to_fix_it.str(), current_path);
 	} else {
-		return CSVError(error.str(), TOO_FEW_COLUMNS, actual_columns, csv_row, error_info, row_byte_position,
-		                byte_position.GetIndex() - 1, options, how_to_fix_it.str(), current_path);
+		return CSVError(error.str(), TOO_FEW_COLUMNS, actual_columns, csv_row, error_info, row_byte_position, byte_pos,
+		                options, how_to_fix_it.str(), current_path);
 	}
 }