duckdb 0.9.3-dev2.0 → 0.10.0

package/src/duckdb/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp

@@ -0,0 +1,221 @@
+#include "duckdb/execution/operator/csv_scanner/table_function/csv_file_scanner.hpp"
+#include "duckdb/function/table/read_csv.hpp"
+#include "duckdb/execution/operator/csv_scanner/sniffer/csv_sniffer.hpp"
+
+namespace duckdb {
+
+CSVFileScan::CSVFileScan(ClientContext &context, shared_ptr<CSVBufferManager> buffer_manager_p,
+                         shared_ptr<CSVStateMachine> state_machine_p, const CSVReaderOptions &options_p,
+                         const ReadCSVData &bind_data, const vector<column_t> &column_ids,
+                         vector<LogicalType> &file_schema)
+    : file_path(options_p.file_path), file_idx(0), buffer_manager(std::move(buffer_manager_p)),
+      state_machine(std::move(state_machine_p)), file_size(buffer_manager->file_handle->FileSize()),
+      error_handler(make_shared<CSVErrorHandler>(options_p.ignore_errors)),
+      on_disk_file(buffer_manager->file_handle->OnDiskFile()), options(options_p) {
+	if (bind_data.initial_reader.get()) {
+		auto &union_reader = *bind_data.initial_reader;
+		names = union_reader.GetNames();
+		options = union_reader.options;
+		types = union_reader.GetTypes();
+		MultiFileReader::InitializeReader(*this, options.file_options, bind_data.reader_bind, bind_data.return_types,
+		                                  bind_data.return_names, column_ids, nullptr, file_path, context);
+		InitializeFileNamesTypes();
+		return;
+	} else if (!bind_data.column_info.empty()) {
+		// Serialized Union By name
+		names = bind_data.column_info[0].names;
+		types = bind_data.column_info[0].types;
+		MultiFileReader::InitializeReader(*this, options.file_options, bind_data.reader_bind, bind_data.return_types,
+		                                  bind_data.return_names, column_ids, nullptr, file_path, context);
+		InitializeFileNamesTypes();
+		return;
+	}
+	names = bind_data.return_names;
+	types = bind_data.return_types;
+	file_schema = bind_data.return_types;
+	MultiFileReader::InitializeReader(*this, options.file_options, bind_data.reader_bind, bind_data.return_types,
+	                                  bind_data.return_names, column_ids, nullptr, file_path, context);
+
+	InitializeFileNamesTypes();
+}
+
+CSVFileScan::CSVFileScan(ClientContext &context, const string &file_path_p, const CSVReaderOptions &options_p,
+                         const idx_t file_idx_p, const ReadCSVData &bind_data, const vector<column_t> &column_ids,
+                         const vector<LogicalType> &file_schema)
+    : file_path(file_path_p), file_idx(file_idx_p),
+      error_handler(make_shared<CSVErrorHandler>(options_p.ignore_errors)), options(options_p) {
+	if (file_idx < bind_data.union_readers.size()) {
+		// we are doing UNION BY NAME - fetch the options from the union reader for this file
+		optional_ptr<CSVFileScan> union_reader_ptr;
+		if (file_idx == 0) {
+			union_reader_ptr = bind_data.initial_reader.get();
+		} else {
+			union_reader_ptr = bind_data.union_readers[file_idx].get();
+		}
+		if (union_reader_ptr) {
+			auto &union_reader = *union_reader_ptr;
+			// Initialize Buffer Manager
+			buffer_manager = union_reader.buffer_manager;
+			// Initialize On Disk and Size of file
+			on_disk_file = union_reader.on_disk_file;
+			file_size = union_reader.file_size;
+			names = union_reader.GetNames();
+			options = union_reader.options;
+			types = union_reader.GetTypes();
+			state_machine = union_reader.state_machine;
+			MultiFileReader::InitializeReader(*this, options.file_options, bind_data.reader_bind,
+			                                  bind_data.return_types, bind_data.return_names, column_ids, nullptr,
+			                                  file_path, context);
+
+			InitializeFileNamesTypes();
+			return;
+		}
+	}
+
+	// Initialize Buffer Manager
+	buffer_manager = make_shared<CSVBufferManager>(context, options, file_path, file_idx);
+	// Initialize On Disk and Size of file
+	on_disk_file = buffer_manager->file_handle->OnDiskFile();
+	file_size = buffer_manager->file_handle->FileSize();
+	// Initialize State Machine
+	auto &state_machine_cache = CSVStateMachineCache::Get(context);
+
+	if (file_idx < bind_data.column_info.size()) {
+		// Serialized Union By name
+		names = bind_data.column_info[file_idx].names;
+		types = bind_data.column_info[file_idx].types;
+		options.dialect_options.num_cols = names.size();
+		state_machine = make_shared<CSVStateMachine>(
+		    state_machine_cache.Get(options.dialect_options.state_machine_options), options);
+
+		MultiFileReader::InitializeReader(*this, options.file_options, bind_data.reader_bind, bind_data.return_types,
+		                                  bind_data.return_names, column_ids, nullptr, file_path, context);
+		InitializeFileNamesTypes();
+		return;
+	}
+	// Sniff it (We only really care about dialect detection, if types or number of columns are different this will
+	// error out during scanning)
+	if (options.auto_detect && file_idx > 0) {
+		CSVSniffer sniffer(options, buffer_manager, state_machine_cache);
+		auto result = sniffer.SniffCSV();
+		if (!file_schema.empty()) {
+			if (!options.file_options.filename && !options.file_options.hive_partitioning &&
+			    file_schema.size() != result.return_types.size()) {
+				throw InvalidInputException("Mismatch between the schema of different files");
+			}
+		}
+	}
+	if (options.dialect_options.num_cols == 0) {
+		// We need to define the number of columns, if the sniffer is not running this must be in the sql_type_list
+		options.dialect_options.num_cols = options.sql_type_list.size();
+	}
+
+	if (options.dialect_options.state_machine_options.new_line == NewLineIdentifier::NOT_SET) {
+		options.dialect_options.state_machine_options.new_line = CSVSniffer::DetectNewLineDelimiter(*buffer_manager);
+	}
+
+	names = bind_data.csv_names;
+	types = bind_data.csv_types;
+	state_machine =
+	    make_shared<CSVStateMachine>(state_machine_cache.Get(options.dialect_options.state_machine_options), options);
+
+	MultiFileReader::InitializeReader(*this, options.file_options, bind_data.reader_bind, bind_data.return_types,
+	                                  bind_data.return_names, column_ids, nullptr, file_path, context);
+	InitializeFileNamesTypes();
+}
+
+CSVFileScan::CSVFileScan(ClientContext &context, const string &file_name, CSVReaderOptions &options_p)
+    : file_path(file_name), file_idx(0), error_handler(make_shared<CSVErrorHandler>(options_p.ignore_errors)),
+      options(options_p) {
+	buffer_manager = make_shared<CSVBufferManager>(context, options, file_path, file_idx);
+	// Initialize On Disk and Size of file
+	on_disk_file = buffer_manager->file_handle->OnDiskFile();
+	file_size = buffer_manager->file_handle->FileSize();
+	// Sniff it (We only really care about dialect detection, if types or number of columns are different this will
+	// error out during scanning)
+	auto &state_machine_cache = CSVStateMachineCache::Get(context);
+	if (options.auto_detect && options.dialect_options.num_cols == 0) {
+		CSVSniffer sniffer(options, buffer_manager, state_machine_cache);
+		auto sniffer_result = sniffer.SniffCSV();
+		if (names.empty()) {
+			names = sniffer_result.names;
+			types = sniffer_result.return_types;
+		}
+	}
+	if (options.dialect_options.num_cols == 0) {
+		// We need to define the number of columns, if the sniffer is not running this must be in the sql_type_list
+		options.dialect_options.num_cols = options.sql_type_list.size();
+	}
+	// Initialize State Machine
+	state_machine =
+	    make_shared<CSVStateMachine>(state_machine_cache.Get(options.dialect_options.state_machine_options), options);
+}
+
+void CSVFileScan::InitializeFileNamesTypes() {
+	if (reader_data.empty_columns && reader_data.column_ids.empty()) {
+		// This means that the columns from this file are irrelevant.
+		// just read the first column
+		file_types.emplace_back(LogicalType::VARCHAR);
+		projected_columns.insert(0);
+		projection_ids.emplace_back(0, 0);
+		return;
+	}
+
+	for (idx_t i = 0; i < reader_data.column_ids.size(); i++) {
+		idx_t result_idx = reader_data.column_ids[i];
+		file_types.emplace_back(types[result_idx]);
+		projected_columns.insert(result_idx);
+		projection_ids.emplace_back(result_idx, i);
+	}
+
+	if (!projected_columns.empty()) {
+		// We might have to add recovery rejects column ids
+		for (idx_t i = 0; i < options.rejects_recovery_column_ids.size(); i++) {
+			idx_t col_id = options.rejects_recovery_column_ids[i];
+			if (projected_columns.find(col_id) == projected_columns.end()) {
+				// We have to insert this column in our projection
+				projected_columns.insert(col_id);
+				file_types.emplace_back(LogicalType::VARCHAR);
+				projected_columns.insert(col_id);
+				projection_ids.emplace_back(col_id, col_id);
+			}
+		}
+	}
+
+	if (reader_data.column_ids.empty()) {
+		file_types = types;
+	}
+
+	// We need to be sure that our types are also following the cast_map
+	for (idx_t i = 0; i < reader_data.column_ids.size(); i++) {
+		if (reader_data.cast_map.find(reader_data.column_ids[i]) != reader_data.cast_map.end()) {
+			file_types[i] = reader_data.cast_map[reader_data.column_ids[i]];
+		}
+	}
+
+	// We sort the types on the order of the parsed chunk
+	std::sort(projection_ids.begin(), projection_ids.end());
+	vector<LogicalType> sorted_types;
+	for (idx_t i = 0; i < projection_ids.size(); ++i) {
+		sorted_types.push_back(file_types[projection_ids[i].second]);
+	}
+	file_types = sorted_types;
+}
+
+const string &CSVFileScan::GetFileName() {
+	return file_path;
+}
+const vector<string> &CSVFileScan::GetNames() {
+	return names;
+}
+const vector<LogicalType> &CSVFileScan::GetTypes() {
+	return types;
+}
+
+void CSVFileScan::InitializeProjection() {
+	for (idx_t i = 0; i < options.dialect_options.num_cols; i++) {
+		reader_data.column_ids.push_back(i);
+		reader_data.column_mapping.push_back(i);
+	}
+}
+} // namespace duckdb

package/src/duckdb/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp

@@ -0,0 +1,204 @@
+#include "duckdb/execution/operator/csv_scanner/table_function/global_csv_state.hpp"
+#include "duckdb/main/client_data.hpp"
+#include "duckdb/execution/operator/csv_scanner/scanner/scanner_boundary.hpp"
+#include "duckdb/execution/operator/csv_scanner/sniffer/csv_sniffer.hpp"
+#include "duckdb/execution/operator/persistent/csv_rejects_table.hpp"
+#include "duckdb/main/appender.hpp"
+
+namespace duckdb {
+
+CSVGlobalState::CSVGlobalState(ClientContext &context_p, const shared_ptr<CSVBufferManager> &buffer_manager,
+                               const CSVReaderOptions &options, idx_t system_threads_p, const vector<string> &files,
+                               vector<column_t> column_ids_p, const ReadCSVData &bind_data_p)
+    : context(context_p), system_threads(system_threads_p), column_ids(std::move(column_ids_p)),
+      sniffer_mismatch_error(options.sniffer_user_mismatch_error), bind_data(bind_data_p) {
+
+	if (buffer_manager && buffer_manager->GetFilePath() == files[0]) {
+		auto state_machine = make_shared<CSVStateMachine>(
+		    CSVStateMachineCache::Get(context).Get(options.dialect_options.state_machine_options), options);
+		// If we already have a buffer manager, we don't need to reconstruct it to the first file
+		file_scans.emplace_back(make_uniq<CSVFileScan>(context, buffer_manager, state_machine, options, bind_data,
+		                                               column_ids, file_schema));
+	} else {
+		// If not we need to construct it for the first file
+		file_scans.emplace_back(
+		    make_uniq<CSVFileScan>(context, files[0], options, 0, bind_data, column_ids, file_schema));
+	};
+	//! There are situations where we only support single threaded scanning
+	bool many_csv_files = files.size() > 1 && files.size() > system_threads * 2;
+	single_threaded = many_csv_files || !options.parallel;
+	last_file_idx = 0;
+	scanner_idx = 0;
+	running_threads = MaxThreads();
+	if (single_threaded) {
+		current_boundary = CSVIterator();
+	} else {
+		auto buffer_size = file_scans.back()->buffer_manager->GetBuffer(0)->actual_size;
+		current_boundary = CSVIterator(0, 0, 0, 0, buffer_size);
+	}
+	current_buffer_in_use = make_shared<CSVBufferUsage>(*file_scans.back()->buffer_manager, 0);
+}
+
+double CSVGlobalState::GetProgress(const ReadCSVData &bind_data_p) const {
+
+	idx_t total_files = bind_data.files.size();
+	// get the progress WITHIN the current file
+	double progress;
+	if (file_scans.back()->file_size == 0) {
+		progress = 1.0;
+	} else {
+		// for compressed files, readed bytes may greater than files size.
+		progress = std::min(1.0, double(file_scans.back()->bytes_read) / double(file_scans.back()->file_size));
+	}
+	// now get the total percentage of files read
+	double percentage = double(current_boundary.GetFileIdx()) / total_files;
+	percentage += (double(1) / double(total_files)) * progress;
+	return percentage * 100;
+}
+
+unique_ptr<StringValueScanner> CSVGlobalState::Next() {
+	if (single_threaded) {
+		idx_t cur_idx = last_file_idx++;
+		if (cur_idx >= bind_data.files.size()) {
+			return nullptr;
+		}
+		shared_ptr<CSVFileScan> current_file;
+		if (cur_idx == 0) {
+			current_file = file_scans.back();
+		} else {
+			current_file = make_shared<CSVFileScan>(context, bind_data.files[cur_idx], bind_data.options, cur_idx,
+			                                        bind_data, column_ids, file_schema);
+		}
+		auto csv_scanner =
+		    make_uniq<StringValueScanner>(scanner_idx++, current_file->buffer_manager, current_file->state_machine,
+		                                  current_file->error_handler, current_file, current_boundary);
+		return csv_scanner;
+	}
+	lock_guard<mutex> parallel_lock(main_mutex);
+	if (finished) {
+		return nullptr;
+	}
+	if (current_buffer_in_use->buffer_idx != current_boundary.GetBufferIdx()) {
+		current_buffer_in_use =
+		    make_shared<CSVBufferUsage>(*file_scans.back()->buffer_manager, current_boundary.GetBufferIdx());
+	}
+	// We first create the scanner for the current boundary
+	auto &current_file = *file_scans.back();
+	auto csv_scanner =
+	    make_uniq<StringValueScanner>(scanner_idx++, current_file.buffer_manager, current_file.state_machine,
+	                                  current_file.error_handler, file_scans.back(), current_boundary);
+
+	csv_scanner->buffer_tracker = current_buffer_in_use;
+
+	// We then produce the next boundary
+	if (!current_boundary.Next(*current_file.buffer_manager)) {
+		// This means we are done scanning the current file
+		auto current_file_idx = current_file.file_idx + 1;
+		if (current_file_idx < bind_data.files.size()) {
+			// If we have a next file we have to construct the file scan for that
+			file_scans.emplace_back(make_shared<CSVFileScan>(context, bind_data.files[current_file_idx],
+			                                                 bind_data.options, current_file_idx, bind_data, column_ids,
+			                                                 file_schema));
+			// And re-start the boundary-iterator
+			auto buffer_size = file_scans.back()->buffer_manager->GetBuffer(0)->actual_size;
+			current_boundary = CSVIterator(current_file_idx, 0, 0, 0, buffer_size);
+			current_buffer_in_use = make_shared<CSVBufferUsage>(*file_scans.back()->buffer_manager, 0);
+		} else {
+			// If not we are done with this CSV Scanning
+			finished = true;
+		}
+	}
+	// We initialize the scan
+	return csv_scanner;
+}
+
+idx_t CSVGlobalState::MaxThreads() const {
+	// We initialize max one thread per our set bytes per thread limit
+	if (single_threaded) {
+		return system_threads;
+	}
+	idx_t total_threads = file_scans.back()->file_size / CSVIterator::BYTES_PER_THREAD + 1;
+
+	if (total_threads < system_threads) {
+		return total_threads;
+	}
+	return system_threads;
+}
+
+void CSVGlobalState::DecrementThread() {
+	lock_guard<mutex> parallel_lock(main_mutex);
+	D_ASSERT(running_threads > 0);
+	running_threads--;
+	if (running_threads == 0) {
+		FillRejectsTable();
+		if (context.client_data->debug_set_max_line_length) {
+			context.client_data->debug_max_line_length = file_scans[0]->error_handler->GetMaxLineLength();
+		}
+	}
+}
+
+void CSVGlobalState::FillRejectsTable() {
+	auto &options = bind_data.options;
+
+	if (!options.rejects_table_name.empty()) {
+		auto limit = options.rejects_limit;
+
+		auto rejects = CSVRejectsTable::GetOrCreate(context, options.rejects_table_name);
+		lock_guard<mutex> lock(rejects->write_lock);
+		auto &table = rejects->GetTable(context);
+		InternalAppender appender(context, table);
+
+		for (auto &file : file_scans) {
+			auto file_name = file->file_path;
+			auto &errors = file->error_handler->errors;
+			for (auto &error_info : errors) {
+				if (error_info.second.type != CSVErrorType::CAST_ERROR) {
+					// For now we only will use it for casting errors
+					continue;
+				}
+				// short circuit if we already have too many rejects
+				if (limit == 0 || rejects->count < limit) {
+					if (limit != 0 && rejects->count >= limit) {
+						break;
+					}
+					rejects->count++;
+					auto error = &error_info.second;
+					auto row_line = file->error_handler->GetLine(error_info.first);
+					auto col_idx = error->column_idx;
+					auto col_name = bind_data.return_names[col_idx];
+					// Add the row to the rejects table
+					appender.BeginRow();
+					appender.Append(string_t(file_name));
+					appender.Append(row_line);
+					appender.Append(col_idx);
+					appender.Append(string_t("\"" + col_name + "\""));
+					appender.Append(error->row[col_idx]);
+
+					if (!options.rejects_recovery_columns.empty()) {
+						child_list_t<Value> recovery_key;
+						for (auto &key_idx : options.rejects_recovery_column_ids) {
+							// Figure out if the recovery key is valid.
+							// If not, error out for real.
+							auto &value = error->row[key_idx];
+							if (value.IsNull()) {
+								throw InvalidInputException("%s at line %llu in column %s. Parser options:\n%s ",
+								                            "Could not parse recovery column", row_line, col_name,
+								                            options.ToString());
+							}
+							recovery_key.emplace_back(bind_data.return_names[key_idx], value);
+						}
+						appender.Append(Value::STRUCT(recovery_key));
+					}
+					auto row_error_msg =
+					    StringUtil::Format("Could not convert string '%s' to '%s'", error->row[col_idx].ToString(),
+					                       file->types[col_idx].ToString());
+					appender.Append(string_t(row_error_msg));
+					appender.EndRow();
+				}
+				appender.Close();
+			}
+		}
+	}
+}
+
+} // namespace duckdb

package/src/duckdb/src/execution/operator/csv_scanner/util/csv_error.cpp

@@ -0,0 +1,186 @@
+#include "duckdb/execution/operator/csv_scanner/util/csv_error.hpp"
+#include "duckdb/common/exception/conversion_exception.hpp"
+#include <sstream>
+
+namespace duckdb {
+
+LinesPerBoundary::LinesPerBoundary() {
+}
+LinesPerBoundary::LinesPerBoundary(idx_t boundary_idx_p, idx_t lines_in_batch_p)
+    : boundary_idx(boundary_idx_p), lines_in_batch(lines_in_batch_p) {
+}
+
+CSVErrorHandler::CSVErrorHandler(bool ignore_errors_p) : ignore_errors(ignore_errors_p) {
+}
+
+void CSVErrorHandler::Error(CSVError &csv_error) {
+	LinesPerBoundary mock;
+	Error(mock, csv_error, true);
+}
+void CSVErrorHandler::Error(LinesPerBoundary &error_info, CSVError &csv_error, bool force_error) {
+	if (ignore_errors && !force_error) {
+		lock_guard<mutex> parallel_lock(main_mutex);
+		// We store this error
+		errors.push_back({error_info, csv_error});
+		return;
+	}
+
+	std::ostringstream error;
+	if (PrintLineNumber(csv_error)) {
+		error << "CSV Error on Line: " << GetLine(error_info) << std::endl;
+	}
+	{
+		lock_guard<mutex> parallel_lock(main_mutex);
+		got_borked = true;
+	}
+	error << csv_error.error_message;
+	switch (csv_error.type) {
+	case CSVErrorType::CAST_ERROR:
+		throw ConversionException(error.str());
+	case CSVErrorType::COLUMN_NAME_TYPE_MISMATCH:
+		throw BinderException(error.str());
+	case CSVErrorType::NULLPADDED_QUOTED_NEW_VALUE:
+		throw ParameterNotAllowedException(error.str());
+	default:
+		throw InvalidInputException(error.str());
+	}
+}
+
+void CSVErrorHandler::Insert(idx_t boundary_idx, idx_t rows) {
+	lock_guard<mutex> parallel_lock(main_mutex);
+	if (lines_per_batch_map.find(boundary_idx) == lines_per_batch_map.end()) {
+		lines_per_batch_map[boundary_idx] = {boundary_idx, rows};
+	} else {
+		lines_per_batch_map[boundary_idx].lines_in_batch += rows;
+	}
+}
+
+void CSVErrorHandler::NewMaxLineSize(idx_t scan_line_size) {
+	lock_guard<mutex> parallel_lock(main_mutex);
+	max_line_length = std::max(scan_line_size, max_line_length);
+}
+
+CSVError::CSVError(string error_message_p, CSVErrorType type_p)
+    : error_message(std::move(error_message_p)), type(type_p) {
+}
+
+CSVError::CSVError(string error_message_p, CSVErrorType type_p, idx_t column_idx_p, vector<Value> row_p)
+    : error_message(std::move(error_message_p)), type(type_p), column_idx(column_idx_p), row(std::move(row_p)) {
+}
+
+CSVError CSVError::ColumnTypesError(case_insensitive_map_t<idx_t> sql_types_per_column, const vector<string> &names) {
+	for (idx_t i = 0; i < names.size(); i++) {
+		auto it = sql_types_per_column.find(names[i]);
+		if (it != sql_types_per_column.end()) {
+			sql_types_per_column.erase(names[i]);
+			continue;
+		}
+	}
+	if (sql_types_per_column.empty()) {
+		return CSVError("", CSVErrorType::COLUMN_NAME_TYPE_MISMATCH);
+	}
+	string exception = "COLUMN_TYPES error: Columns with names: ";
+	for (auto &col : sql_types_per_column) {
+		exception += "\"" + col.first + "\",";
+	}
+	exception.pop_back();
+	exception += " do not exist in the CSV File";
+	return CSVError(exception, CSVErrorType::COLUMN_NAME_TYPE_MISMATCH);
+}
+
+CSVError CSVError::CastError(const CSVReaderOptions &options, string &column_name, string &cast_error, idx_t column_idx,
+                             vector<Value> &row) {
+	std::ostringstream error;
+	// Which column
+	error << "Error when converting column \"" << column_name << "\"." << std::endl;
+	// What was the cast error
+	error << cast_error << std::endl;
+	error << std::endl;
+	// What were the options
+	error << options.ToString();
+	return CSVError(error.str(), CSVErrorType::CAST_ERROR, column_idx, row);
+}
+
+CSVError CSVError::LineSizeError(const CSVReaderOptions &options, idx_t actual_size) {
+	std::ostringstream error;
+	error << "Maximum line size of " << options.maximum_line_size << " bytes exceeded. ";
+	error << "Actual Size:" << actual_size << " bytes." << std::endl;
+	error << options.ToString();
+	return CSVError(error.str(), CSVErrorType::MAXIMUM_LINE_SIZE);
+}
+
+CSVError CSVError::SniffingError(string &file_path) {
+	std::ostringstream error;
+	// Which column
+	error << "Error when sniffing file \"" << file_path << "\"." << std::endl;
+	error << "CSV options could not be auto-detected. Consider setting parser options manually." << std::endl;
+	return CSVError(error.str(), CSVErrorType::SNIFFING);
+}
+
+CSVError CSVError::NullPaddingFail(const CSVReaderOptions &options) {
+	std::ostringstream error;
+	error << " The parallel scanner does not support null_padding in conjunction with quoted new lines. Please "
+	         "disable the parallel csv reader with parallel=false"
+	      << std::endl;
+	// What were the options
+	error << options.ToString();
+	return CSVError(error.str(), CSVErrorType::NULLPADDED_QUOTED_NEW_VALUE);
+}
+
+CSVError CSVError::UnterminatedQuotesError(const CSVReaderOptions &options, string_t *vector_ptr,
+                                           idx_t vector_line_start, idx_t current_column) {
+	std::ostringstream error;
+	error << "Value with unterminated quote found." << std::endl;
+	error << std::endl;
+	// What were the options
+	error << options.ToString();
+	return CSVError(error.str(), CSVErrorType::UNTERMINATED_QUOTES);
+}
+
+CSVError CSVError::IncorrectColumnAmountError(const CSVReaderOptions &options, string_t *vector_ptr,
+                                              idx_t vector_line_start, idx_t actual_columns) {
+	std::ostringstream error;
+	// How many columns were expected and how many were found
+	error << "Expected Number of Columns: " << options.dialect_options.num_cols << " Found: " << actual_columns
+	      << std::endl;
+	// What were the options
+	error << options.ToString();
+	return CSVError(error.str(), CSVErrorType::INCORRECT_COLUMN_AMOUNT);
+}
+
+bool CSVErrorHandler::PrintLineNumber(CSVError &error) {
+	switch (error.type) {
+	case CSVErrorType::CAST_ERROR:
+	case CSVErrorType::UNTERMINATED_QUOTES:
+	case CSVErrorType::INCORRECT_COLUMN_AMOUNT:
+	case CSVErrorType::MAXIMUM_LINE_SIZE:
+	case CSVErrorType::NULLPADDED_QUOTED_NEW_VALUE:
+		return true;
+	default:
+		return false;
+	}
+}
+
+idx_t CSVErrorHandler::GetLine(LinesPerBoundary &error_info) {
+	idx_t current_line = 1 + error_info.lines_in_batch; // We start from one, since the lines are 1-indexed
+	for (idx_t boundary_idx = 0; boundary_idx < error_info.boundary_idx; boundary_idx++) {
+		bool batch_done = false;
+		while (!batch_done) {
+			if (boundary_idx == 0) {
+				// if it's the first boundary, we just return
+				break;
+			}
+			lock_guard<mutex> parallel_lock(main_mutex);
+			if (lines_per_batch_map.find(boundary_idx) != lines_per_batch_map.end()) {
+				batch_done = true;
+				current_line += lines_per_batch_map[boundary_idx].lines_in_batch;
+			}
+			if (got_borked) {
+				return current_line;
+			}
+		}
+	}
+	return current_line;
+}
+
+} // namespace duckdb