duckdb 0.9.3-dev2.0 → 0.10.0

package/src/duckdb/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp

@@ -0,0 +1,1091 @@
+#include "duckdb/execution/operator/csv_scanner/scanner/string_value_scanner.hpp"
+#include "duckdb/execution/operator/csv_scanner/util/csv_casting.hpp"
+#include "duckdb/execution/operator/csv_scanner/scanner/skip_scanner.hpp"
+#include "duckdb/execution/operator/csv_scanner/table_function/csv_file_scanner.hpp"
+#include "duckdb/main/client_data.hpp"
+#include "duckdb/common/operator/integer_cast_operator.hpp"
+#include "duckdb/common/operator/double_cast_operator.hpp"
+#include <algorithm>
+
+namespace duckdb {
+
+StringValueResult::StringValueResult(CSVStates &states, CSVStateMachine &state_machine,
+                                     const shared_ptr<CSVBufferHandle> &buffer_handle, Allocator &buffer_allocator,
+                                     idx_t result_size_p, idx_t buffer_position, CSVErrorHandler &error_hander_p,
+                                     CSVIterator &iterator_p, bool store_line_size_p,
+                                     shared_ptr<CSVFileScan> csv_file_scan_p, idx_t &lines_read_p)
+    : ScannerResult(states, state_machine), number_of_columns(state_machine.dialect_options.num_cols),
+      null_padding(state_machine.options.null_padding), ignore_errors(state_machine.options.ignore_errors),
+      null_str_ptr(state_machine.options.null_str.c_str()), null_str_size(state_machine.options.null_str.size()),
+      result_size(result_size_p), error_handler(error_hander_p), iterator(iterator_p),
+      store_line_size(store_line_size_p), csv_file_scan(std::move(csv_file_scan_p)), lines_read(lines_read_p) {
+	// Vector information
+	D_ASSERT(number_of_columns > 0);
+	buffer_handles.push_back(buffer_handle);
+	// Buffer Information
+	buffer_ptr = buffer_handle->Ptr();
+	buffer_size = buffer_handle->actual_size;
+	last_position = buffer_position;
+
+	// Current Result information
+	previous_line_start = {iterator.pos.buffer_idx, iterator.pos.buffer_pos, buffer_handle->actual_size};
+	pre_previous_line_start = previous_line_start;
+	// Fill out Parse Types
+	vector<LogicalType> logical_types;
+	parse_types = make_unsafe_uniq_array<LogicalTypeId>(number_of_columns);
+	if (!csv_file_scan) {
+		for (idx_t i = 0; i < number_of_columns; i++) {
+			parse_types[i] = LogicalTypeId::VARCHAR;
+			logical_types.emplace_back(LogicalType::VARCHAR);
+			string name = "Column_" + to_string(i);
+			names.emplace_back(name);
+		}
+	} else {
+		if (csv_file_scan->file_types.size() > number_of_columns) {
+			throw InvalidInputException(
+			    "Mismatch between the number of columns (%d) in the CSV file and what is expected in the scanner (%d).",
+			    number_of_columns, csv_file_scan->file_types.size());
+		}
+		for (idx_t i = 0; i < csv_file_scan->file_types.size(); i++) {
+			auto &type = csv_file_scan->file_types[i];
+			if (StringValueScanner::CanDirectlyCast(type, state_machine.options.dialect_options.date_format)) {
+				parse_types[i] = type.id();
+				logical_types.emplace_back(type);
+			} else {
+				parse_types[i] = LogicalTypeId::VARCHAR;
+				logical_types.emplace_back(LogicalType::VARCHAR);
+			}
+		}
+		names = csv_file_scan->names;
+		if (!csv_file_scan->projected_columns.empty()) {
+			projecting_columns = false;
+			projected_columns = make_unsafe_uniq_array<bool>(number_of_columns);
+			for (idx_t col_idx = 0; col_idx < number_of_columns; col_idx++) {
+				if (csv_file_scan->projected_columns.find(col_idx) == csv_file_scan->projected_columns.end()) {
+					// Column is not projected
+					projecting_columns = true;
+					projected_columns[col_idx] = false;
+				} else {
+					projected_columns[col_idx] = true;
+				}
+			}
+		}
+		if (!projecting_columns) {
+			for (idx_t j = logical_types.size(); j < number_of_columns; j++) {
+				// This can happen if we have sneaky null columns at the end that we wish to ignore
+				parse_types[j] = LogicalTypeId::VARCHAR;
+				logical_types.emplace_back(LogicalType::VARCHAR);
+			}
+		}
+	}
+
+	// Initialize Parse Chunk
+	parse_chunk.Initialize(buffer_allocator, logical_types, result_size);
+	for (auto &col : parse_chunk.data) {
+		vector_ptr.push_back(FlatVector::GetData<string_t>(col));
+		validity_mask.push_back(&FlatVector::Validity(col));
+	}
+}
+
+inline bool IsValueNull(const char *null_str_ptr, const char *value_ptr, const idx_t size) {
+	for (idx_t i = 0; i < size; i++) {
+		if (null_str_ptr[i] != value_ptr[i]) {
+			return false;
+		}
+	}
+	return true;
+}
+
+void StringValueResult::AddValueToVector(const char *value_ptr, const idx_t size, bool allocate) {
+	if (cur_col_id >= number_of_columns) {
+		bool error = true;
+		if (cur_col_id == number_of_columns && ((quoted && state_machine.options.allow_quoted_nulls) || !quoted)) {
+			// we make an exception if the first over-value is null
+			error = !IsValueNull(null_str_ptr, value_ptr, size);
+		}
+		if (error) {
+			HandleOverLimitRows();
+		}
+	}
+	if (ignore_current_row) {
+		return;
+	}
+	if (projecting_columns) {
+		if (!projected_columns[cur_col_id]) {
+			cur_col_id++;
+			return;
+		}
+	}
+	if (size == null_str_size) {
+		if (((quoted && state_machine.options.allow_quoted_nulls) || !quoted)) {
+			if (IsValueNull(null_str_ptr, value_ptr, size)) {
+				bool empty = false;
+				if (chunk_col_id < state_machine.options.force_not_null.size()) {
+					empty = state_machine.options.force_not_null[chunk_col_id];
+				}
+				if (empty) {
+					if (parse_types[chunk_col_id] != LogicalTypeId::VARCHAR) {
+						// If it is not a varchar, empty values are not accepted, we must error.
+						cast_errors[chunk_col_id] = std::string("");
+					}
+					static_cast<string_t *>(vector_ptr[chunk_col_id])[number_of_rows] = string_t();
+				} else {
+					if (chunk_col_id == number_of_columns) {
+						// We check for a weird case, where we ignore an extra value, if it is a null value
+						return;
+					}
+					validity_mask[chunk_col_id]->SetInvalid(number_of_rows);
+				}
+				cur_col_id++;
+				chunk_col_id++;
+				return;
+			}
+		}
+	}
+	bool success = true;
+	switch (parse_types[chunk_col_id]) {
+	case LogicalTypeId::TINYINT:
+		success = TrySimpleIntegerCast(value_ptr, size, static_cast<int8_t *>(vector_ptr[chunk_col_id])[number_of_rows],
+		                               false);
+		break;
+	case LogicalTypeId::SMALLINT:
+		success = TrySimpleIntegerCast(value_ptr, size,
+		                               static_cast<int16_t *>(vector_ptr[chunk_col_id])[number_of_rows], false);
+		break;
+	case LogicalTypeId::INTEGER:
+		success = TrySimpleIntegerCast(value_ptr, size,
+		                               static_cast<int32_t *>(vector_ptr[chunk_col_id])[number_of_rows], false);
+		break;
+	case LogicalTypeId::BIGINT:
+		success = TrySimpleIntegerCast(value_ptr, size,
+		                               static_cast<int64_t *>(vector_ptr[chunk_col_id])[number_of_rows], false);
+		break;
+	case LogicalTypeId::UTINYINT:
+		success = TrySimpleIntegerCast<uint8_t, false>(
+		    value_ptr, size, static_cast<uint8_t *>(vector_ptr[chunk_col_id])[number_of_rows], false);
+		break;
+	case LogicalTypeId::USMALLINT:
+		success = TrySimpleIntegerCast<uint16_t, false>(
+		    value_ptr, size, static_cast<uint16_t *>(vector_ptr[chunk_col_id])[number_of_rows], false);
+		break;
+	case LogicalTypeId::UINTEGER:
+		success = TrySimpleIntegerCast<uint32_t, false>(
+		    value_ptr, size, static_cast<uint32_t *>(vector_ptr[chunk_col_id])[number_of_rows], false);
+		break;
+	case LogicalTypeId::UBIGINT:
+		success = TrySimpleIntegerCast<uint64_t, false>(
+		    value_ptr, size, static_cast<uint64_t *>(vector_ptr[chunk_col_id])[number_of_rows], false);
+		break;
+	case LogicalTypeId::DOUBLE:
+		success =
+		    TryDoubleCast<double>(value_ptr, size, static_cast<double *>(vector_ptr[chunk_col_id])[number_of_rows],
+		                          false, state_machine.options.decimal_separator[0]);
+		break;
+	case LogicalTypeId::FLOAT:
+		success = TryDoubleCast<float>(value_ptr, size, static_cast<float *>(vector_ptr[chunk_col_id])[number_of_rows],
+		                               false, state_machine.options.decimal_separator[0]);
+		break;
+	case LogicalTypeId::DATE: {
+		idx_t pos;
+		bool special;
+		success = Date::TryConvertDate(value_ptr, size, pos,
+		                               static_cast<date_t *>(vector_ptr[chunk_col_id])[number_of_rows], special, false);
+		break;
+	}
+	case LogicalTypeId::TIMESTAMP: {
+		success = Timestamp::TryConvertTimestamp(
+		              value_ptr, size, static_cast<timestamp_t *>(vector_ptr[chunk_col_id])[number_of_rows]) ==
+		          TimestampCastResult::SUCCESS;
+		break;
+	}
+	default:
+		if (allocate) {
+			static_cast<string_t *>(vector_ptr[chunk_col_id])[number_of_rows] =
+			    StringVector::AddStringOrBlob(parse_chunk.data[chunk_col_id], string_t(value_ptr, size));
+		} else {
+			static_cast<string_t *>(vector_ptr[chunk_col_id])[number_of_rows] = string_t(value_ptr, size);
+		}
+		break;
+	}
+	if (!success) {
+		// We had a casting error, we push it here because we can only error when finishing the line read.
+		cast_errors[cur_col_id] = std::string(value_ptr, size);
+	}
+	cur_col_id++;
+	chunk_col_id++;
+}
+
+Value StringValueResult::GetValue(idx_t row_idx, idx_t col_idx) {
+	if (validity_mask[col_idx]->AllValid()) {
+		return Value(static_cast<string_t *>(vector_ptr[col_idx])[row_idx]);
+	} else {
+		if (validity_mask[col_idx]->RowIsValid(row_idx)) {
+			return Value(static_cast<string_t *>(vector_ptr[col_idx])[row_idx]);
+		} else {
+			return Value();
+		}
+	}
+}
+DataChunk &StringValueResult::ToChunk() {
+	parse_chunk.SetCardinality(number_of_rows);
+	return parse_chunk;
+}
+
+void StringValueResult::Reset() {
+	if (number_of_rows == 0) {
+		return;
+	}
+	number_of_rows = 0;
+	cur_col_id = 0;
+	chunk_col_id = 0;
+	for (auto &v : validity_mask) {
+		v->SetAllValid(result_size);
+	}
+	buffer_handles.clear();
+}
+
+void StringValueResult::AddQuotedValue(StringValueResult &result, const idx_t buffer_pos) {
+	if (result.escaped) {
+		if (result.projecting_columns) {
+			if (!result.projected_columns[result.cur_col_id]) {
+				result.cur_col_id++;
+				result.quoted = false;
+				result.escaped = false;
+				return;
+			}
+		}
+		// If it's an escaped value we have to remove all the escapes, this is not really great
+		auto value = StringValueScanner::RemoveEscape(
+		    result.buffer_ptr + result.quoted_position + 1, buffer_pos - result.quoted_position - 2,
+		    result.state_machine.dialect_options.state_machine_options.escape.GetValue(),
+		    result.parse_chunk.data[result.chunk_col_id]);
+		result.AddValueToVector(value.GetData(), value.GetSize());
+	} else {
+		if (buffer_pos < result.last_position + 2) {
+			// empty value
+			auto value = string_t();
+			result.AddValueToVector(value.GetData(), value.GetSize());
+		} else {
+			result.AddValueToVector(result.buffer_ptr + result.quoted_position + 1,
+			                        buffer_pos - result.quoted_position - 2);
+		}
+	}
+	result.quoted = false;
+	result.escaped = false;
+}
+
+void StringValueResult::AddValue(StringValueResult &result, const idx_t buffer_pos) {
+	if (result.last_position > buffer_pos) {
+		return;
+	}
+	if (result.quoted) {
+		StringValueResult::AddQuotedValue(result, buffer_pos);
+	} else {
+		result.AddValueToVector(result.buffer_ptr + result.last_position, buffer_pos - result.last_position);
+	}
+	result.last_position = buffer_pos + 1;
+}
+
+void StringValueResult::HandleOverLimitRows() {
+	auto csv_error =
+	    CSVError::IncorrectColumnAmountError(state_machine.options, nullptr, number_of_columns, cur_col_id + 1);
+	LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), number_of_rows + 1);
+	error_handler.Error(lines_per_batch, csv_error);
+	// If we get here we need to remove the last line
+	cur_col_id = 0;
+	chunk_col_id = 0;
+	ignore_current_row = true;
+}
+
+void StringValueResult::QuotedNewLine(StringValueResult &result) {
+	result.quoted_new_line = true;
+}
+
+void StringValueResult::NullPaddingQuotedNewlineCheck() {
+	// We do some checks for null_padding correctness
+	if (state_machine.options.null_padding && iterator.IsBoundarySet() && quoted_new_line && iterator.done) {
+		// If we have null_padding set, we found a quoted new line, we are scanning the file in parallel and it's the
+		// last row of this thread.
+		auto csv_error = CSVError::NullPaddingFail(state_machine.options);
+		LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), number_of_rows + 1);
+		error_handler.Error(lines_per_batch, csv_error, true);
+	}
+}
+
+bool StringValueResult::AddRowInternal() {
+	if (ignore_current_row) {
+		// An error occurred on this row, we are ignoring it and resetting our control flag
+		ignore_current_row = false;
+		return false;
+	}
+	if (!cast_errors.empty()) {
+		// A wild casting error appears
+		// Recreate row for rejects-table
+		vector<Value> row;
+		if (!state_machine.options.rejects_table_name.empty()) {
+			for (idx_t col = 0; col < parse_chunk.ColumnCount(); col++) {
+				if (cast_errors.find(col) != cast_errors.end()) {
+					row.push_back(cast_errors[col]);
+				} else {
+					row.push_back(parse_chunk.data[col].GetValue(number_of_rows));
+				}
+			}
+		}
+		for (auto &cast_error : cast_errors) {
+			std::ostringstream error;
+			// Casting Error Message
+			error << "Could not convert string \"" << cast_error.second << "\" to \'"
+			      << LogicalTypeIdToString(parse_types[cast_error.first]) << "\'";
+			auto error_string = error.str();
+			auto csv_error = CSVError::CastError(state_machine.options, names[cast_error.first], error_string,
+			                                     cast_error.first, row);
+			LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read - 1);
+			error_handler.Error(lines_per_batch, csv_error);
+		}
+		// If we got here it means we are ignoring errors, hence we need to signify to our result scanner to ignore this
+		// row
+		// Cleanup this line and continue
+		cast_errors.clear();
+		cur_col_id = 0;
+		chunk_col_id = 0;
+		return false;
+	}
+	NullPaddingQuotedNewlineCheck();
+	quoted_new_line = false;
+	// We need to check if we are getting the correct number of columns here.
+	// If columns are correct, we add it, and that's it.
+	if (cur_col_id != number_of_columns) {
+		// We have too few columns:
+		if (null_padding) {
+			while (cur_col_id < number_of_columns) {
+				bool empty = false;
+				if (cur_col_id < state_machine.options.force_not_null.size()) {
+					empty = state_machine.options.force_not_null[cur_col_id];
+				}
+				if (projecting_columns) {
+					if (!projected_columns[cur_col_id]) {
+						cur_col_id++;
+						continue;
+					}
+				}
+				if (empty) {
+					static_cast<string_t *>(vector_ptr[chunk_col_id])[number_of_rows] = string_t();
+				} else {
+					validity_mask[chunk_col_id]->SetInvalid(number_of_rows);
+				}
+				cur_col_id++;
+				chunk_col_id++;
+			}
+		} else {
+			// If we are not null-padding this is an error
+			auto csv_error =
+			    CSVError::IncorrectColumnAmountError(state_machine.options, nullptr, number_of_columns, cur_col_id);
+			LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), number_of_rows + 1);
+			error_handler.Error(lines_per_batch, csv_error);
+			// If we are here we ignore_errors, so we delete this line
+			number_of_rows--;
+		}
+	}
+	cur_col_id = 0;
+	chunk_col_id = 0;
+	number_of_rows++;
+	if (number_of_rows >= result_size) {
+		// We have a full chunk
+		return true;
+	}
+	return false;
+}
+
+bool StringValueResult::AddRow(StringValueResult &result, const idx_t buffer_pos) {
+	if (result.last_position <= buffer_pos) {
+		LinePosition current_line_start = {result.iterator.pos.buffer_idx, result.iterator.pos.buffer_pos,
+		                                   result.buffer_size};
+		idx_t current_line_size = current_line_start - result.previous_line_start;
+		if (result.store_line_size) {
+			result.error_handler.NewMaxLineSize(current_line_size);
+		}
+		if (current_line_size > result.state_machine.options.maximum_line_size) {
+			auto csv_error = CSVError::LineSizeError(result.state_machine.options, current_line_size);
+			LinesPerBoundary lines_per_batch(result.iterator.GetBoundaryIdx(), result.number_of_rows);
+			result.error_handler.Error(lines_per_batch, csv_error, true);
+		}
+		result.pre_previous_line_start = result.previous_line_start;
+		result.previous_line_start = current_line_start;
+		// We add the value
+		if (result.quoted) {
+			StringValueResult::AddQuotedValue(result, buffer_pos);
+		} else {
+			result.AddValueToVector(result.buffer_ptr + result.last_position, buffer_pos - result.last_position);
+		}
+		if (result.state_machine.dialect_options.state_machine_options.new_line == NewLineIdentifier::CARRY_ON) {
+			if (result.states.states[1] == CSVState::RECORD_SEPARATOR) {
+				// Even though this is marked as a carry on, this is a hippie mixie
+				result.last_position = buffer_pos + 1;
+			} else {
+				result.last_position = buffer_pos + 2;
+			}
+		} else {
+			result.last_position = buffer_pos + 1;
+		}
+	}
+
+	// We add the value
+	return result.AddRowInternal();
+}
+
+void StringValueResult::InvalidState(StringValueResult &result) {
+	// FIXME: How do we recover from an invalid state? Can we restart the state machine and jump to the next row?
+	auto csv_error = CSVError::UnterminatedQuotesError(result.state_machine.options,
+	                                                   static_cast<string_t *>(result.vector_ptr[result.chunk_col_id]),
+	                                                   result.number_of_rows, result.cur_col_id);
+	LinesPerBoundary lines_per_batch(result.iterator.GetBoundaryIdx(), result.number_of_rows);
+	result.error_handler.Error(lines_per_batch, csv_error);
+}
+
+bool StringValueResult::EmptyLine(StringValueResult &result, const idx_t buffer_pos) {
+	// We care about empty lines if this is a single column csv file
+	result.last_position = buffer_pos + 1;
+	if (result.states.IsCarriageReturn() &&
+	    result.state_machine.dialect_options.state_machine_options.new_line == NewLineIdentifier::CARRY_ON) {
+		result.last_position++;
+	}
+	if (result.number_of_columns == 1) {
+		if (result.null_str_size == 0) {
+			bool empty = false;
+			if (!result.state_machine.options.force_not_null.empty()) {
+				empty = result.state_machine.options.force_not_null[0];
+			}
+			if (empty) {
+				static_cast<string_t *>(result.vector_ptr[0])[result.number_of_rows] = string_t();
+			} else {
+				result.validity_mask[0]->SetInvalid(result.number_of_rows);
+			}
+			result.number_of_rows++;
+		}
+		if (result.number_of_rows >= result.result_size) {
+			// We have a full chunk
+			return true;
+		}
+	}
+	return false;
+}
+
+StringValueScanner::StringValueScanner(idx_t scanner_idx_p, const shared_ptr<CSVBufferManager> &buffer_manager,
+                                       const shared_ptr<CSVStateMachine> &state_machine,
+                                       const shared_ptr<CSVErrorHandler> &error_handler,
+                                       const shared_ptr<CSVFileScan> &csv_file_scan, CSVIterator boundary,
+                                       idx_t result_size)
+    : BaseScanner(buffer_manager, state_machine, error_handler, csv_file_scan, boundary), scanner_idx(scanner_idx_p),
+      result(states, *state_machine, cur_buffer_handle, BufferAllocator::Get(buffer_manager->context), result_size,
+             iterator.pos.buffer_pos, *error_handler, iterator,
+             buffer_manager->context.client_data->debug_set_max_line_length, csv_file_scan, lines_read) {
+}
+
+StringValueScanner::StringValueScanner(const shared_ptr<CSVBufferManager> &buffer_manager,
+                                       const shared_ptr<CSVStateMachine> &state_machine,
+                                       const shared_ptr<CSVErrorHandler> &error_handler)
+    : BaseScanner(buffer_manager, state_machine, error_handler, nullptr, {}), scanner_idx(0),
+      result(states, *state_machine, cur_buffer_handle, Allocator::DefaultAllocator(), STANDARD_VECTOR_SIZE,
+             iterator.pos.buffer_pos, *error_handler, iterator,
+             buffer_manager->context.client_data->debug_set_max_line_length, csv_file_scan, lines_read) {
+}
+
+unique_ptr<StringValueScanner> StringValueScanner::GetCSVScanner(ClientContext &context, CSVReaderOptions &options) {
+	auto state_machine = make_shared<CSVStateMachine>(options, options.dialect_options.state_machine_options,
+	                                                  CSVStateMachineCache::Get(context));
+
+	state_machine->dialect_options.num_cols = options.dialect_options.num_cols;
+	state_machine->dialect_options.header = options.dialect_options.header;
+	auto buffer_manager = make_shared<CSVBufferManager>(context, options, options.file_path, 0);
+	auto scanner = make_uniq<StringValueScanner>(buffer_manager, state_machine, make_shared<CSVErrorHandler>());
+	scanner->csv_file_scan = make_shared<CSVFileScan>(context, options.file_path, options);
+	scanner->csv_file_scan->InitializeProjection();
+	return scanner;
+}
+
+bool StringValueScanner::FinishedIterator() {
+	return iterator.done;
+}
+
+StringValueResult &StringValueScanner::ParseChunk() {
+	result.Reset();
+	ParseChunkInternal(result);
+	return result;
+}
+
+void StringValueScanner::Flush(DataChunk &insert_chunk) {
+	auto &process_result = ParseChunk();
+	// First Get Parsed Chunk
+	auto &parse_chunk = process_result.ToChunk();
+
+	if (parse_chunk.size() == 0) {
+		return;
+	}
+	// convert the columns in the parsed chunk to the types of the table
+	insert_chunk.SetCardinality(parse_chunk);
+
+	// We keep track of the borked lines, in case we are ignoring errors
+	unordered_set<idx_t> borked_lines;
+	D_ASSERT(csv_file_scan);
+
+	auto &reader_data = csv_file_scan->reader_data;
+	// Now Do the cast-aroo
+	for (idx_t c = 0; c < reader_data.column_ids.size(); c++) {
+		idx_t col_idx = c;
+		idx_t result_idx = reader_data.column_mapping[c];
+		if (!csv_file_scan->projection_ids.empty()) {
+			result_idx = reader_data.column_mapping[csv_file_scan->projection_ids[c].second];
+		}
+		if (col_idx >= parse_chunk.ColumnCount()) {
+			throw InvalidInputException("Mismatch between the schema of different files");
+		}
+		auto &parse_vector = parse_chunk.data[col_idx];
+		auto &result_vector = insert_chunk.data[result_idx];
+		auto &type = result_vector.GetType();
+		auto &parse_type = parse_vector.GetType();
+		if (type == LogicalType::VARCHAR || (type != LogicalType::VARCHAR && parse_type != LogicalType::VARCHAR)) {
+			// reinterpret rather than reference
+			result_vector.Reinterpret(parse_vector);
+		} else {
+			string error_message;
+			bool success;
+			idx_t line_error = 0;
+			bool line_error_set = true;
+
+			if (!state_machine->options.dialect_options.date_format.at(LogicalTypeId::DATE).GetValue().Empty() &&
+			    type.id() == LogicalTypeId::DATE) {
+				// use the date format to cast the chunk
+				success = CSVCast::TryCastDateVector(state_machine->options.dialect_options.date_format, parse_vector,
+				                                     result_vector, parse_chunk.size(), error_message, line_error);
+			} else if (!state_machine->options.dialect_options.date_format.at(LogicalTypeId::TIMESTAMP)
+			                .GetValue()
+			                .Empty() &&
+			           type.id() == LogicalTypeId::TIMESTAMP) {
+				// use the date format to cast the chunk
+				success =
+				    CSVCast::TryCastTimestampVector(state_machine->options.dialect_options.date_format, parse_vector,
+				                                    result_vector, parse_chunk.size(), error_message);
+			} else if (state_machine->options.decimal_separator != "." &&
+			           (type.id() == LogicalTypeId::FLOAT || type.id() == LogicalTypeId::DOUBLE)) {
+				success =
+				    CSVCast::TryCastFloatingVectorCommaSeparated(state_machine->options, parse_vector, result_vector,
+				                                                 parse_chunk.size(), error_message, type, line_error);
+			} else if (state_machine->options.decimal_separator != "." && type.id() == LogicalTypeId::DECIMAL) {
+				success = CSVCast::TryCastDecimalVectorCommaSeparated(
+				    state_machine->options, parse_vector, result_vector, parse_chunk.size(), error_message, type);
+			} else {
+				// target type is not varchar: perform a cast
+				success = VectorOperations::TryCast(buffer_manager->context, parse_vector, result_vector,
+				                                    parse_chunk.size(), &error_message);
+				line_error_set = false;
+			}
+			if (success) {
+				continue;
+			}
+			// An error happened, to propagate it we need to figure out the exact line where the casting failed.
+			UnifiedVectorFormat inserted_column_data;
+			result_vector.ToUnifiedFormat(parse_chunk.size(), inserted_column_data);
+			UnifiedVectorFormat parse_column_data;
+			parse_vector.ToUnifiedFormat(parse_chunk.size(), parse_column_data);
+			if (!line_error_set) {
+				for (; line_error < parse_chunk.size(); line_error++) {
+					if (!inserted_column_data.validity.RowIsValid(line_error) &&
+					    parse_column_data.validity.RowIsValid(line_error)) {
+						break;
+					}
+				}
+			}
+			{
+				vector<Value> row;
+				for (idx_t col = 0; col < parse_chunk.ColumnCount(); col++) {
+					row.push_back(parse_chunk.GetValue(col, line_error));
+				}
+				auto csv_error = CSVError::CastError(state_machine->options, csv_file_scan->names[col_idx],
+				                                     error_message, col_idx, row);
+				LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(),
+				                                 lines_read - parse_chunk.size() + line_error);
+				error_handler->Error(lines_per_batch, csv_error);
+			}
+			borked_lines.insert(line_error++);
+			D_ASSERT(state_machine->options.ignore_errors);
+			// We are ignoring errors. We must continue but ignoring borked rows
+			for (; line_error < parse_chunk.size(); line_error++) {
+				if (!inserted_column_data.validity.RowIsValid(line_error) &&
+				    parse_column_data.validity.RowIsValid(line_error)) {
+					borked_lines.insert(line_error);
+					vector<Value> row;
+					for (idx_t col = 0; col < parse_chunk.ColumnCount(); col++) {
+						row.push_back(parse_chunk.GetValue(col, line_error));
+					}
+					auto csv_error = CSVError::CastError(state_machine->options, csv_file_scan->names[col_idx],
+					                                     error_message, col_idx, row);
+					LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(),
+					                                 lines_read - parse_chunk.size() + line_error);
+					error_handler->Error(lines_per_batch, csv_error);
+				}
+			}
+		}
+	}
+	if (!borked_lines.empty()) {
+		// We must remove the borked lines from our chunk
+		SelectionVector succesful_rows(parse_chunk.size() - borked_lines.size());
+		idx_t sel_idx = 0;
+		for (idx_t row_idx = 0; row_idx < parse_chunk.size(); row_idx++) {
+			if (borked_lines.find(row_idx) == borked_lines.end()) {
+				succesful_rows.set_index(sel_idx++, row_idx);
+			}
+		}
+		// Now we slice the result
+		insert_chunk.Slice(succesful_rows, sel_idx);
+	}
+}
+
+void StringValueScanner::Initialize() {
+	states.Initialize();
+
+	if (result.result_size != 1 && !(sniffing && state_machine->options.null_padding &&
+	                                 !state_machine->options.dialect_options.skip_rows.IsSetByUser())) {
+		SetStart();
+	}
+	result.last_position = iterator.pos.buffer_pos;
+	result.previous_line_start = {iterator.pos.buffer_idx, iterator.pos.buffer_pos, cur_buffer_handle->actual_size};
+
+	result.pre_previous_line_start = result.previous_line_start;
+}
+
+void StringValueScanner::ProcessExtraRow() {
+	result.NullPaddingQuotedNewlineCheck();
+	idx_t to_pos = cur_buffer_handle->actual_size;
+	while (iterator.pos.buffer_pos < to_pos) {
+		state_machine->Transition(states, buffer_handle_ptr[iterator.pos.buffer_pos]);
+		switch (states.states[1]) {
+		case CSVState::INVALID:
+			result.InvalidState(result);
+			iterator.pos.buffer_pos++;
+			return;
+		case CSVState::RECORD_SEPARATOR:
+			if (states.states[0] == CSVState::RECORD_SEPARATOR) {
+				lines_read++;
+				result.EmptyLine(result, iterator.pos.buffer_pos);
+				iterator.pos.buffer_pos++;
+				return;
+			} else if (states.states[0] != CSVState::CARRIAGE_RETURN) {
+				lines_read++;
+				result.AddRow(result, iterator.pos.buffer_pos);
+				iterator.pos.buffer_pos++;
+				return;
+			}
+			iterator.pos.buffer_pos++;
+			break;
+		case CSVState::CARRIAGE_RETURN:
+			lines_read++;
+			if (states.states[0] != CSVState::RECORD_SEPARATOR) {
+				result.AddRow(result, iterator.pos.buffer_pos);
+				iterator.pos.buffer_pos++;
+				return;
+			} else {
+				result.EmptyLine(result, iterator.pos.buffer_pos);
+				iterator.pos.buffer_pos++;
+				return;
+			}
+		case CSVState::DELIMITER:
+			result.AddValue(result, iterator.pos.buffer_pos);
+			iterator.pos.buffer_pos++;
+			break;
+		case CSVState::QUOTED:
+			if (states.states[0] == CSVState::UNQUOTED) {
+				result.SetEscaped(result);
+			}
+			result.SetQuoted(result, iterator.pos.buffer_pos);
+			iterator.pos.buffer_pos++;
+			while (state_machine->transition_array
+			           .skip_quoted[static_cast<uint8_t>(buffer_handle_ptr[iterator.pos.buffer_pos])] &&
+			       iterator.pos.buffer_pos < to_pos - 1) {
+				iterator.pos.buffer_pos++;
+			}
+			break;
+		case CSVState::ESCAPE:
+			result.SetEscaped(result);
+			iterator.pos.buffer_pos++;
+			break;
+		case CSVState::STANDARD:
+			iterator.pos.buffer_pos++;
+			while (state_machine->transition_array
+			           .skip_standard[static_cast<uint8_t>(buffer_handle_ptr[iterator.pos.buffer_pos])] &&
+			       iterator.pos.buffer_pos < to_pos - 1) {
+				iterator.pos.buffer_pos++;
+			}
+			break;
+		case CSVState::QUOTED_NEW_LINE:
+			result.quoted_new_line = true;
+			result.NullPaddingQuotedNewlineCheck();
+			iterator.pos.buffer_pos++;
+			break;
+		default:
+			iterator.pos.buffer_pos++;
+			break;
+		}
+	}
+}
+
+string_t StringValueScanner::RemoveEscape(const char *str_ptr, idx_t end, char escape, Vector &vector) {
+	// Figure out the exact size
+	idx_t str_pos = 0;
+	bool just_escaped = false;
+	for (idx_t cur_pos = 0; cur_pos < end; cur_pos++) {
+		if (str_ptr[cur_pos] == escape && !just_escaped) {
+			just_escaped = true;
+		} else {
+			just_escaped = false;
+			str_pos++;
+		}
+	}
+
+	auto removed_escapes = StringVector::EmptyString(vector, str_pos);
+	auto removed_escapes_ptr = removed_escapes.GetDataWriteable();
+	// Allocate string and copy it
+	str_pos = 0;
+	just_escaped = false;
+	for (idx_t cur_pos = 0; cur_pos < end; cur_pos++) {
+		char c = str_ptr[cur_pos];
+		if (c == escape && !just_escaped) {
+			just_escaped = true;
+		} else {
+			just_escaped = false;
+			removed_escapes_ptr[str_pos++] = c;
+		}
+	}
+	removed_escapes.Finalize();
+	return removed_escapes;
+}
+
+void StringValueScanner::ProcessOverbufferValue() {
+	// Process first string
+	states.Initialize();
+	string overbuffer_string;
+	auto previous_buffer = previous_buffer_handle->Ptr();
+	if (result.last_position == previous_buffer_handle->actual_size) {
+		state_machine->Transition(states, previous_buffer[result.last_position - 1]);
+	}
+	idx_t j = 0;
+	result.quoted = false;
+	for (idx_t i = result.last_position; i < previous_buffer_handle->actual_size; i++) {
+		state_machine->Transition(states, previous_buffer[i]);
+		if (states.EmptyLine() || states.IsCurrentNewRow()) {
+			continue;
+		}
+		if (states.NewRow() || states.NewValue()) {
+			break;
+		} else {
+			overbuffer_string += previous_buffer[i];
+		}
+		if (states.IsQuoted()) {
+			result.SetQuoted(result, j);
+		}
+		if (states.IsEscaped()) {
+			result.escaped = true;
+		}
+		j++;
+	}
+	if (overbuffer_string.empty() &&
+	    state_machine->dialect_options.state_machine_options.new_line == NewLineIdentifier::CARRY_ON) {
+		if (buffer_handle_ptr[iterator.pos.buffer_pos] == '\n') {
+			iterator.pos.buffer_pos++;
+		}
+	}
+	// second buffer
+	for (; iterator.pos.buffer_pos < cur_buffer_handle->actual_size; iterator.pos.buffer_pos++) {
+		state_machine->Transition(states, buffer_handle_ptr[iterator.pos.buffer_pos]);
+		if (states.EmptyLine()) {
+			if (state_machine->dialect_options.num_cols == 1) {
+				break;
+			} else {
+				continue;
+			}
+		}
+		if (states.NewRow() || states.NewValue()) {
+			break;
+		} else {
+			overbuffer_string += buffer_handle_ptr[iterator.pos.buffer_pos];
+		}
+		if (states.IsQuoted()) {
+			result.SetQuoted(result, j);
+		}
+		if (states.IsEscaped()) {
+			result.escaped = true;
+		}
+		j++;
+	}
+	string_t value;
+	if (result.quoted) {
+		value = string_t(overbuffer_string.c_str() + result.quoted_position,
+		                 overbuffer_string.size() - 1 - result.quoted_position);
+		if (result.escaped) {
+			const auto str_ptr = static_cast<const char *>(overbuffer_string.c_str() + result.quoted_position);
+			value =
+			    StringValueScanner::RemoveEscape(str_ptr, overbuffer_string.size() - 2,
+			                                     state_machine->dialect_options.state_machine_options.escape.GetValue(),
+			                                     result.parse_chunk.data[result.chunk_col_id]);
+		}
+	} else {
+		value = string_t(overbuffer_string.c_str(), overbuffer_string.size());
+	}
+
+	if (states.EmptyLine() && state_machine->dialect_options.num_cols == 1) {
+		result.EmptyLine(result, iterator.pos.buffer_pos);
+	} else if (!states.IsNotSet()) {
+		result.AddValueToVector(value.GetData(), value.GetSize(), true);
+	}
+
+	if (states.NewRow() && !states.IsNotSet()) {
+		result.AddRowInternal();
+		lines_read++;
+	}
+
+	if (iterator.pos.buffer_pos >= cur_buffer_handle->actual_size && cur_buffer_handle->is_last_buffer) {
+		result.added_last_line = true;
+	}
+	if (states.IsCarriageReturn() &&
+	    state_machine->dialect_options.state_machine_options.new_line == NewLineIdentifier::CARRY_ON) {
+		result.last_position = ++iterator.pos.buffer_pos + 1;
+	} else {
+		result.last_position = ++iterator.pos.buffer_pos;
+	}
+	// Be sure to reset the quoted and escaped variables
+	result.quoted = false;
+	result.escaped = false;
+}
+
+bool StringValueScanner::MoveToNextBuffer() {
+	if (iterator.pos.buffer_pos >= cur_buffer_handle->actual_size) {
+		previous_buffer_handle = cur_buffer_handle;
+		cur_buffer_handle = buffer_manager->GetBuffer(++iterator.pos.buffer_idx);
+		result.buffer_handles.push_back(cur_buffer_handle);
+		if (!cur_buffer_handle) {
+			iterator.pos.buffer_idx--;
+			buffer_handle_ptr = nullptr;
+			// We do not care if it's a quoted new line on the last row of our file.
+			result.quoted_new_line = false;
+			// This means we reached the end of the file, we must add a last line if there is any to be added
+			if (states.EmptyLine() || states.NewRow() || result.added_last_line || states.IsCurrentNewRow() ||
+			    states.IsNotSet()) {
+				if (result.cur_col_id == result.number_of_columns) {
+					result.number_of_rows++;
+				}
+				result.cur_col_id = 0;
+				result.chunk_col_id = 0;
+				return false;
+			} else if (states.NewValue()) {
+				lines_read++;
+				// we add the value
+				result.AddValue(result, previous_buffer_handle->actual_size);
+				// And an extra empty value to represent what comes after the delimiter
+				result.AddRow(result, previous_buffer_handle->actual_size);
+			} else if (states.IsQuotedCurrent()) {
+				// Unterminated quote
+				result.InvalidState(result);
+			} else {
+				lines_read++;
+				result.AddRow(result, previous_buffer_handle->actual_size);
+			}
+			return false;
+		}
+		iterator.pos.buffer_pos = 0;
+		buffer_handle_ptr = cur_buffer_handle->Ptr();
+		// Handle overbuffer value
+		ProcessOverbufferValue();
+		result.buffer_ptr = buffer_handle_ptr;
+		result.buffer_size = cur_buffer_handle->actual_size;
+		return true;
+	}
+	return false;
+}
+
+void StringValueScanner::SkipBOM() {
+	if (cur_buffer_handle->actual_size >= 3 && result.buffer_ptr[0] == '\xEF' && result.buffer_ptr[1] == '\xBB' &&
+	    result.buffer_ptr[2] == '\xBF') {
+		iterator.pos.buffer_pos = 3;
+	}
+}
+
+void StringValueScanner::SkipCSVRows() {
+	idx_t rows_to_skip =
+	    state_machine->dialect_options.skip_rows.GetValue() + state_machine->dialect_options.header.GetValue();
+	if (rows_to_skip == 0) {
+		return;
+	}
+	SkipScanner row_skipper(buffer_manager, state_machine, error_handler, rows_to_skip);
+	row_skipper.ParseChunk();
+	iterator.pos.buffer_pos = row_skipper.GetIteratorPosition();
+	if (row_skipper.state_machine->options.dialect_options.state_machine_options.new_line ==
+	        NewLineIdentifier::CARRY_ON &&
+	    row_skipper.states.states[1] == CSVState::CARRIAGE_RETURN) {
+		iterator.pos.buffer_pos++;
+	}
+	if (result.store_line_size) {
+		result.error_handler.NewMaxLineSize(iterator.pos.buffer_pos);
+	}
+	lines_read += row_skipper.GetLinesRead();
+}
+
+void StringValueScanner::SkipUntilNewLine() {
+	// Now skip until next newline
+	if (state_machine->options.dialect_options.state_machine_options.new_line.GetValue() ==
+	    NewLineIdentifier::CARRY_ON) {
+		bool carriage_return = false;
+		for (; iterator.pos.buffer_pos < cur_buffer_handle->actual_size; iterator.pos.buffer_pos++) {
+			if (buffer_handle_ptr[iterator.pos.buffer_pos] == '\r') {
+				carriage_return = true;
+			}
+			if (buffer_handle_ptr[iterator.pos.buffer_pos] == '\n') {
+				if (carriage_return) {
+					iterator.pos.buffer_pos++;
+					return;
+				}
+			}
+		}
+	} else {
+		for (; iterator.pos.buffer_pos < cur_buffer_handle->actual_size; iterator.pos.buffer_pos++) {
+			if (buffer_handle_ptr[iterator.pos.buffer_pos] == '\n' ||
+			    buffer_handle_ptr[iterator.pos.buffer_pos] == '\r') {
+				iterator.pos.buffer_pos++;
+				return;
+			}
+		}
+	}
+}
+
+bool StringValueScanner::CanDirectlyCast(const LogicalType &type,
+                                         const map<LogicalTypeId, CSVOption<StrpTimeFormat>> &format_options) {
+
+	switch (type.id()) {
+		// All Integers (Except HugeInt)
+	case LogicalTypeId::TINYINT:
+	case LogicalTypeId::SMALLINT:
+	case LogicalTypeId::INTEGER:
+	case LogicalTypeId::BIGINT:
+	case LogicalTypeId::UTINYINT:
+	case LogicalTypeId::USMALLINT:
+	case LogicalTypeId::UINTEGER:
+	case LogicalTypeId::UBIGINT:
+	case LogicalTypeId::DOUBLE:
+	case LogicalTypeId::FLOAT:
+		return true;
+	case LogicalTypeId::DATE:
+		// We can only internally cast YYYY-MM-DD
+		if (format_options.at(LogicalTypeId::DATE).GetValue().format_specifier == "%Y-%m-%d") {
+			return true;
+		} else {
+			return false;
+		}
+	case LogicalTypeId::TIMESTAMP:
+		if (format_options.at(LogicalTypeId::TIMESTAMP).GetValue().format_specifier == "%Y-%m-%d %H:%M:%S") {
+			return true;
+		} else {
+			return false;
+		}
+	case LogicalType::VARCHAR:
+		return true;
+	default:
+		return false;
+	}
+}
+
+void StringValueScanner::SetStart() {
+	if (iterator.pos.buffer_idx == 0 && iterator.pos.buffer_pos == 0) {
+		// This means this is the very first buffer
+		// This CSV is not from auto-detect, so we don't know where exactly it starts
+		// Hence we potentially have to skip empty lines and headers.
+		SkipBOM();
+		SkipCSVRows();
+		return;
+	}
+	// We have to look for a new line that fits our schema
+	// 1. We walk until the next new line
+	bool line_found;
+	unique_ptr<StringValueScanner> scan_finder;
+	do {
+		SkipUntilNewLine();
+		if (state_machine->options.null_padding) {
+			// When Null Padding, we assume we start from the correct new-line
+			return;
+		}
+		scan_finder = make_uniq<StringValueScanner>(0, buffer_manager, state_machine,
+		                                            make_shared<CSVErrorHandler>(true), csv_file_scan, iterator, 1);
+		auto &tuples = scan_finder->ParseChunk();
+		line_found = true;
+		if (tuples.number_of_rows != 1) {
+			line_found = false;
+			// If no tuples were parsed, this is not the correct start, we need to skip until the next new line
+			// Or if columns don't match, this is not the correct start, we need to skip until the next new line
+			if (scan_finder->previous_buffer_handle) {
+				if (scan_finder->iterator.pos.buffer_pos >= scan_finder->previous_buffer_handle->actual_size &&
+				    scan_finder->previous_buffer_handle->is_last_buffer) {
+					iterator.pos.buffer_idx = scan_finder->iterator.pos.buffer_idx;
+					iterator.pos.buffer_pos = scan_finder->iterator.pos.buffer_pos;
+					result.last_position = iterator.pos.buffer_pos;
+					iterator.done = scan_finder->iterator.done;
+					return;
+				}
+			}
+		}
+	} while (!line_found);
+	iterator.pos.buffer_idx = scan_finder->result.pre_previous_line_start.buffer_idx;
+	iterator.pos.buffer_pos = scan_finder->result.pre_previous_line_start.buffer_pos;
+	result.last_position = iterator.pos.buffer_pos;
+}
+
+void StringValueScanner::FinalizeChunkProcess() {
+	if (result.number_of_rows >= result.result_size || iterator.done) {
+		// We are done
+		if (!sniffing) {
+			if (csv_file_scan) {
+				csv_file_scan->bytes_read += bytes_read;
+				bytes_read = 0;
+			}
+		}
+		return;
+	}
+	// If we are not done we have two options.
+	// 1) If a boundary is set.
+	if (iterator.IsBoundarySet()) {
+		iterator.done = true;
+		// We read until the next line or until we have nothing else to read.
+		// Move to next buffer
+		if (!cur_buffer_handle) {
+			return;
+		}
+		bool moved = MoveToNextBuffer();
+		if (cur_buffer_handle) {
+			if (moved && result.cur_col_id < result.number_of_columns && result.cur_col_id > 0) {
+				ProcessExtraRow();
+			} else if (!moved) {
+				ProcessExtraRow();
+			}
+			if (cur_buffer_handle->is_last_buffer && iterator.pos.buffer_pos >= cur_buffer_handle->actual_size) {
+				MoveToNextBuffer();
+			}
+		}
+	} else {
+		// 2) If a boundary is not set
+		// We read until the chunk is complete, or we have nothing else to read.
+		while (!FinishedFile() && result.number_of_rows < result.result_size) {
+			MoveToNextBuffer();
+			if (result.number_of_rows >= result.result_size) {
+				return;
+			}
+			if (cur_buffer_handle) {
+				Process(result);
+			}
+		}
+		iterator.done = FinishedFile();
+		if (result.null_padding && result.number_of_rows < STANDARD_VECTOR_SIZE) {
+			while (result.chunk_col_id < result.parse_chunk.ColumnCount()) {
+				result.validity_mask[result.chunk_col_id++]->SetInvalid(result.number_of_rows);
+				result.cur_col_id++;
+			}
+			result.number_of_rows++;
+		}
+	}
+}
+} // namespace duckdb