duckdb 0.7.2-dev225.0 → 0.7.2-dev2320.0

package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp

@@ -42,7 +42,10 @@ BaseCSVReader::~BaseCSVReader() {
 unique_ptr<CSVFileHandle> BaseCSVReader::OpenCSV(const BufferedCSVReaderOptions &options_p) {
 	auto file_handle = fs.OpenFile(options_p.file_path.c_str(), FileFlags::FILE_FLAGS_READ, FileLockType::NO_LOCK,
 	                               options_p.compression, this->opener);
-	return make_unique<CSVFileHandle>(std::move(file_handle));
+	if (file_handle->CanSeek()) {
+		file_handle->Reset();
+	}
+	return make_uniq<CSVFileHandle>(std::move(file_handle));
 }
 
 void BaseCSVReader::InitParseChunk(idx_t num_cols) {
@@ -61,9 +64,10 @@ void BaseCSVReader::InitParseChunk(idx_t num_cols) {
 	}
 }
 
-void BaseCSVReader::InitInsertChunkIdx(idx_t num_cols) {
-	for (idx_t col = 0; col < num_cols; ++col) {
-		insert_cols_idx.push_back(col);
+void BaseCSVReader::InitializeProjection() {
+	for (idx_t i = 0; i < GetTypes().size(); i++) {
+		reader_data.column_ids.push_back(i);
+		reader_data.column_mapping.push_back(i);
 	}
 }
 
@@ -121,6 +125,9 @@ bool TryCastFloatingValueCommaSeparated(const string_t &value_str, const Logical
 }
 
 bool BaseCSVReader::TryCastValue(const Value &value, const LogicalType &sql_type) {
+	if (value.IsNull()) {
+		return true;
+	}
 	if (options.has_format[LogicalTypeId::DATE] && sql_type.id() == LogicalTypeId::DATE) {
 		date_t result;
 		string error_message;
@@ -317,7 +324,11 @@ bool BaseCSVReader::AddRow(DataChunk &insert_chunk, idx_t &column, string &error
 	}
 
 	if (column < return_types.size() && mode != ParserMode::SNIFFING_DIALECT) {
-		if (options.ignore_errors) {
+		if (options.null_padding) {
+			for (; column < return_types.size(); column++) {
+				FlatVector::SetNull(parse_chunk.data[column], parse_chunk.size(), true);
+			}
+		} else if (options.ignore_errors) {
 			column = 0;
 			return false;
 		} else {
@@ -393,42 +404,36 @@ void BaseCSVReader::VerifyUTF8(idx_t col_idx) {
 }
 
 bool TryCastDecimalVectorCommaSeparated(BufferedCSVReaderOptions &options, Vector &input_vector, Vector &result_vector,
-                                        idx_t count, string &error_message, LogicalType &result_type) {
+                                        idx_t count, string &error_message, const LogicalType &result_type) {
 	auto width = DecimalType::GetWidth(result_type);
 	auto scale = DecimalType::GetScale(result_type);
 	switch (result_type.InternalType()) {
 	case PhysicalType::INT16:
 		return TemplatedTryCastDecimalVector<TryCastToDecimalCommaSeparated, int16_t>(
 		    options, input_vector, result_vector, count, error_message, width, scale);
-		break;
 	case PhysicalType::INT32:
 		return TemplatedTryCastDecimalVector<TryCastToDecimalCommaSeparated, int32_t>(
 		    options, input_vector, result_vector, count, error_message, width, scale);
-		break;
 	case PhysicalType::INT64:
 		return TemplatedTryCastDecimalVector<TryCastToDecimalCommaSeparated, int64_t>(
 		    options, input_vector, result_vector, count, error_message, width, scale);
-		break;
 	case PhysicalType::INT128:
 		return TemplatedTryCastDecimalVector<TryCastToDecimalCommaSeparated, hugeint_t>(
 		    options, input_vector, result_vector, count, error_message, width, scale);
-		break;
 	default:
 		throw InternalException("Unimplemented physical type for decimal");
 	}
 }
 
 bool TryCastFloatingVectorCommaSeparated(BufferedCSVReaderOptions &options, Vector &input_vector, Vector &result_vector,
-                                         idx_t count, string &error_message, LogicalType &result_type) {
+                                         idx_t count, string &error_message, const LogicalType &result_type) {
 	switch (result_type.InternalType()) {
 	case PhysicalType::DOUBLE:
 		return TemplatedTryCastFloatingVector<TryCastErrorMessageCommaSeparated, double>(
 		    options, input_vector, result_vector, count, error_message);
-		break;
 	case PhysicalType::FLOAT:
 		return TemplatedTryCastFloatingVector<TryCastErrorMessageCommaSeparated, float>(
 		    options, input_vector, result_vector, count, error_message);
-		break;
 	default:
 		throw InternalException("Unimplemented physical type for floating");
 	}
@@ -443,41 +448,44 @@ bool BaseCSVReader::Flush(DataChunk &insert_chunk, bool try_add_line) {
 
 	// convert the columns in the parsed chunk to the types of the table
 	insert_chunk.SetCardinality(parse_chunk);
-	for (idx_t col_idx = 0; col_idx < return_types.size(); col_idx++) {
-		auto insert_idx = insert_cols_idx[col_idx];
-		auto &type = return_types[col_idx];
+	if (reader_data.column_ids.empty() && !reader_data.empty_columns) {
+		throw InternalException("BaseCSVReader::Flush called on a CSV reader that was not correctly initialized. Call "
+		                        "MultiFileReader::InitializeReader or InitializeProjection");
+	}
+	D_ASSERT(reader_data.column_ids.size() == reader_data.column_mapping.size());
+	for (idx_t c = 0; c < reader_data.column_ids.size(); c++) {
+		auto col_idx = reader_data.column_ids[c];
+		auto result_idx = reader_data.column_mapping[c];
+		auto &parse_vector = parse_chunk.data[col_idx];
+		auto &result_vector = insert_chunk.data[result_idx];
+		auto &type = result_vector.GetType();
 		if (type.id() == LogicalTypeId::VARCHAR) {
 			// target type is varchar: no need to convert
 			// just test that all strings are valid utf-8 strings
 			VerifyUTF8(col_idx);
 			// reinterpret rather than reference so we can deal with user-defined types
-			insert_chunk.data[insert_idx].Reinterpret(parse_chunk.data[col_idx]);
-
+			result_vector.Reinterpret(parse_vector);
 		} else {
 			string error_message;
 			bool success;
 			if (options.has_format[LogicalTypeId::DATE] && type.id() == LogicalTypeId::DATE) {
 				// use the date format to cast the chunk
-				success = TryCastDateVector(options, parse_chunk.data[col_idx], insert_chunk.data[insert_idx],
-				                            parse_chunk.size(), error_message);
-			} else if (options.has_format[LogicalTypeId::TIMESTAMP] &&
-			           return_types[col_idx].id() == LogicalTypeId::TIMESTAMP) {
+				success = TryCastDateVector(options, parse_vector, result_vector, parse_chunk.size(), error_message);
+			} else if (options.has_format[LogicalTypeId::TIMESTAMP] && type.id() == LogicalTypeId::TIMESTAMP) {
 				// use the date format to cast the chunk
-				success = TryCastTimestampVector(options, parse_chunk.data[col_idx], insert_chunk.data[insert_idx],
-				                                 parse_chunk.size(), error_message);
-			} else if (options.decimal_separator != "." && (return_types[col_idx].id() == LogicalTypeId::FLOAT ||
-			                                                return_types[col_idx].id() == LogicalTypeId::DOUBLE)) {
-				success = TryCastFloatingVectorCommaSeparated(options, parse_chunk.data[col_idx],
-				                                              insert_chunk.data[insert_idx], parse_chunk.size(),
-				                                              error_message, return_types[col_idx]);
-			} else if (options.decimal_separator != "." && return_types[col_idx].id() == LogicalTypeId::DECIMAL) {
-				success = TryCastDecimalVectorCommaSeparated(options, parse_chunk.data[col_idx],
-				                                             insert_chunk.data[insert_idx], parse_chunk.size(),
-				                                             error_message, return_types[col_idx]);
+				success =
+				    TryCastTimestampVector(options, parse_vector, result_vector, parse_chunk.size(), error_message);
+			} else if (options.decimal_separator != "." &&
+			           (type.id() == LogicalTypeId::FLOAT || type.id() == LogicalTypeId::DOUBLE)) {
+				success = TryCastFloatingVectorCommaSeparated(options, parse_vector, result_vector, parse_chunk.size(),
+				                                              error_message, type);
+			} else if (options.decimal_separator != "." && type.id() == LogicalTypeId::DECIMAL) {
+				success = TryCastDecimalVectorCommaSeparated(options, parse_vector, result_vector, parse_chunk.size(),
+				                                             error_message, type);
 			} else {
 				// target type is not varchar: perform a cast
-				success = VectorOperations::TryCast(context, parse_chunk.data[col_idx], insert_chunk.data[insert_idx],
-				                                    parse_chunk.size(), &error_message);
+				success =
+				    VectorOperations::TryCast(context, parse_vector, result_vector, parse_chunk.size(), &error_message);
 			}
 			if (success) {
 				continue;
@@ -495,12 +503,11 @@ bool BaseCSVReader::Flush(DataChunk &insert_chunk, bool try_add_line) {
 			}
 
 			// figure out the exact line number
+			UnifiedVectorFormat inserted_column_data;
+			result_vector.ToUnifiedFormat(parse_chunk.size(), inserted_column_data);
 			idx_t row_idx;
 			for (row_idx = 0; row_idx < parse_chunk.size(); row_idx++) {
-				auto &inserted_column = insert_chunk.data[col_idx];
-				auto &parsed_column = parse_chunk.data[col_idx];
-
-				if (FlatVector::IsNull(inserted_column, row_idx) && !FlatVector::IsNull(parsed_column, row_idx)) {
+				if (!inserted_column_data.validity.RowIsValid(row_idx) && !FlatVector::IsNull(parse_vector, row_idx)) {
 					break;
 				}
 			}
@@ -520,19 +527,20 @@ bool BaseCSVReader::Flush(DataChunk &insert_chunk, bool try_add_line) {
 	}
 	if (conversion_error_ignored) {
 		D_ASSERT(options.ignore_errors);
-		SelectionVector succesful_rows;
-		succesful_rows.Initialize(parse_chunk.size());
+		SelectionVector succesful_rows(parse_chunk.size());
 		idx_t sel_size = 0;
 
 		for (idx_t row_idx = 0; row_idx < parse_chunk.size(); row_idx++) {
 			bool failed = false;
-			for (idx_t column_idx = 0; column_idx < return_types.size(); column_idx++) {
+			for (idx_t c = 0; c < reader_data.column_ids.size(); c++) {
+				auto col_idx = reader_data.column_ids[c];
+				auto result_idx = reader_data.column_mapping[c];
 
-				auto &inserted_column = insert_chunk.data[column_idx];
-				auto &parsed_column = parse_chunk.data[column_idx];
+				auto &parse_vector = parse_chunk.data[col_idx];
+				auto &result_vector = insert_chunk.data[result_idx];
 
-				bool was_already_null = FlatVector::IsNull(parsed_column, row_idx);
-				if (!was_already_null && FlatVector::IsNull(inserted_column, row_idx)) {
+				bool was_already_null = FlatVector::IsNull(parse_vector, row_idx);
+				if (!was_already_null && FlatVector::IsNull(result_vector, row_idx)) {
 					failed = true;
 					break;
 				}

package/src/duckdb/src/execution/operator/persistent/buffered_csv_reader.cpp

@@ -15,6 +15,7 @@
 #include "utf8proc.hpp"
 #include "duckdb/parser/keyword_helper.hpp"
 #include "duckdb/main/error_manager.hpp"
+#include "duckdb/main/client_data.hpp"
 
 #include <algorithm>
 #include <cctype>
@@ -250,7 +251,6 @@ void BufferedCSVReader::Initialize(const vector<LogicalType> &requested_types) {
 		SkipRowsAndReadHeader(options.skip_rows, options.header);
 	}
 	InitParseChunk(return_types.size());
-	InitInsertChunkIdx(return_types.size());
 	// we only need reset support during the automatic CSV type detection
 	// since reset support might require caching (in the case of streams), we disable it for the remainder
 	file_handle->DisableReset();
@@ -419,6 +419,7 @@ void BufferedCSVReader::DetectDialect(const vector<LogicalType> &requested_types
 	}
 
 	idx_t best_consistent_rows = 0;
+	idx_t prev_padding_count = 0;
 	for (auto quoterule : quoterule_candidates) {
 		const auto &quote_candidates = quote_candidates_map[static_cast<uint8_t>(quoterule)];
 		for (const auto &quote : quote_candidates) {
@@ -441,20 +442,29 @@ void BufferedCSVReader::DetectDialect(const vector<LogicalType> &requested_types
 
 					idx_t start_row = original_options.skip_rows;
 					idx_t consistent_rows = 0;
-					idx_t num_cols = 0;
-
+					idx_t num_cols = sniffed_column_counts.empty() ? 0 : sniffed_column_counts[0];
+					idx_t padding_count = 0;
+					bool allow_padding = original_options.null_padding;
 					for (idx_t row = 0; row < sniffed_column_counts.size(); row++) {
 						if (sniffed_column_counts[row] == num_cols) {
 							consistent_rows++;
-						} else {
+						} else if (num_cols < sniffed_column_counts[row] && !original_options.skip_rows_set) {
+							// we use the maximum amount of num_cols that we find
 							num_cols = sniffed_column_counts[row];
 							start_row = row + original_options.skip_rows;
 							consistent_rows = 1;
+							padding_count = 0;
+						} else if (num_cols >= sniffed_column_counts[row] && allow_padding) {
+							// we are missing some columns, we can parse this as long as we add padding
+							padding_count++;
 						}
 					}
 
 					// some logic
+					consistent_rows += padding_count;
 					bool more_values = (consistent_rows > best_consistent_rows && num_cols >= best_num_cols);
+					bool require_more_padding = padding_count > prev_padding_count;
+					bool require_less_padding = padding_count < prev_padding_count;
 					bool single_column_before = best_num_cols < 2 && num_cols > best_num_cols;
 					bool rows_consistent =
 					    start_row + consistent_rows - original_options.skip_rows == sniffed_column_counts.size();
@@ -464,16 +474,19 @@ void BufferedCSVReader::DetectDialect(const vector<LogicalType> &requested_types
 
 					if (!requested_types.empty() && requested_types.size() != num_cols) {
 						continue;
-					} else if ((more_values || single_column_before) && rows_consistent) {
+					} else if (rows_consistent && (single_column_before || (more_values && !require_more_padding) ||
+					                               (more_than_one_column && require_less_padding))) {
 						sniff_info.skip_rows = start_row;
 						sniff_info.num_cols = num_cols;
 						sniff_info.new_line = options.new_line;
 						best_consistent_rows = consistent_rows;
 						best_num_cols = num_cols;
+						prev_padding_count = padding_count;
 
 						info_candidates.clear();
 						info_candidates.push_back(sniff_info);
-					} else if (more_than_one_row && more_than_one_column && start_good && rows_consistent) {
+					} else if (more_than_one_row && more_than_one_column && start_good && rows_consistent &&
+					           !require_more_padding) {
 						bool same_quote_is_candidate = false;
 						for (auto &info_candidate : info_candidates) {
 							if (quote.compare(info_candidate.quote) == 0) {
@@ -797,7 +810,7 @@ vector<LogicalType> BufferedCSVReader::RefineTypeDetection(const vector<LogicalT
 				if ((sample_chunk_idx)*options.sample_chunk_size <= options.buffer_size) {
 					// cache parse chunk
 					// create a new chunk and fill it with the remainder
-					auto chunk = make_unique<DataChunk>();
+					auto chunk = make_uniq<DataChunk>();
 					auto parse_chunk_types = parse_chunk.GetTypes();
 					chunk->Move(parse_chunk);
 					cached_chunks.push(std::move(chunk));
@@ -870,16 +883,7 @@ vector<LogicalType> BufferedCSVReader::SniffCSV(const vector<LogicalType> &reque
 	// #######
 	// ### type detection (initial)
 	// #######
-	// type candidates, ordered by descending specificity (~ from high to low)
-	vector<LogicalType> type_candidates = {
-	    LogicalType::VARCHAR,
-	    LogicalType::TIMESTAMP,
-	    LogicalType::DATE,
-	    LogicalType::TIME,
-	    LogicalType::DOUBLE,
-	    /* LogicalType::FLOAT,*/ LogicalType::BIGINT,
-	    /*LogicalType::INTEGER,*/ /*LogicalType::SMALLINT, LogicalType::TINYINT,*/ LogicalType::BOOLEAN,
-	    LogicalType::SQLNULL};
+
 	// format template candidates, ordered by descending specificity (~ from high to low)
 	std::map<LogicalTypeId, vector<const char *>> format_template_candidates = {
 	    {LogicalTypeId::DATE, {"%m-%d-%Y", "%m-%d-%y", "%d-%m-%Y", "%d-%m-%y", "%Y-%m-%d", "%y-%m-%d"}},
@@ -890,8 +894,8 @@ vector<LogicalType> BufferedCSVReader::SniffCSV(const vector<LogicalType> &reque
 	vector<vector<LogicalType>> best_sql_types_candidates;
 	map<LogicalTypeId, vector<string>> best_format_candidates;
 	DataChunk best_header_row;
-	DetectCandidateTypes(type_candidates, format_template_candidates, info_candidates, original_options, best_num_cols,
-	                     best_sql_types_candidates, best_format_candidates, best_header_row);
+	DetectCandidateTypes(options.auto_type_candidates, format_template_candidates, info_candidates, original_options,
+	                     best_num_cols, best_sql_types_candidates, best_format_candidates, best_header_row);
 
 	if (best_format_candidates.empty() || best_header_row.size() == 0) {
 		throw InvalidInputException(
@@ -918,7 +922,7 @@ vector<LogicalType> BufferedCSVReader::SniffCSV(const vector<LogicalType> &reque
 					continue;
 				}
 			}
-			if (!options.union_by_name && found < options.sql_types_per_column.size()) {
+			if (!options.file_options.union_by_name && found < options.sql_types_per_column.size()) {
 				string exception = ColumnTypesError(options.sql_types_per_column, names);
 				if (!exception.empty()) {
 					throw BinderException(exception);
@@ -939,7 +943,8 @@ vector<LogicalType> BufferedCSVReader::SniffCSV(const vector<LogicalType> &reque
 	// #######
 	// ### type detection (refining)
 	// #######
-	return RefineTypeDetection(type_candidates, requested_types, best_sql_types_candidates, best_format_candidates);
+	return RefineTypeDetection(options.auto_type_candidates, requested_types, best_sql_types_candidates,
+	                           best_format_candidates);
 }
 
 bool BufferedCSVReader::TryParseComplexCSV(DataChunk &insert_chunk, string &error_message) {
@@ -950,10 +955,10 @@ bool BufferedCSVReader::TryParseComplexCSV(DataChunk &insert_chunk, string &erro
 	bool has_quotes = false;
 	uint8_t delimiter_pos = 0, escape_pos = 0, quote_pos = 0;
 	idx_t offset = 0;
-
+	idx_t line_start = 0;
 	// read values into the buffer (if any)
 	if (position >= buffer_size) {
-		if (!ReadBuffer(start)) {
+		if (!ReadBuffer(start, line_start)) {
 			return true;
 		}
 	}
@@ -990,7 +995,7 @@ value_start:
 				goto in_quotes;
 			}
 		}
-	} while (ReadBuffer(start));
+	} while (ReadBuffer(start, line_start));
 	// file ends while scanning for quote/delimiter, go to final state
 	goto final_state;
 normal:
@@ -1007,7 +1012,7 @@ normal:
 				goto add_row;
 			}
 		}
-	} while (ReadBuffer(start));
+	} while (ReadBuffer(start, line_start));
 	goto final_state;
 add_value:
 	AddValue(string_t(buffer.get() + start, position - start - offset), column, escape_positions, has_quotes);
@@ -1015,7 +1020,7 @@ add_value:
 	offset = 0;
 	has_quotes = false;
 	start = ++position;
-	if (position >= buffer_size && !ReadBuffer(start)) {
+	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
 		// file ends right after delimiter, go to final state
 		goto final_state;
 	}
@@ -1025,14 +1030,17 @@ add_row : {
 	bool carriage_return = buffer[position] == '\r';
 	AddValue(string_t(buffer.get() + start, position - start - offset), column, escape_positions, has_quotes);
 	finished_chunk = AddRow(insert_chunk, column, error_message);
+
 	if (!error_message.empty()) {
 		return false;
 	}
 	// increase position by 1 and move start to the new position
 	offset = 0;
 	has_quotes = false;
-	start = ++position;
-	if (position >= buffer_size && !ReadBuffer(start)) {
+	position++;
+	SkipEmptyLines();
+	start = position;
+	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
 		// file ends right after newline, go to final state
 		goto final_state;
 	}
@@ -1065,7 +1073,7 @@ in_quotes:
 				goto handle_escape;
 			}
 		}
-	} while (ReadBuffer(start));
+	} while (ReadBuffer(start, line_start));
 	// still in quoted state at the end of the file, error:
 	error_message = StringUtil::Format("Error in file \"%s\" on line %s: unterminated quotes. (%s)", options.file_path,
 	                                   GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
@@ -1078,7 +1086,7 @@ unquote:
 	delimiter_pos = 0;
 	quote_pos = 0;
 	position++;
-	if (position >= buffer_size && !ReadBuffer(start)) {
+	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
 		// file ends right after unquote, go to final state
 		offset = options.quote.size();
 		goto final_state;
@@ -1112,7 +1120,7 @@ unquote:
 				goto in_quotes;
 			}
 		}
-	} while (ReadBuffer(start));
+	} while (ReadBuffer(start, line_start));
 	error_message = StringUtil::Format(
 	    "Error in file \"%s\" on line %s: quote should be followed by end of value, end of row or another quote. (%s)",
 	    options.file_path, GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
@@ -1138,7 +1146,7 @@ handle_escape:
 				goto in_quotes;
 			}
 		}
-	} while (ReadBuffer(start));
+	} while (ReadBuffer(start, line_start));
 	error_message =
 	    StringUtil::Format("Error in file \"%s\" on line %s: neither QUOTE nor ESCAPE is proceeded by ESCAPE. (%s)",
 	                       options.file_path, GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
@@ -1149,7 +1157,7 @@ carriage_return:
 	if (buffer[position] == '\n') {
 		// newline after carriage return: skip
 		start = ++position;
-		if (position >= buffer_size && !ReadBuffer(start)) {
+		if (position >= buffer_size && !ReadBuffer(start, line_start)) {
 			// file ends right after newline, go to final state
 			goto final_state;
 		}
@@ -1166,6 +1174,7 @@ final_state:
 		// remaining values to be added to the chunk
 		AddValue(string_t(buffer.get() + start, position - start - offset), column, escape_positions, has_quotes);
 		finished_chunk = AddRow(insert_chunk, column, error_message);
+		SkipEmptyLines();
 		if (!error_message.empty()) {
 			return false;
 		}
@@ -1180,6 +1189,18 @@ final_state:
 	return true;
 }
 
+void BufferedCSVReader::SkipEmptyLines() {
+	if (parse_chunk.data.size() == 1) {
+		// Empty lines are null data.
+		return;
+	}
+	for (; position < buffer_size; position++) {
+		if (!StringUtil::CharacterIsNewline(buffer[position])) {
+			return;
+		}
+	}
+}
+
 bool BufferedCSVReader::TryParseSimpleCSV(DataChunk &insert_chunk, string &error_message) {
 	// used for parsing algorithm
 	bool finished_chunk = false;
@@ -1188,12 +1209,14 @@ bool BufferedCSVReader::TryParseSimpleCSV(DataChunk &insert_chunk, string &error
 	bool has_quotes = false;
 	vector<idx_t> escape_positions;
 
+	idx_t line_start = position;
 	// read values into the buffer (if any)
 	if (position >= buffer_size) {
-		if (!ReadBuffer(start)) {
+		if (!ReadBuffer(start, line_start)) {
 			return true;
 		}
 	}
+
 	// start parsing the first value
 	goto value_start;
 value_start:
@@ -1223,7 +1246,7 @@ normal:
 				goto add_row;
 			}
 		}
-	} while (ReadBuffer(start));
+	} while (ReadBuffer(start, line_start));
 	// file ends during normal scan: go to end state
 	goto final_state;
 add_value:
@@ -1232,7 +1255,7 @@ add_value:
 	offset = 0;
 	has_quotes = false;
 	start = ++position;
-	if (position >= buffer_size && !ReadBuffer(start)) {
+	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
 		// file ends right after delimiter, go to final state
 		goto final_state;
 	}
@@ -1245,14 +1268,19 @@ add_row : {
 		return false;
 	}
 	finished_chunk = AddRow(insert_chunk, column, error_message);
+	if (context.client_data->max_line_length < position - line_start) {
+		context.client_data->max_line_length = position - line_start;
+	}
 	if (!error_message.empty()) {
 		return false;
 	}
 	// increase position by 1 and move start to the new position
 	offset = 0;
 	has_quotes = false;
-	start = ++position;
-	if (position >= buffer_size && !ReadBuffer(start)) {
+	position++;
+	start = position;
+	line_start = position;
+	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
 		// file ends right after delimiter, go to final state
 		goto final_state;
 	}
@@ -1261,6 +1289,14 @@ add_row : {
 		goto carriage_return;
 	} else {
 		SetNewLineDelimiter();
+		SkipEmptyLines();
+
+		start = position;
+		line_start = position;
+		if (position >= buffer_size && !ReadBuffer(start, line_start)) {
+			// file ends right after delimiter, go to final state
+			goto final_state;
+		}
 		// \n newline, move to value start
 		if (finished_chunk) {
 			return true;
@@ -1284,7 +1320,7 @@ in_quotes:
 				goto handle_escape;
 			}
 		}
-	} while (ReadBuffer(start));
+	} while (ReadBuffer(start, line_start));
 	// still in quoted state at the end of the file, error:
 	throw InvalidInputException("Error in file \"%s\" on line %s: unterminated quotes. (%s)", options.file_path,
 	                            GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
@@ -1294,7 +1330,7 @@ unquote:
 	// in this state we expect either another quote (entering the quoted state again, and escaping the quote)
 	// or a delimiter/newline, ending the current value and moving on to the next value
 	position++;
-	if (position >= buffer_size && !ReadBuffer(start)) {
+	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
 		// file ends right after unquote, go to final state
 		offset = 1;
 		goto final_state;
@@ -1321,7 +1357,7 @@ handle_escape:
 	/* state: handle_escape */
 	// escape should be followed by a quote or another escape character
 	position++;
-	if (position >= buffer_size && !ReadBuffer(start)) {
+	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
 		error_message = StringUtil::Format(
 		    "Error in file \"%s\" on line %s: neither QUOTE nor ESCAPE is proceeded by ESCAPE. (%s)", options.file_path,
 		    GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
@@ -1343,7 +1379,7 @@ carriage_return:
 		// newline after carriage return: skip
 		// increase position by 1 and move start to the new position
 		start = ++position;
-		if (position >= buffer_size && !ReadBuffer(start)) {
+		if (position >= buffer_size && !ReadBuffer(start, line_start)) {
 			// file ends right after delimiter, go to final state
 			goto final_state;
 		}
@@ -1353,6 +1389,14 @@ carriage_return:
 	if (finished_chunk) {
 		return true;
 	}
+	SkipEmptyLines();
+	start = position;
+	line_start = position;
+	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
+		// file ends right after delimiter, go to final state
+		goto final_state;
+	}
+
 	goto value_start;
 final_state:
 	if (finished_chunk) {
@@ -1363,6 +1407,10 @@ final_state:
 		// remaining values to be added to the chunk
 		AddValue(string_t(buffer.get() + start, position - start - offset), column, escape_positions, has_quotes);
 		finished_chunk = AddRow(insert_chunk, column, error_message);
+		SkipEmptyLines();
+		if (context.client_data->max_line_length < position - line_start) {
+			context.client_data->max_line_length = position - line_start;
+		}
 		if (!error_message.empty()) {
 			return false;
 		}
@@ -1378,7 +1426,10 @@ final_state:
 	return true;
 }
 
-bool BufferedCSVReader::ReadBuffer(idx_t &start) {
+bool BufferedCSVReader::ReadBuffer(idx_t &start, idx_t &line_start) {
+	if (start > buffer_size) {
+		return false;
+	}
 	auto old_buffer = std::move(buffer);
 
 	// the remaining part of the last buffer
@@ -1416,9 +1467,11 @@ bool BufferedCSVReader::ReadBuffer(idx_t &start) {
 	if (!bom_checked) {
 		bom_checked = true;
 		if (read_count >= 3 && buffer[0] == '\xEF' && buffer[1] == '\xBB' && buffer[2] == '\xBF') {
+			start += 3;
 			position += 3;
 		}
 	}
+	line_start = start;
 
 	return read_count > 0;
 }

package/src/duckdb/src/execution/operator/persistent/csv_buffer.cpp

@@ -4,8 +4,8 @@
 namespace duckdb {
 
 CSVBuffer::CSVBuffer(ClientContext &context, idx_t buffer_size_p, CSVFileHandle &file_handle,
-                     idx_t &global_csv_current_position)
-    : context(context), first_buffer(true) {
+                     idx_t &global_csv_current_position, idx_t file_number_p)
+    : context(context), first_buffer(true), file_number(file_number_p) {
 	this->handle = AllocateBuffer(buffer_size_p);
 
 	auto buffer = Ptr();
@@ -19,23 +19,23 @@ CSVBuffer::CSVBuffer(ClientContext &context, idx_t buffer_size_p, CSVFileHandle
 }
 
 CSVBuffer::CSVBuffer(ClientContext &context, BufferHandle buffer_p, idx_t buffer_size_p, idx_t actual_size_p,
-                     bool final_buffer, idx_t global_csv_current_position)
+                     bool final_buffer, idx_t global_csv_current_position, idx_t file_number_p)
     : context(context), handle(std::move(buffer_p)), actual_size(actual_size_p), last_buffer(final_buffer),
-      global_csv_start(global_csv_current_position) {
+      global_csv_start(global_csv_current_position), file_number(file_number_p) {
 }
 
-unique_ptr<CSVBuffer> CSVBuffer::Next(CSVFileHandle &file_handle, idx_t buffer_size,
-                                      idx_t &global_csv_current_position) {
-	if (file_handle.FinishedReading()) {
-		// this was the last buffer
-		return nullptr;
-	}
-
+unique_ptr<CSVBuffer> CSVBuffer::Next(CSVFileHandle &file_handle, idx_t buffer_size, idx_t &global_csv_current_position,
+                                      idx_t file_number_p) {
 	auto next_buffer = AllocateBuffer(buffer_size);
 	idx_t next_buffer_actual_size = file_handle.Read(next_buffer.Ptr(), buffer_size);
+	if (next_buffer_actual_size == 0) {
+		// We are done reading
+		return nullptr;
+	}
 
-	auto next_csv_buffer = make_unique<CSVBuffer>(context, std::move(next_buffer), buffer_size, next_buffer_actual_size,
-	                                              file_handle.FinishedReading(), global_csv_current_position);
+	auto next_csv_buffer =
+	    make_uniq<CSVBuffer>(context, std::move(next_buffer), buffer_size, next_buffer_actual_size,
+	                         file_handle.FinishedReading(), global_csv_current_position, file_number_p);
 	global_csv_current_position += next_buffer_actual_size;
 	return next_csv_buffer;
 }
@@ -65,4 +65,8 @@ idx_t CSVBuffer::GetCSVGlobalStart() {
 	return global_csv_start;
 }
 
+idx_t CSVBuffer::GetFileNumber() {
+	return file_number;
+}
+
 } // namespace duckdb