duckdb 1.1.2-dev6.0 → 1.1.4-dev11.0

package/src/duckdb/src/execution/operator/csv_scanner/sniffer/csv_sniffer.cpp

@@ -6,7 +6,7 @@ namespace duckdb {
 CSVSniffer::CSVSniffer(CSVReaderOptions &options_p, shared_ptr<CSVBufferManager> buffer_manager_p,
                        CSVStateMachineCache &state_machine_cache_p, bool default_null_to_varchar_p)
     : state_machine_cache(state_machine_cache_p), options(options_p), buffer_manager(std::move(buffer_manager_p)),
-      default_null_to_varchar(default_null_to_varchar_p) {
+      lines_sniffed(0), default_null_to_varchar(default_null_to_varchar_p) {
 	// Initialize Format Candidates
 	for (const auto &format_template : format_template_candidates) {
 		auto &logical_type = format_template.first;
@@ -48,8 +48,9 @@ void MatchAndReplace(CSVOption<T> &original, CSVOption<T> &sniffed, const string
 		original.Set(sniffed.GetValue(), false);
 	}
 }
-void MatchAndRepaceUserSetVariables(DialectOptions &original, DialectOptions &sniffed, string &error, bool found_date,
-                                    bool found_timestamp) {
+
+void MatchAndReplaceUserSetVariables(DialectOptions &original, DialectOptions &sniffed, string &error, bool found_date,
+                                     bool found_timestamp) {
 	MatchAndReplace(original.header, sniffed.header, "Header", error);
 	if (sniffed.state_machine_options.new_line.GetValue() != NewLineIdentifier::NOT_SET) {
 		// Is sniffed line is not set (e.g., single-line file) , we don't try to replace and match.
@@ -72,7 +73,7 @@ void MatchAndRepaceUserSetVariables(DialectOptions &original, DialectOptions &sn
 	}
 }
 // Set the CSV Options in the reference
-void CSVSniffer::SetResultOptions() {
+void CSVSniffer::SetResultOptions() const {
 	bool found_date = false;
 	bool found_timestamp = false;
 	for (auto &type : detected_types) {
@@ -82,8 +83,8 @@ void CSVSniffer::SetResultOptions() {
 			found_timestamp = true;
 		}
 	}
-	MatchAndRepaceUserSetVariables(options.dialect_options, best_candidate->GetStateMachine().dialect_options,
-	                               options.sniffer_user_mismatch_error, found_date, found_timestamp);
+	MatchAndReplaceUserSetVariables(options.dialect_options, best_candidate->GetStateMachine().dialect_options,
+	                                options.sniffer_user_mismatch_error, found_date, found_timestamp);
 	options.dialect_options.num_cols = best_candidate->GetStateMachine().dialect_options.num_cols;
 	options.dialect_options.rows_until_header = best_candidate->GetStateMachine().dialect_options.rows_until_header;
 }
@@ -96,9 +97,8 @@ AdaptiveSnifferResult CSVSniffer::MinimalSniff() {
 	// Return Types detected
 	vector<LogicalType> return_types;
 	// Column Names detected
-
 	buffer_manager->sniffing = true;
-	constexpr idx_t result_size = 2;
+	constexpr idx_t result_size = STANDARD_VECTOR_SIZE;
 
 	auto state_machine =
 	    make_shared_ptr<CSVStateMachine>(options, options.dialect_options.state_machine_options, state_machine_cache);
@@ -114,6 +114,7 @@ AdaptiveSnifferResult CSVSniffer::MinimalSniff() {
 
 	// First figure out the number of columns on this configuration
 	auto scanner = count_scanner.UpgradeToStringValueScanner();
+	scanner->error_handler->SetIgnoreErrors(true);
 	// Parse chunk and read csv with info candidate
 	auto &data_chunk = scanner->ParseChunk().ToChunk();
 	idx_t start_row = 0;
@@ -130,10 +131,9 @@ AdaptiveSnifferResult CSVSniffer::MinimalSniff() {
 
 	// Possibly Gather Header
 	vector<HeaderValue> potential_header;
-
 	for (idx_t col_idx = 0; col_idx < data_chunk.ColumnCount(); col_idx++) {
 		auto &cur_vector = data_chunk.data[col_idx];
-		auto vector_data = FlatVector::GetData<string_t>(cur_vector);
+		const auto vector_data = FlatVector::GetData<string_t>(cur_vector);
 		auto &validity = FlatVector::Validity(cur_vector);
 		HeaderValue val;
 		if (validity.RowIsValid(0)) {
@@ -142,8 +142,8 @@ AdaptiveSnifferResult CSVSniffer::MinimalSniff() {
 		potential_header.emplace_back(val);
 	}
 
-	vector<string> names = DetectHeaderInternal(buffer_manager->context, potential_header, *state_machine, set_columns,
-	                                            best_sql_types_candidates_per_column_idx, options, *error_handler);
+	auto names = DetectHeaderInternal(buffer_manager->context, potential_header, *state_machine, set_columns,
+	                                  best_sql_types_candidates_per_column_idx, options, *error_handler);
 
 	for (idx_t column_idx = 0; column_idx < best_sql_types_candidates_per_column_idx.size(); column_idx++) {
 		LogicalType d_type = best_sql_types_candidates_per_column_idx[column_idx].back();
@@ -152,7 +152,6 @@ AdaptiveSnifferResult CSVSniffer::MinimalSniff() {
 		}
 		detected_types.push_back(d_type);
 	}
-
 	return {detected_types, names, sniffed_column_counts.result_position > 1};
 }
 
@@ -181,10 +180,15 @@ SnifferResult CSVSniffer::AdaptiveSniff(const CSVSchema &file_schema) {
 	}
 	return min_sniff_res.ToSnifferResult();
 }
-SnifferResult CSVSniffer::SniffCSV(bool force_match) {
+
+SnifferResult CSVSniffer::SniffCSV(const bool force_match) {
 	buffer_manager->sniffing = true;
 	// 1. Dialect Detection
 	DetectDialect();
+	if (buffer_manager->file_handle->compression_type != FileCompressionType::UNCOMPRESSED &&
+	    buffer_manager->IsBlockUnloaded(0)) {
+		buffer_manager->ResetBufferManager();
+	}
 	// 2. Type Detection
 	DetectTypes();
 	// 3. Type Refinement
@@ -201,15 +205,8 @@ SnifferResult CSVSniffer::SniffCSV(bool force_match) {
 		buffer_manager->ResetBufferManager();
 	}
 	buffer_manager->sniffing = false;
-	if (!best_candidate->error_handler->errors.empty() && !options.ignore_errors.GetValue()) {
-		for (auto &error_vector : best_candidate->error_handler->errors) {
-			for (auto &error : error_vector.second) {
-				if (error.type == MAXIMUM_LINE_SIZE) {
-					// If it's a maximum line size error, we can do it now.
-					error_handler->Error(error);
-				}
-			}
-		}
+	if (best_candidate->error_handler->AnyErrors() && !options.ignore_errors.GetValue()) {
+		best_candidate->error_handler->ErrorIfTypeExists(MAXIMUM_LINE_SIZE);
 	}
 	D_ASSERT(best_sql_types_candidates_per_column_idx.size() == names.size());
 	// We are done, Set the CSV Options in the reference. Construct and return the result.

package/src/duckdb/src/execution/operator/csv_scanner/sniffer/dialect_detection.cpp

@@ -14,20 +14,20 @@ bool IsQuoteDefault(char quote) {
 	return false;
 }
 
-vector<char> DialectCandidates::GetDefaultDelimiter() {
-	return {',', '|', ';', '\t'};
+vector<string> DialectCandidates::GetDefaultDelimiter() {
+	return {",", "|", ";", "\t"};
 }
 
 vector<vector<char>> DialectCandidates::GetDefaultQuote() {
-	return {{'\"'}, {'\"', '\''}, {'\0'}};
+	return {{'\0'}, {'\"', '\''}, {'\"'}};
 }
 
 vector<QuoteRule> DialectCandidates::GetDefaultQuoteRule() {
-	return {QuoteRule::QUOTES_RFC, QuoteRule::QUOTES_OTHER, QuoteRule::NO_QUOTES};
+	return {QuoteRule::NO_QUOTES, QuoteRule::QUOTES_OTHER, QuoteRule::QUOTES_RFC};
 }
 
 vector<vector<char>> DialectCandidates::GetDefaultEscape() {
-	return {{'\"', '\0', '\''}, {'\\'}, {'\0'}};
+	return {{'\0'}, {'\\'}, {'\"', '\0', '\''}};
 }
 
 vector<char> DialectCandidates::GetDefaultComment() {
@@ -51,7 +51,19 @@ string DialectCandidates::Print() {
 		auto escape_candidate = escape_candidates_map[i];
 		for (idx_t j = 0; j < quote_candidate.size(); j++) {
 			for (idx_t k = 0; k < escape_candidate.size(); k++) {
-				search_space << "[\'" << quote_candidate[j] << "\',\'" << escape_candidate[k] << "\']";
+				search_space << "[\'";
+				if (quote_candidate[j] == '\0') {
+					search_space << "(no quote)";
+				} else {
+					search_space << quote_candidate[j];
+				}
+				search_space << "\',\'";
+				if (escape_candidate[k] == '\0') {
+					search_space << "(no escape)";
+				} else {
+					search_space << escape_candidate[k];
+				}
+				search_space << "\']";
 				if (k < escape_candidate.size() - 1) {
 					search_space << ",";
 				}
@@ -80,11 +92,11 @@ string DialectCandidates::Print() {
 
 DialectCandidates::DialectCandidates(const CSVStateMachineOptions &options) {
 	// assert that quotes escapes and rules have equal size
-	auto default_quote = GetDefaultQuote();
-	auto default_escape = GetDefaultEscape();
-	auto default_quote_rule = GetDefaultQuoteRule();
-	auto default_delimiter = GetDefaultDelimiter();
-	auto default_comment = GetDefaultComment();
+	const auto default_quote = GetDefaultQuote();
+	const auto default_escape = GetDefaultEscape();
+	const auto default_quote_rule = GetDefaultQuoteRule();
+	const auto default_delimiter = GetDefaultDelimiter();
+	const auto default_comment = GetDefaultComment();
 
 	D_ASSERT(default_quote.size() == default_quote_rule.size() && default_quote_rule.size() == default_escape.size());
 	// fill the escapes
@@ -144,6 +156,11 @@ void CSVSniffer::GenerateStateMachineSearchSpace(vector<unique_ptr<ColumnCountSc
 	} else {
 		new_line_id = DetectNewLineDelimiter(*buffer_manager);
 	}
+	// We only sniff RFC 4180 rules, unless manually set by user.
+	bool rfc_4180 = true;
+	if (options.dialect_options.state_machine_options.rfc_4180.IsSetByUser()) {
+		rfc_4180 = options.dialect_options.state_machine_options.rfc_4180.GetValue();
+	}
 	CSVIterator first_iterator;
 	bool iterator_set = false;
 	for (const auto quote_rule : dialect_candidates.quote_rule_candidates) {
@@ -155,7 +172,8 @@ void CSVSniffer::GenerateStateMachineSearchSpace(vector<unique_ptr<ColumnCountSc
 				for (const auto &escape : escape_candidates) {
 					for (const auto &comment : dialect_candidates.comment_candidates) {
 						D_ASSERT(buffer_manager);
-						CSVStateMachineOptions state_machine_options(delimiter, quote, escape, comment, new_line_id);
+						CSVStateMachineOptions state_machine_options(delimiter, quote, escape, comment, new_line_id,
+						                                             rfc_4180);
 						auto sniffing_state_machine =
 						    make_shared_ptr<CSVStateMachine>(options, state_machine_options, state_machine_cache);
 						if (options.dialect_options.skip_rows.IsSetByUser()) {
@@ -181,6 +199,9 @@ void CSVSniffer::GenerateStateMachineSearchSpace(vector<unique_ptr<ColumnCountSc
 
 // Returns true if a comment is acceptable
 bool AreCommentsAcceptable(const ColumnCountResult &result, idx_t num_cols, bool comment_set_by_user) {
+	if (comment_set_by_user) {
+		return true;
+	}
 	// For a comment to be acceptable, we want 3/5th's the majority of unmatched in the columns
 	constexpr double min_majority = 0.6;
 	// detected comments, are all lines that started with a comment character.
@@ -202,7 +223,7 @@ bool AreCommentsAcceptable(const ColumnCountResult &result, idx_t num_cols, bool
 		}
 	}
 	// If we do not encounter at least one full line comment, we do not consider this comment option.
-	if (valid_comments == 0 || (!has_full_line_comment && !comment_set_by_user)) {
+	if (valid_comments == 0 || !has_full_line_comment) {
 		// this is only valid if our comment character is \0
 		if (result.state_machine.state_machine_options.comment.GetValue() == '\0') {
 			return true;
@@ -228,14 +249,14 @@ void CSVSniffer::AnalyzeDialectCandidate(unique_ptr<ColumnCountScanner> scanner,
 	idx_t num_cols = sniffed_column_counts.result_position == 0 ? 1 : sniffed_column_counts[0].number_of_columns;
 	const bool ignore_errors = options.ignore_errors.GetValue();
 	// If we are ignoring errors and not null_padding , we pick the most frequent number of columns as the right one
-	bool use_most_frequent_columns = ignore_errors && !options.null_padding;
+	const bool use_most_frequent_columns = ignore_errors && !options.null_padding;
 	if (use_most_frequent_columns) {
 		num_cols = sniffed_column_counts.GetMostFrequentColumnCount();
 	}
 	idx_t padding_count = 0;
 	idx_t comment_rows = 0;
 	idx_t ignored_rows = 0;
-	bool allow_padding = options.null_padding;
+	const bool allow_padding = options.null_padding;
 	bool first_valid = false;
 	if (sniffed_column_counts.result_position > rows_read) {
 		rows_read = sniffed_column_counts.result_position;
@@ -300,63 +321,96 @@ void CSVSniffer::AnalyzeDialectCandidate(unique_ptr<ColumnCountScanner> scanner,
 	consistent_rows += padding_count;
 
 	// Whether there are more values (rows) available that are consistent, exceeding the current best.
-	bool more_values = consistent_rows > best_consistent_rows && num_cols >= max_columns_found;
+	const bool more_values = consistent_rows > best_consistent_rows && num_cols >= max_columns_found;
 
-	bool more_columns = consistent_rows == best_consistent_rows && num_cols > max_columns_found;
+	const bool more_columns = consistent_rows == best_consistent_rows && num_cols > max_columns_found;
 
 	// If additional padding is required when compared to the previous padding count.
-	bool require_more_padding = padding_count > prev_padding_count;
+	const bool require_more_padding = padding_count > prev_padding_count;
 
 	// If less padding is now required when compared to the previous padding count.
-	bool require_less_padding = padding_count < prev_padding_count;
+	const bool require_less_padding = padding_count < prev_padding_count;
 
 	// If there was only a single column before, and the new number of columns exceeds that.
-	bool single_column_before = max_columns_found < 2 && num_cols > max_columns_found * candidates.size();
+	const bool single_column_before = max_columns_found < 2 && num_cols > max_columns_found * candidates.size();
 
 	// If the number of rows is consistent with the calculated value after accounting for skipped rows and the
 	// start row.
-	bool rows_consistent =
+	const bool rows_consistent =
 	    consistent_rows + (dirty_notes_minus_comments - options.dialect_options.skip_rows.GetValue()) + comment_rows ==
 	    sniffed_column_counts.result_position - options.dialect_options.skip_rows.GetValue();
 	// If there are more than one consistent row.
-	bool more_than_one_row = consistent_rows > 1;
+	const bool more_than_one_row = consistent_rows > 1;
 
 	// If there are more than one column.
-	bool more_than_one_column = num_cols > 1;
+	const bool more_than_one_column = num_cols > 1;
 
 	// If the start position is valid.
-	bool start_good = !candidates.empty() &&
-	                  dirty_notes <= candidates.front()->GetStateMachine().dialect_options.skip_rows.GetValue();
+	const bool start_good = !candidates.empty() &&
+	                        dirty_notes <= candidates.front()->GetStateMachine().dialect_options.skip_rows.GetValue();
 
 	// If padding happened but it is not allowed.
-	bool invalid_padding = !allow_padding && padding_count > 0;
+	const bool invalid_padding = !allow_padding && padding_count > 0;
 
-	bool comments_are_acceptable = AreCommentsAcceptable(
+	const bool comments_are_acceptable = AreCommentsAcceptable(
 	    sniffed_column_counts, num_cols, options.dialect_options.state_machine_options.comment.IsSetByUser());
 
+	const bool quoted =
+	    scanner->ever_quoted &&
+	    sniffed_column_counts.state_machine.dialect_options.state_machine_options.quote.GetValue() != '\0';
+
+	// For our columns to match, we either don't have them manually set, or they match in value with the sniffed value
+	const bool columns_match_set =
+	    num_cols == set_columns.Size() ||
+	    (num_cols == set_columns.Size() + 1 && sniffed_column_counts[0].last_value_always_empty) ||
+	    !set_columns.IsSet();
+
 	// If rows are consistent and no invalid padding happens, this is the best suitable candidate if one of the
 	// following is valid:
 	// - There's a single column before.
 	// - There are more values and no additional padding is required.
 	// - There's more than one column and less padding is required.
-	if (rows_consistent &&
+	if (columns_match_set && (rows_consistent || (set_columns.IsSet() && ignore_errors)) &&
 	    (single_column_before || ((more_values || more_columns) && !require_more_padding) ||
-	     (more_than_one_column && require_less_padding)) &&
+	     (more_than_one_column && require_less_padding) || quoted) &&
 	    !invalid_padding && comments_are_acceptable) {
-		if (!candidates.empty() && set_columns.IsSet() && max_columns_found == set_columns.Size()) {
+		if (!candidates.empty() && set_columns.IsSet() && max_columns_found == set_columns.Size() &&
+		    consistent_rows <= best_consistent_rows) {
 			// We have a candidate that fits our requirements better
-			return;
+			if (candidates.front()->ever_quoted || !scanner->ever_quoted) {
+				return;
+			}
 		}
 		auto &sniffing_state_machine = scanner->GetStateMachine();
 
-		if (!candidates.empty() && candidates.front()->ever_quoted && !scanner->ever_quoted) {
+		if (!candidates.empty() && candidates.front()->ever_quoted) {
 			// Give preference to quoted boys.
-			return;
+			if (!scanner->ever_quoted) {
+				return;
+			} else {
+				// Give preference to one that got escaped
+				if (!scanner->ever_escaped && candidates.front()->ever_escaped) {
+					return;
+				}
+				if (best_consistent_rows == consistent_rows && num_cols >= max_columns_found) {
+					// If both have not been escaped, this might get solved later on.
+					sniffing_state_machine.dialect_options.num_cols = num_cols;
+					candidates.emplace_back(std::move(scanner));
+					max_columns_found = num_cols;
+					return;
+				}
+			}
 		}
 		if (max_columns_found == num_cols && ignored_rows > min_ignored_rows) {
 			return;
 		}
-
+		if (quoted && num_cols < max_columns_found) {
+			for (auto &candidate : candidates) {
+				if (candidate->ever_quoted) {
+					return;
+				}
+			}
+		}
 		best_consistent_rows = consistent_rows;
 		max_columns_found = num_cols;
 		prev_padding_count = padding_count;
@@ -375,18 +429,19 @@ void CSVSniffer::AnalyzeDialectCandidate(unique_ptr<ColumnCountScanner> scanner,
 
 		candidates.clear();
 		sniffing_state_machine.dialect_options.num_cols = num_cols;
+		lines_sniffed = sniffed_column_counts.result_position;
 		candidates.emplace_back(std::move(scanner));
 		return;
 	}
 	// If there's more than one row and column, the start is good, rows are consistent,
 	// no additional padding is required, and there is no invalid padding, and there is not yet a candidate
 	// with the same quote, we add this state_machine as a suitable candidate.
-	if (more_than_one_row && more_than_one_column && start_good && rows_consistent && !require_more_padding &&
-	    !invalid_padding && num_cols == max_columns_found && comments_are_acceptable) {
+	if (columns_match_set && more_than_one_row && more_than_one_column && start_good && rows_consistent &&
+	    !require_more_padding && !invalid_padding && num_cols == max_columns_found && comments_are_acceptable) {
 		auto &sniffing_state_machine = scanner->GetStateMachine();
 
 		bool same_quote_is_candidate = false;
-		for (auto &candidate : candidates) {
+		for (const auto &candidate : candidates) {
 			if (sniffing_state_machine.dialect_options.state_machine_options.quote ==
 			    candidate->GetStateMachine().dialect_options.state_machine_options.quote) {
 				same_quote_is_candidate = true;
@@ -403,8 +458,8 @@ void CSVSniffer::AnalyzeDialectCandidate(unique_ptr<ColumnCountScanner> scanner,
 			} else if (!options.null_padding) {
 				sniffing_state_machine.dialect_options.skip_rows = dirty_notes;
 			}
-
 			sniffing_state_machine.dialect_options.num_cols = num_cols;
+			lines_sniffed = sniffed_column_counts.result_position;
 			candidates.emplace_back(std::move(scanner));
 		}
 	}
@@ -437,24 +492,35 @@ void CSVSniffer::RefineCandidates() {
 		// Only one candidate nothing to refine or all candidates already checked
 		return;
 	}
-	vector<unique_ptr<ColumnCountScanner>> successful_candidates;
-	for (auto &cur_candidate : candidates) {
-		for (idx_t i = 1; i <= options.sample_size_chunks; i++) {
-			bool finished_file = cur_candidate->FinishedFile();
+
+	for (idx_t i = 1; i <= options.sample_size_chunks; i++) {
+		vector<unique_ptr<ColumnCountScanner>> successful_candidates;
+		bool done = false;
+		for (auto &cur_candidate : candidates) {
+			const bool finished_file = cur_candidate->FinishedFile();
+			if (successful_candidates.empty()) {
+				lines_sniffed += cur_candidate->GetResult().result_position;
+			}
 			if (finished_file || i == options.sample_size_chunks) {
 				// we finished the file or our chunk sample successfully
-				successful_candidates.push_back(std::move(cur_candidate));
-				break;
+				if (!cur_candidate->GetResult().error) {
+					successful_candidates.push_back(std::move(cur_candidate));
+				}
+				done = true;
+				continue;
 			}
-			if (!RefineCandidateNextChunk(*cur_candidate) || cur_candidate->GetResult().error) {
-				// This candidate failed, move to the next one
-				break;
+			if (RefineCandidateNextChunk(*cur_candidate) && !cur_candidate->GetResult().error) {
+				successful_candidates.push_back(std::move(cur_candidate));
 			}
 		}
+		candidates = std::move(successful_candidates);
+		if (done) {
+			break;
+		}
 	}
 	// If we have multiple candidates with quotes set, we will give the preference to ones
 	// that have actually quoted values, otherwise we will choose quotes = \0
-	candidates.clear();
+	vector<unique_ptr<ColumnCountScanner>> successful_candidates = std::move(candidates);
 	if (!successful_candidates.empty()) {
 		for (idx_t i = 0; i < successful_candidates.size(); i++) {
 			unique_ptr<ColumnCountScanner> cc_best_candidate = std::move(successful_candidates[i]);
@@ -525,7 +591,7 @@ void CSVSniffer::DetectDialect() {
 	// if no dialect candidate was found, we throw an exception
 	if (candidates.empty()) {
 		auto error = CSVError::SniffingError(options, dialect_candidates.Print());
-		error_handler->Error(error);
+		error_handler->Error(error, true);
 	}
 }
 } // namespace duckdb

package/src/duckdb/src/execution/operator/csv_scanner/sniffer/header_detection.cpp

@@ -17,8 +17,8 @@ static string GenerateColumnName(const idx_t total_cols, const idx_t col_number,
 // Helper function for UTF-8 aware space trimming
 static string TrimWhitespace(const string &col_name) {
 	utf8proc_int32_t codepoint;
-	auto str = reinterpret_cast<const utf8proc_uint8_t *>(col_name.c_str());
-	idx_t size = col_name.size();
+	const auto str = reinterpret_cast<const utf8proc_uint8_t *>(col_name.c_str());
+	const idx_t size = col_name.size();
 	// Find the first character that is not left trimmed
 	idx_t begin = 0;
 	while (begin < size) {
@@ -96,6 +96,44 @@ static string NormalizeColumnName(const string &col_name) {
 	return col_name_cleaned;
 }
 
+static void ReplaceNames(vector<string> &detected_names, CSVStateMachine &state_machine,
+                         unordered_map<idx_t, vector<LogicalType>> &best_sql_types_candidates_per_column_idx,
+                         CSVReaderOptions &options, const vector<HeaderValue> &best_header_row,
+                         CSVErrorHandler &error_handler) {
+	auto &dialect_options = state_machine.dialect_options;
+	if (!options.columns_set) {
+		if (options.file_options.hive_partitioning || options.file_options.union_by_name || options.multi_file_reader) {
+			// Just do the replacement
+			for (idx_t i = 0; i < MinValue<idx_t>(detected_names.size(), options.name_list.size()); i++) {
+				detected_names[i] = options.name_list[i];
+			}
+			return;
+		}
+		if (options.name_list.size() > dialect_options.num_cols) {
+			if (options.null_padding) {
+				// we increase our types
+				idx_t col = 0;
+				for (idx_t i = dialect_options.num_cols; i < options.name_list.size(); i++) {
+					detected_names.push_back(GenerateColumnName(options.name_list.size(), col++));
+					best_sql_types_candidates_per_column_idx[i] = {LogicalType::VARCHAR};
+				}
+
+				dialect_options.num_cols = options.name_list.size();
+
+			} else {
+				// we throw an error
+				const auto error = CSVError::HeaderSniffingError(
+				    options, best_header_row, options.name_list.size(),
+				    state_machine.dialect_options.state_machine_options.delimiter.GetValue());
+				error_handler.Error(error);
+			}
+		}
+		for (idx_t i = 0; i < options.name_list.size(); i++) {
+			detected_names[i] = options.name_list[i];
+		}
+	}
+}
+
 // If our columns were set by the user, we verify if their names match with the first row
 bool CSVSniffer::DetectHeaderWithSetColumn(ClientContext &context, vector<HeaderValue> &best_header_row,
                                            const SetColumns &set_columns, CSVReaderOptions &options) {
@@ -174,17 +212,15 @@ CSVSniffer::DetectHeaderInternal(ClientContext &context, vector<HeaderValue> &be
                                  CSVReaderOptions &options, CSVErrorHandler &error_handler) {
 	vector<string> detected_names;
 	auto &dialect_options = state_machine.dialect_options;
+	dialect_options.num_cols = best_sql_types_candidates_per_column_idx.size();
 	if (best_header_row.empty()) {
 		dialect_options.header = false;
 		for (idx_t col = 0; col < dialect_options.num_cols; col++) {
 			detected_names.push_back(GenerateColumnName(dialect_options.num_cols, col));
 		}
 		// If the user provided names, we must replace our header with the user provided names
-		if (!options.columns_set) {
-			for (idx_t i = 0; i < MinValue<idx_t>(best_header_row.size(), options.name_list.size()); i++) {
-				detected_names[i] = options.name_list[i];
-			}
-		}
+		ReplaceNames(detected_names, state_machine, best_sql_types_candidates_per_column_idx, options, best_header_row,
+		             error_handler);
 		return detected_names;
 	}
 	// information for header detection
@@ -192,6 +228,16 @@ CSVSniffer::DetectHeaderInternal(ClientContext &context, vector<HeaderValue> &be
 	// If null-padding is not allowed and there is a mismatch between our header candidate and the number of columns
 	// We can't detect the dialect/type options properly
 	if (!options.null_padding && best_sql_types_candidates_per_column_idx.size() != best_header_row.size()) {
+		if (options.ignore_errors.GetValue()) {
+			dialect_options.header = false;
+			for (idx_t col = 0; col < dialect_options.num_cols; col++) {
+				detected_names.push_back(GenerateColumnName(dialect_options.num_cols, col));
+			}
+			dialect_options.rows_until_header += 1;
+			ReplaceNames(detected_names, state_machine, best_sql_types_candidates_per_column_idx, options,
+			             best_header_row, error_handler);
+			return detected_names;
+		}
 		auto error =
 		    CSVError::HeaderSniffingError(options, best_header_row, best_sql_types_candidates_per_column_idx.size(),
 		                                  state_machine.dialect_options.state_machine_options.delimiter.GetValue());
@@ -281,16 +327,24 @@ CSVSniffer::DetectHeaderInternal(ClientContext &context, vector<HeaderValue> &be
 	}
 
 	// If the user provided names, we must replace our header with the user provided names
-	if (!options.columns_set) {
-		for (idx_t i = 0; i < MinValue<idx_t>(detected_names.size(), options.name_list.size()); i++) {
-			detected_names[i] = options.name_list[i];
-		}
-	}
+	ReplaceNames(detected_names, state_machine, best_sql_types_candidates_per_column_idx, options, best_header_row,
+	             error_handler);
 	return detected_names;
 }
 void CSVSniffer::DetectHeader() {
 	auto &sniffer_state_machine = best_candidate->GetStateMachine();
 	names = DetectHeaderInternal(buffer_manager->context, best_header_row, sniffer_state_machine, set_columns,
 	                             best_sql_types_candidates_per_column_idx, options, *error_handler);
+	if (single_row_file && sniffer_state_machine.dialect_options.header.GetValue()) {
+		// This file only contains a header, lets default to the lowest type of all.
+		detected_types.clear();
+		for (idx_t i = 0; i < names.size(); i++) {
+			detected_types.push_back(LogicalType::BOOLEAN);
+		}
+	}
+	for (idx_t i = max_columns_found; i < names.size(); i++) {
+		detected_types.push_back(LogicalType::VARCHAR);
+	}
+	max_columns_found = names.size();
 }
 } // namespace duckdb

package/src/duckdb/src/execution/operator/csv_scanner/sniffer/type_detection.cpp

@@ -95,6 +95,10 @@ void CSVSniffer::SetDateFormat(CSVStateMachine &candidate, const string &format_
 	candidate.dialect_options.date_format[sql_type].Set(strpformat, false);
 }
 
+idx_t CSVSniffer::LinesSniffed() const {
+	return lines_sniffed;
+}
+
 bool CSVSniffer::CanYouCastIt(ClientContext &context, const string_t value, const LogicalType &type,
                               const DialectOptions &dialect_options, const bool is_null, const char decimal_separator) {
 	if (is_null) {
@@ -158,7 +162,7 @@ bool CSVSniffer::CanYouCastIt(ClientContext &context, const string_t value, cons
 		idx_t pos;
 		bool special;
 		date_t dummy_value;
-		return Date::TryConvertDate(value_ptr, value_size, pos, dummy_value, special, true);
+		return Date::TryConvertDate(value_ptr, value_size, pos, dummy_value, special, true) == DateCastResult::SUCCESS;
 	}
 	case LogicalTypeId::TIMESTAMP: {
 		timestamp_t dummy_value;
@@ -292,10 +296,10 @@ void CSVSniffer::DetectDateAndTimeStampFormats(CSVStateMachine &candidate, const
 	// check all formats and keep the first one that works
 	StrpTimeFormat::ParseResult result;
 	auto save_format_candidates = type_format_candidates;
-	bool had_format_candidates = !save_format_candidates.empty();
-	bool initial_format_candidates =
+	const bool had_format_candidates = !save_format_candidates.empty();
+	const bool initial_format_candidates =
 	    save_format_candidates.size() == original_format_candidates.at(sql_type.id()).format.size();
-	bool is_set_by_user = options.dialect_options.date_format.find(sql_type.id())->second.IsSetByUser();
+	const bool is_set_by_user = options.dialect_options.date_format.find(sql_type.id())->second.IsSetByUser();
 	while (!type_format_candidates.empty() && !is_set_by_user) {
 		//	avoid using exceptions for flow control...
 		auto &current_format = candidate.dialect_options.date_format[sql_type.id()].GetValue();
@@ -367,7 +371,9 @@ void CSVSniffer::SniffTypes(DataChunk &data_chunk, CSVStateMachine &state_machin
 					break;
 				}
 
-				if (row_idx != start_idx_detection && cur_top_candidate == LogicalType::BOOLEAN) {
+				if (row_idx != start_idx_detection &&
+				    (cur_top_candidate == LogicalType::BOOLEAN || cur_top_candidate == LogicalType::DATE ||
+				     cur_top_candidate == LogicalType::TIME || cur_top_candidate == LogicalType::TIMESTAMP)) {
 					// If we thought this was a boolean value (i.e., T,F, True, False) and it is not, we
 					// immediately pop to varchar.
 					while (col_type_candidates.back() != LogicalType::VARCHAR) {
@@ -419,19 +425,9 @@ void CSVSniffer::DetectTypes() {
 		SetUserDefinedDateTimeFormat(*candidate->state_machine);
 		// Parse chunk and read csv with info candidate
 		auto &data_chunk = candidate->ParseChunk().ToChunk();
-		if (!candidate->error_handler->errors.empty()) {
-			bool break_loop = false;
-			for (auto &errors : candidate->error_handler->errors) {
-				for (auto &error : errors.second) {
-					if (error.type != CSVErrorType::MAXIMUM_LINE_SIZE) {
-						break_loop = true;
-						break;
-					}
-				}
-			}
-			if (break_loop && !candidate->state_machine->options.ignore_errors.GetValue()) {
-				continue;
-			}
+		if (candidate->error_handler->AnyErrors() && !candidate->error_handler->HasError(MAXIMUM_LINE_SIZE) &&
+		    !candidate->state_machine->options.ignore_errors.GetValue()) {
+			continue;
 		}
 		idx_t start_idx_detection = 0;
 		idx_t chunk_size = data_chunk.size();
@@ -457,11 +453,11 @@ void CSVSniffer::DetectTypes() {
 
 		// it's good if the dialect creates more non-varchar columns, but only if we sacrifice < 30% of
 		// best_num_cols.
-		if (!best_candidate ||
-		    (varchar_cols<min_varchar_cols &&static_cast<double>(info_sql_types_candidates.size())>(
-		         static_cast<double>(max_columns_found) * 0.7) &&
-		     (!options.ignore_errors.GetValue() || candidate->error_handler->errors.size() < min_errors))) {
-			min_errors = candidate->error_handler->errors.size();
+		const idx_t number_of_errors = candidate->error_handler->GetSize();
+		if (!best_candidate || (varchar_cols<min_varchar_cols &&static_cast<double>(info_sql_types_candidates.size())>(
+		                            static_cast<double>(max_columns_found) * 0.7) &&
+		                        (!options.ignore_errors.GetValue() || number_of_errors < min_errors))) {
+			min_errors = number_of_errors;
 			best_header_row.clear();
 			// we have a new best_options candidate
 			best_candidate = std::move(candidate);
@@ -471,6 +467,7 @@ void CSVSniffer::DetectTypes() {
 				best_format_candidates[format_candidate.first] = format_candidate.second.format;
 			}
 			if (chunk_size > 0) {
+				single_row_file = chunk_size == 1;
 				for (idx_t col_idx = 0; col_idx < data_chunk.ColumnCount(); col_idx++) {
 					auto &cur_vector = data_chunk.data[col_idx];
 					auto vector_data = FlatVector::GetData<string_t>(cur_vector);