duckdb 0.10.3-dev0.0 → 0.10.3-dev6.0

package/src/duckdb/src/execution/operator/csv_scanner/sniffer/csv_sniffer.cpp

@@ -13,8 +13,8 @@ CSVSniffer::CSVSniffer(CSVReaderOptions &options_p, shared_ptr<CSVBufferManager>
 	}
 	// Initialize max columns found to either 0 or however many were set
 	max_columns_found = set_columns.Size();
-	error_handler = make_shared<CSVErrorHandler>(options.ignore_errors.GetValue());
-	detection_error_handler = make_shared<CSVErrorHandler>(true);
+	error_handler = make_shared_ptr<CSVErrorHandler>(options.ignore_errors.GetValue());
+	detection_error_handler = make_shared_ptr<CSVErrorHandler>(true);
 }
 
 bool SetColumns::IsSet() {

package/src/duckdb/src/execution/operator/csv_scanner/sniffer/dialect_detection.cpp

@@ -1,5 +1,6 @@
 #include "duckdb/execution/operator/csv_scanner/csv_sniffer.hpp"
 #include "duckdb/main/client_data.hpp"
+#include "duckdb/common/shared_ptr.hpp"
 
 namespace duckdb {
 
@@ -89,13 +90,13 @@ void CSVSniffer::AnalyzeDialectCandidate(unique_ptr<ColumnCountScanner> scanner,
                                          idx_t &best_consistent_rows, idx_t &prev_padding_count) {
 	// The sniffed_column_counts variable keeps track of the number of columns found for each row
 	auto &sniffed_column_counts = scanner->ParseChunk();
+	idx_t dirty_notes = 0;
 	if (sniffed_column_counts.error) {
 		// This candidate has an error (i.e., over maximum line size or never unquoting quoted values)
 		return;
 	}
-	idx_t start_row = options.dialect_options.skip_rows.GetValue();
 	idx_t consistent_rows = 0;
-	idx_t num_cols = sniffed_column_counts.result_position == 0 ? 1 : sniffed_column_counts[start_row];
+	idx_t num_cols = sniffed_column_counts.result_position == 0 ? 1 : sniffed_column_counts[0];
 	idx_t padding_count = 0;
 	bool allow_padding = options.null_padding;
 	if (sniffed_column_counts.result_position > rows_read) {
@@ -106,7 +107,7 @@ void CSVSniffer::AnalyzeDialectCandidate(unique_ptr<ColumnCountScanner> scanner,
 		// Not acceptable
 		return;
 	}
-	for (idx_t row = start_row; row < sniffed_column_counts.result_position; row++) {
+	for (idx_t row = 0; row < sniffed_column_counts.result_position; row++) {
 		if (set_columns.IsCandidateUnacceptable(sniffed_column_counts[row], options.null_padding,
 		                                        options.ignore_errors.GetValue(),
 		                                        sniffed_column_counts.last_value_always_empty)) {
@@ -121,7 +122,7 @@ void CSVSniffer::AnalyzeDialectCandidate(unique_ptr<ColumnCountScanner> scanner,
 			padding_count = 0;
 			// we use the maximum amount of num_cols that we find
 			num_cols = sniffed_column_counts[row];
-			start_row = row;
+			dirty_notes = row;
 			consistent_rows = 1;
 
 		} else if (num_cols >= sniffed_column_counts[row]) {
@@ -147,7 +148,7 @@ void CSVSniffer::AnalyzeDialectCandidate(unique_ptr<ColumnCountScanner> scanner,
 
 	// If the number of rows is consistent with the calculated value after accounting for skipped rows and the
 	// start row.
-	bool rows_consistent = consistent_rows + (start_row - options.dialect_options.skip_rows.GetValue()) ==
+	bool rows_consistent = consistent_rows + (dirty_notes - options.dialect_options.skip_rows.GetValue()) ==
 	                       sniffed_column_counts.result_position - options.dialect_options.skip_rows.GetValue();
 	// If there are more than one consistent row.
 	bool more_than_one_row = (consistent_rows > 1);
@@ -157,7 +158,7 @@ void CSVSniffer::AnalyzeDialectCandidate(unique_ptr<ColumnCountScanner> scanner,
 
 	// If the start position is valid.
 	bool start_good = !candidates.empty() &&
-	                  (start_row <= candidates.front()->GetStateMachine().dialect_options.skip_rows.GetValue());
+	                  (dirty_notes <= candidates.front()->GetStateMachine().dialect_options.skip_rows.GetValue());
 
 	// If padding happened but it is not allowed.
 	bool invalid_padding = !allow_padding && padding_count > 0;
@@ -185,11 +186,17 @@ void CSVSniffer::AnalyzeDialectCandidate(unique_ptr<ColumnCountScanner> scanner,
 		best_consistent_rows = consistent_rows;
 		max_columns_found = num_cols;
 		prev_padding_count = padding_count;
-		if (!options.null_padding && !options.ignore_errors.GetValue()) {
-			sniffing_state_machine.dialect_options.skip_rows = start_row;
-		} else {
+		if (options.dialect_options.skip_rows.IsSetByUser()) {
+			// If skip rows is set by user, and we found dirty notes, we only accept it if either null_padding or
+			// ignore_errors is set
+			if (dirty_notes != 0 && !options.null_padding && !options.ignore_errors.GetValue()) {
+				return;
+			}
 			sniffing_state_machine.dialect_options.skip_rows = options.dialect_options.skip_rows.GetValue();
+		} else if (!options.null_padding && !options.ignore_errors.GetValue()) {
+			sniffing_state_machine.dialect_options.skip_rows = dirty_notes;
 		}
+
 		candidates.clear();
 		sniffing_state_machine.dialect_options.num_cols = num_cols;
 		candidates.emplace_back(std::move(scanner));
@@ -210,11 +217,17 @@ void CSVSniffer::AnalyzeDialectCandidate(unique_ptr<ColumnCountScanner> scanner,
 			}
 		}
 		if (!same_quote_is_candidate) {
-			if (!options.null_padding && !options.ignore_errors.GetValue()) {
-				sniffing_state_machine.dialect_options.skip_rows = start_row;
-			} else {
+			if (options.dialect_options.skip_rows.IsSetByUser()) {
+				// If skip rows is set by user, and we found dirty notes, we only accept it if either null_padding or
+				// ignore_errors is set
+				if (dirty_notes != 0 && !options.null_padding && !options.ignore_errors.GetValue()) {
+					return;
+				}
 				sniffing_state_machine.dialect_options.skip_rows = options.dialect_options.skip_rows.GetValue();
+			} else if (!options.null_padding && !options.ignore_errors.GetValue()) {
+				sniffing_state_machine.dialect_options.skip_rows = dirty_notes;
 			}
+
 			sniffing_state_machine.dialect_options.num_cols = num_cols;
 			candidates.emplace_back(std::move(scanner));
 		}
@@ -277,9 +290,7 @@ void CSVSniffer::RefineCandidates() {
 				candidates.push_back(std::move(cc_best_candidate));
 				return;
 			}
-			if (cc_best_candidate->state_machine->state_machine_options.quote == '\0') {
-				candidates.push_back(std::move(cc_best_candidate));
-			}
+			candidates.push_back(std::move(cc_best_candidate));
 		}
 		return;
 	}
@@ -308,6 +319,24 @@ NewLineIdentifier CSVSniffer::DetectNewLineDelimiter(CSVBufferManager &buffer_ma
 	return NewLineIdentifier::SINGLE;
 }
 
+void CSVSniffer::SkipLines(vector<unique_ptr<ColumnCountScanner>> &csv_state_machines) {
+	if (csv_state_machines.empty()) {
+		return;
+	}
+	auto &first_scanner = *csv_state_machines[0];
+	// We figure out the iterator position for the first scanner
+	if (options.dialect_options.skip_rows.IsSetByUser()) {
+		first_scanner.SkipCSVRows(options.dialect_options.skip_rows.GetValue());
+	}
+	// The iterator position is the same regardless of the scanner configuration, hence we apply the same iterator
+	// To the remaining scanners
+	const auto first_iterator = first_scanner.GetIterator();
+	for (idx_t i = 1; i < csv_state_machines.size(); i++) {
+		auto &cur_scanner = *csv_state_machines[i];
+		cur_scanner.SetIterator(first_iterator);
+	}
+}
+
 // Dialect Detection consists of five steps:
 // 1. Generate a search space of all possible dialects
 // 2. Generate a state machine for each dialect
@@ -341,9 +370,9 @@ void CSVSniffer::DetectDialect() {
 	// Step 2: Generate state machines
 	GenerateStateMachineSearchSpace(csv_state_machines, delim_candidates, quoterule_candidates, quote_candidates_map,
 	                                escape_candidates_map);
+	SkipLines(csv_state_machines);
 	// Step 3: Analyze all candidates on the first chunk
 	for (auto &state_machine : csv_state_machines) {
-		state_machine->Reset();
 		AnalyzeDialectCandidate(std::move(state_machine), rows_read, best_consistent_rows, prev_padding_count);
 	}
 	// Step 4: Loop over candidates and find if they can still produce good results for the remaining chunks

package/src/duckdb/src/execution/operator/csv_scanner/sniffer/header_detection.cpp

@@ -1,15 +1,17 @@
 #include "duckdb/common/types/cast_helpers.hpp"
 #include "duckdb/execution/operator/csv_scanner/csv_sniffer.hpp"
 #include "duckdb/execution/operator/csv_scanner/csv_reader_options.hpp"
+#include "duckdb/common/types/value.hpp"
+
 #include "utf8proc.hpp"
 
 namespace duckdb {
 
 // Helper function to generate column names
 static string GenerateColumnName(const idx_t total_cols, const idx_t col_number, const string &prefix = "column") {
-	int max_digits = NumericHelper::UnsignedLength(total_cols - 1);
-	int digits = NumericHelper::UnsignedLength(col_number);
-	string leading_zeros = string(max_digits - digits, '0');
+	auto max_digits = NumericHelper::UnsignedLength(total_cols - 1);
+	auto digits = NumericHelper::UnsignedLength(col_number);
+	string leading_zeros = string(NumericCast<idx_t>(max_digits - digits), '0');
 	string value = to_string(col_number);
 	return string(prefix + leading_zeros + value);
 }
@@ -22,21 +24,21 @@ static string TrimWhitespace(const string &col_name) {
 	// Find the first character that is not left trimmed
 	idx_t begin = 0;
 	while (begin < size) {
-		auto bytes = utf8proc_iterate(str + begin, size - begin, &codepoint);
+		auto bytes = utf8proc_iterate(str + begin, NumericCast<utf8proc_ssize_t>(size - begin), &codepoint);
 		D_ASSERT(bytes > 0);
 		if (utf8proc_category(codepoint) != UTF8PROC_CATEGORY_ZS) {
 			break;
 		}
-		begin += bytes;
+		begin += NumericCast<idx_t>(bytes);
 	}
 
 	// Find the last character that is not right trimmed
 	idx_t end;
 	end = begin;
 	for (auto next = begin; next < col_name.size();) {
-		auto bytes = utf8proc_iterate(str + next, size - next, &codepoint);
+		auto bytes = utf8proc_iterate(str + next, NumericCast<utf8proc_ssize_t>(size - next), &codepoint);
 		D_ASSERT(bytes > 0);
-		next += bytes;
+		next += NumericCast<idx_t>(bytes);
 		if (utf8proc_category(codepoint) != UTF8PROC_CATEGORY_ZS) {
 			end = next;
 		}
@@ -48,7 +50,8 @@ static string TrimWhitespace(const string &col_name) {
 
 static string NormalizeColumnName(const string &col_name) {
 	// normalize UTF8 characters to NFKD
-	auto nfkd = utf8proc_NFKD(reinterpret_cast<const utf8proc_uint8_t *>(col_name.c_str()), col_name.size());
+	auto nfkd = utf8proc_NFKD(reinterpret_cast<const utf8proc_uint8_t *>(col_name.c_str()),
+	                          NumericCast<utf8proc_ssize_t>(col_name.size()));
 	const string col_name_nfkd = string(const_char_ptr_cast(nfkd), strlen(const_char_ptr_cast(nfkd)));
 	free(nfkd);
 
@@ -109,7 +112,7 @@ bool CSVSniffer::DetectHeaderWithSetColumn() {
 			if (best_header_row[i].IsNull()) {
 				return false;
 			}
-			if (best_header_row[i] != (*set_columns.names)[i]) {
+			if (best_header_row[i].value.GetString() != (*set_columns.names)[i]) {
 				has_header = false;
 				break;
 			}
@@ -118,12 +121,12 @@ bool CSVSniffer::DetectHeaderWithSetColumn() {
 	if (!has_header) {
 		// We verify if the types are consistent
 		for (idx_t col = 0; col < set_columns.Size(); col++) {
-			auto dummy_val = best_header_row[col];
 			// try cast to sql_type of column
 			const auto &sql_type = (*set_columns.types)[col];
 			if (sql_type != LogicalType::VARCHAR) {
 				all_varchar = false;
-				if (!TryCastValue(options.dialect_options, options.decimal_separator, dummy_val, sql_type)) {
+				if (!CanYouCastIt(best_header_row[col].value, sql_type, options.dialect_options,
+				                  best_header_row[col].IsNull(), options.decimal_separator[0])) {
 					first_row_consistent = false;
 				}
 			}
@@ -167,16 +170,15 @@ void CSVSniffer::DetectHeader() {
 		has_header = DetectHeaderWithSetColumn();
 	} else {
 		for (idx_t col = 0; col < best_header_row.size(); col++) {
-			auto dummy_val = best_header_row[col];
-			if (!dummy_val.IsNull()) {
+			if (!best_header_row[col].IsNull()) {
 				first_row_nulls = false;
 			}
 			// try cast to sql_type of column
 			const auto &sql_type = best_sql_types_candidates_per_column_idx[col].back();
 			if (sql_type != LogicalType::VARCHAR) {
 				all_varchar = false;
-				if (!TryCastValue(sniffer_state_machine.dialect_options,
-				                  sniffer_state_machine.options.decimal_separator, dummy_val, sql_type)) {
+				if (!CanYouCastIt(best_header_row[col].value, sql_type, sniffer_state_machine.dialect_options,
+				                  best_header_row[col].IsNull(), options.decimal_separator[0])) {
 					first_row_consistent = false;
 				}
 			}
@@ -207,11 +209,10 @@ void CSVSniffer::DetectHeader() {
 
 		// get header names from CSV
 		for (idx_t col = 0; col < best_header_row.size(); col++) {
-			const auto &val = best_header_row[col];
-			string col_name = val.ToString();
+			string col_name = best_header_row[col].value.GetString();
 
 			// generate name if field is empty
-			if (col_name.empty() || val.IsNull()) {
+			if (col_name.empty() || best_header_row[col].IsNull()) {
 				col_name = GenerateColumnName(sniffer_state_machine.dialect_options.num_cols, col);
 			}
 

package/src/duckdb/src/execution/operator/csv_scanner/sniffer/type_detection.cpp

@@ -1,7 +1,10 @@
-#include "duckdb/common/operator/decimal_cast_operators.hpp"
-#include "duckdb/execution/operator/csv_scanner/csv_sniffer.hpp"
 #include "duckdb/common/algorithm.hpp"
+#include "duckdb/common/operator/decimal_cast_operators.hpp"
+#include "duckdb/common/operator/double_cast_operator.hpp"
+#include "duckdb/common/operator/integer_cast_operator.hpp"
 #include "duckdb/common/string.hpp"
+#include "duckdb/common/types/time.hpp"
+#include "duckdb/execution/operator/csv_scanner/csv_sniffer.hpp"
 
 namespace duckdb {
 struct TryCastFloatingOperator {
@@ -14,9 +17,9 @@ struct TryCastFloatingOperator {
 	}
 };
 
-static bool StartsWithNumericDate(string &separator, const string &value) {
-	auto begin = value.c_str();
-	auto end = begin + value.size();
+static bool StartsWithNumericDate(string &separator, const string_t &value) {
+	auto begin = value.GetData();
+	auto end = begin + value.GetSize();
 
 	//	StrpTimeFormat::Parse will skip whitespace, so we can too
 	auto field1 = std::find_if_not(begin, end, StringUtil::CharacterIsSpace);
@@ -50,7 +53,8 @@ static bool StartsWithNumericDate(string &separator, const string &value) {
 	}
 
 	//	second literal must match first
-	if (((field3 - literal2) != (field2 - literal1)) || strncmp(literal1, literal2, (field2 - literal1)) != 0) {
+	if (((field3 - literal2) != (field2 - literal1)) ||
+	    strncmp(literal1, literal2, NumericCast<size_t>((field2 - literal1))) != 0) {
 		return false;
 	}
 
@@ -69,7 +73,7 @@ static bool StartsWithNumericDate(string &separator, const string &value) {
 
 string GenerateDateFormat(const string &separator, const char *format_template) {
 	string format_specifier = format_template;
-	auto amount_of_dashes = std::count(format_specifier.begin(), format_specifier.end(), '-');
+	auto amount_of_dashes = NumericCast<idx_t>(std::count(format_specifier.begin(), format_specifier.end(), '-'));
 	// All our date formats must have at least one -
 	D_ASSERT(amount_of_dashes);
 	string result;
@@ -84,40 +88,161 @@ string GenerateDateFormat(const string &separator, const char *format_template)
 	return result;
 }
 
-bool CSVSniffer::TryCastValue(const DialectOptions &dialect_options, const string &decimal_separator,
-                              const Value &value, const LogicalType &sql_type) {
-	if (value.IsNull()) {
+void CSVSniffer::SetDateFormat(CSVStateMachine &candidate, const string &format_specifier,
+                               const LogicalTypeId &sql_type) {
+	StrpTimeFormat strpformat;
+	StrTimeFormat::ParseFormatSpecifier(format_specifier, strpformat);
+	candidate.dialect_options.date_format[sql_type].Set(strpformat, false);
+}
+
+bool CSVSniffer::CanYouCastIt(const string_t value, const LogicalType &type, const DialectOptions &dialect_options,
+                              const bool is_null, const char decimal_separator) {
+	if (is_null) {
 		return true;
 	}
-	if (!dialect_options.date_format.find(LogicalTypeId::DATE)->second.GetValue().Empty() &&
-	    sql_type.id() == LogicalTypeId::DATE) {
-		date_t result;
-		string error_message;
-		return dialect_options.date_format.find(LogicalTypeId::DATE)
-		    ->second.GetValue()
-		    .TryParseDate(string_t(StringValue::Get(value)), result, error_message);
+	auto value_ptr = value.GetData();
+	auto value_size = value.GetSize();
+	switch (type.id()) {
+	case LogicalTypeId::TINYINT: {
+		int8_t dummy_value;
+		return TrySimpleIntegerCast(value_ptr, value_size, dummy_value, false);
 	}
-	if (!dialect_options.date_format.find(LogicalTypeId::TIMESTAMP)->second.GetValue().Empty() &&
-	    sql_type.id() == LogicalTypeId::TIMESTAMP) {
-		timestamp_t result;
-		string error_message;
-		return dialect_options.date_format.find(LogicalTypeId::TIMESTAMP)
-		    ->second.GetValue()
-		    .TryParseTimestamp(string_t(StringValue::Get(value)), result, error_message);
+	case LogicalTypeId::SMALLINT: {
+		int16_t dummy_value;
+		return TrySimpleIntegerCast(value_ptr, value_size, dummy_value, true);
 	}
-	if (decimal_separator != "." && (sql_type.id() == LogicalTypeId::DOUBLE)) {
-		return TryCastFloatingOperator::Operation<TryCastErrorMessageCommaSeparated, double>(StringValue::Get(value));
+	case LogicalTypeId::INTEGER: {
+		int32_t dummy_value;
+		return TrySimpleIntegerCast(value_ptr, value_size, dummy_value, true);
 	}
-	Value new_value;
-	string error_message;
-	return value.TryCastAs(buffer_manager->context, sql_type, new_value, &error_message, true);
-}
+	case LogicalTypeId::BIGINT: {
+		int64_t dummy_value;
+		return TrySimpleIntegerCast(value_ptr, value_size, dummy_value, true);
+	}
+	case LogicalTypeId::UTINYINT: {
+		uint8_t dummy_value;
+		return TrySimpleIntegerCast(value_ptr, value_size, dummy_value, true);
+	}
+	case LogicalTypeId::USMALLINT: {
+		uint16_t dummy_value;
+		return TrySimpleIntegerCast(value_ptr, value_size, dummy_value, true);
+	}
+	case LogicalTypeId::UINTEGER: {
+		uint32_t dummy_value;
+		return TrySimpleIntegerCast(value_ptr, value_size, dummy_value, true);
+	}
+	case LogicalTypeId::UBIGINT: {
+		uint64_t dummy_value;
+		return TrySimpleIntegerCast(value_ptr, value_size, dummy_value, true);
+	}
+	case LogicalTypeId::DOUBLE: {
+		double dummy_value;
+		return TryDoubleCast<double>(value_ptr, value_size, dummy_value, true, options.decimal_separator[0]);
+	}
+	case LogicalTypeId::FLOAT: {
+		float dummy_value;
+		return TryDoubleCast<float>(value_ptr, value_size, dummy_value, true, options.decimal_separator[0]);
+	}
+	case LogicalTypeId::DATE: {
+		if (!dialect_options.date_format.find(LogicalTypeId::DATE)->second.GetValue().Empty()) {
+			date_t result;
+			string error_message;
+			return dialect_options.date_format.find(LogicalTypeId::DATE)
+			    ->second.GetValue()
+			    .TryParseDate(value, result, error_message);
+		} else {
+			idx_t pos;
+			bool special;
+			date_t dummy_value;
+			return Date::TryConvertDate(value_ptr, value_size, pos, dummy_value, special, true);
+		}
+	}
+	case LogicalTypeId::TIMESTAMP: {
+		timestamp_t dummy_value;
+		if (!dialect_options.date_format.find(LogicalTypeId::TIMESTAMP)->second.GetValue().Empty()) {
+			string error_message;
+			return dialect_options.date_format.find(LogicalTypeId::TIMESTAMP)
+			    ->second.GetValue()
+			    .TryParseTimestamp(value, dummy_value, error_message);
+		} else {
+			return Timestamp::TryConvertTimestamp(value_ptr, value_size, dummy_value) == TimestampCastResult::SUCCESS;
+		}
+	}
+	case LogicalTypeId::TIME: {
+		idx_t pos;
+		dtime_t dummy_value;
+		return Time::TryConvertTime(value_ptr, value_size, pos, dummy_value, true);
+	}
+	case LogicalTypeId::DECIMAL: {
+		uint8_t width, scale;
+		type.GetDecimalProperties(width, scale);
+		if (decimal_separator == ',') {
+			switch (type.InternalType()) {
+			case PhysicalType::INT16: {
+				int16_t dummy_value;
+				return TryDecimalStringCast<int16_t, ','>(value_ptr, value_size, dummy_value, width, scale);
+			}
 
-void CSVSniffer::SetDateFormat(CSVStateMachine &candidate, const string &format_specifier,
-                               const LogicalTypeId &sql_type) {
-	StrpTimeFormat strpformat;
-	StrTimeFormat::ParseFormatSpecifier(format_specifier, strpformat);
-	candidate.dialect_options.date_format[sql_type].Set(strpformat, false);
+			case PhysicalType::INT32: {
+				int32_t dummy_value;
+				return TryDecimalStringCast<int32_t, ','>(value_ptr, value_size, dummy_value, width, scale);
+			}
+
+			case PhysicalType::INT64: {
+				int64_t dummy_value;
+				return TryDecimalStringCast<int64_t, ','>(value_ptr, value_size, dummy_value, width, scale);
+			}
+
+			case PhysicalType::INT128: {
+				hugeint_t dummy_value;
+				return TryDecimalStringCast<hugeint_t, ','>(value_ptr, value_size, dummy_value, width, scale);
+			}
+
+			default:
+				throw InternalException("Invalid Physical Type for Decimal Value. Physical Type: " +
+				                        TypeIdToString(type.InternalType()));
+			}
+
+		} else if (decimal_separator == '.') {
+			switch (type.InternalType()) {
+			case PhysicalType::INT16: {
+				int16_t dummy_value;
+				return TryDecimalStringCast(value_ptr, value_size, dummy_value, width, scale);
+			}
+
+			case PhysicalType::INT32: {
+				int32_t dummy_value;
+				return TryDecimalStringCast(value_ptr, value_size, dummy_value, width, scale);
+			}
+
+			case PhysicalType::INT64: {
+				int64_t dummy_value;
+				return TryDecimalStringCast(value_ptr, value_size, dummy_value, width, scale);
+			}
+
+			case PhysicalType::INT128: {
+				hugeint_t dummy_value;
+				return TryDecimalStringCast(value_ptr, value_size, dummy_value, width, scale);
+			}
+
+			default:
+				throw InternalException("Invalid Physical Type for Decimal Value. Physical Type: " +
+				                        TypeIdToString(type.InternalType()));
+			}
+		} else {
+			throw InvalidInputException("Decimals can only have ',' and '.' as decimal separators");
+		}
+	}
+	case LogicalTypeId::VARCHAR:
+		return true;
+	default: {
+		// We do Value Try Cast for non-basic types.
+		Value new_value;
+		string error_message;
+		Value str_value(value);
+		return str_value.TryCastAs(buffer_manager->context, type, new_value, &error_message, true);
+	}
+	}
 }
 
 void CSVSniffer::InitializeDateAndTimeStampDetection(CSVStateMachine &candidate, const string &separator,
@@ -142,6 +267,7 @@ void CSVSniffer::InitializeDateAndTimeStampDetection(CSVStateMachine &candidate,
 				}
 			}
 		}
+		original_format_candidates = format_candidates;
 	}
 	//	initialise the first candidate
 	//	all formats are constructed to be valid
@@ -149,8 +275,8 @@ void CSVSniffer::InitializeDateAndTimeStampDetection(CSVStateMachine &candidate,
 }
 
 void CSVSniffer::DetectDateAndTimeStampFormats(CSVStateMachine &candidate, const LogicalType &sql_type,
-                                               const string &separator, Value &dummy_val) {
-	// If it is the first time running date/timestamp detection we must initilize the format variables
+                                               const string &separator, string_t &dummy_val) {
+	// If it is the first time running date/timestamp detection we must initialize the format variables
 	InitializeDateAndTimeStampDetection(candidate, separator, sql_type);
 	// generate date format candidates the first time through
 	auto &type_format_candidates = format_candidates[sql_type.id()].format;
@@ -159,11 +285,12 @@ void CSVSniffer::DetectDateAndTimeStampFormats(CSVStateMachine &candidate, const
 	auto save_format_candidates = type_format_candidates;
 	bool had_format_candidates = !save_format_candidates.empty();
 	bool initial_format_candidates =
-	    save_format_candidates.size() == format_template_candidates.at(sql_type.id()).size();
+	    save_format_candidates.size() == original_format_candidates.at(sql_type.id()).format.size();
 	while (!type_format_candidates.empty()) {
 		//	avoid using exceptions for flow control...
 		auto &current_format = candidate.dialect_options.date_format[sql_type.id()].GetValue();
-		if (current_format.Parse(StringValue::Get(dummy_val), result)) {
+		if (current_format.Parse(dummy_val, result)) {
+			format_candidates[sql_type.id()].had_match = true;
 			break;
 		}
 		//	doesn't work - move to the next one
@@ -177,14 +304,15 @@ void CSVSniffer::DetectDateAndTimeStampFormats(CSVStateMachine &candidate, const
 		//	so restore the candidates that did work.
 		//	or throw them out if they were generated by this value.
 		if (had_format_candidates) {
-			if (initial_format_candidates) {
+			if (initial_format_candidates && !format_candidates[sql_type.id()].had_match) {
 				// we reset the whole thing because we tried to sniff the wrong type.
 				format_candidates[sql_type.id()].initialized = false;
 				format_candidates[sql_type.id()].format.clear();
+				SetDateFormat(candidate, "", sql_type.id());
 				return;
 			}
 			type_format_candidates.swap(save_format_candidates);
-			SetDateFormat(candidate, "", sql_type.id());
+			SetDateFormat(candidate, type_format_candidates.back(), sql_type.id());
 		}
 	}
 }
@@ -210,9 +338,10 @@ void CSVSniffer::DetectTypes() {
 		auto candidate = candidate_cc->UpgradeToStringValueScanner();
 
 		// Parse chunk and read csv with info candidate
-		auto &tuples = candidate->ParseChunk();
+		auto &data_chunk = candidate->ParseChunk().ToChunk();
 		idx_t row_idx = 0;
-		if (tuples.number_of_rows > 1 &&
+		idx_t chunk_size = data_chunk.size();
+		if (chunk_size > 1 &&
 		    (!options.dialect_options.header.IsSetByUser() ||
 		     (options.dialect_options.header.IsSetByUser() && options.dialect_options.header.GetValue()))) {
 			// This means we have more than one row, hence we can use the first row to detect if we have a header
@@ -220,14 +349,19 @@ void CSVSniffer::DetectTypes() {
 		}
 		// First line where we start our type detection
 		const idx_t start_idx_detection = row_idx;
-		for (; row_idx < tuples.number_of_rows; row_idx++) {
-			for (idx_t col_idx = 0; col_idx < tuples.number_of_columns; col_idx++) {
-				auto &col_type_candidates = info_sql_types_candidates[col_idx];
+
+		for (idx_t col_idx = 0; col_idx < data_chunk.ColumnCount(); col_idx++) {
+			auto &cur_vector = data_chunk.data[col_idx];
+			D_ASSERT(cur_vector.GetVectorType() == VectorType::FLAT_VECTOR);
+			D_ASSERT(cur_vector.GetType() == LogicalType::VARCHAR);
+			auto vector_data = FlatVector::GetData<string_t>(cur_vector);
+			auto null_mask = FlatVector::Validity(cur_vector);
+			auto &col_type_candidates = info_sql_types_candidates[col_idx];
+			for (row_idx = start_idx_detection; row_idx < chunk_size; row_idx++) {
 				// col_type_candidates can't be empty since anything in a CSV file should at least be a string
 				// and we validate utf-8 compatibility when creating the type
 				D_ASSERT(!col_type_candidates.empty());
 				auto cur_top_candidate = col_type_candidates.back();
-				auto dummy_val = tuples.GetValue(row_idx, col_idx);
 				// try cast from string to sql_type
 				while (col_type_candidates.size() > 1) {
 					const auto &sql_type = col_type_candidates.back();
@@ -236,18 +370,20 @@ void CSVSniffer::DetectTypes() {
 					string separator;
 					// If Value is not Null, Has a numeric date format, and the current investigated candidate is
 					// either a timestamp or a date
-					if (!dummy_val.IsNull() && StartsWithNumericDate(separator, StringValue::Get(dummy_val)) &&
+					if (null_mask.RowIsValid(row_idx) && StartsWithNumericDate(separator, vector_data[row_idx]) &&
 					    (col_type_candidates.back().id() == LogicalTypeId::TIMESTAMP ||
 					     col_type_candidates.back().id() == LogicalTypeId::DATE)) {
-						DetectDateAndTimeStampFormats(candidate->GetStateMachine(), sql_type, separator, dummy_val);
+						DetectDateAndTimeStampFormats(candidate->GetStateMachine(), sql_type, separator,
+						                              vector_data[row_idx]);
 					}
 					// try cast from string to sql_type
 					if (sql_type == LogicalType::VARCHAR) {
 						// Nothing to convert it to
 						continue;
 					}
-					if (TryCastValue(sniffing_state_machine.dialect_options,
-					                 sniffing_state_machine.options.decimal_separator, dummy_val, sql_type)) {
+					if (CanYouCastIt(vector_data[row_idx], sql_type, sniffing_state_machine.dialect_options,
+					                 !null_mask.RowIsValid(row_idx),
+					                 sniffing_state_machine.options.decimal_separator[0])) {
 						break;
 					} else {
 						if (row_idx != start_idx_detection && cur_top_candidate == LogicalType::BOOLEAN) {
@@ -277,7 +413,8 @@ void CSVSniffer::DetectTypes() {
 
 		// it's good if the dialect creates more non-varchar columns, but only if we sacrifice < 30% of
 		// best_num_cols.
-		if (varchar_cols < min_varchar_cols && info_sql_types_candidates.size() > (max_columns_found * 0.7) &&
+		if (varchar_cols<min_varchar_cols &&static_cast<double>(info_sql_types_candidates.size())>(
+		        static_cast<double>(max_columns_found) * 0.7) &&
 		    (!options.ignore_errors.GetValue() || candidate->error_handler->errors.size() < min_errors)) {
 			min_errors = candidate->error_handler->errors.size();
 			best_header_row.clear();
@@ -288,9 +425,17 @@ void CSVSniffer::DetectTypes() {
 			for (auto &format_candidate : format_candidates) {
 				best_format_candidates[format_candidate.first] = format_candidate.second.format;
 			}
-			if (tuples.number_of_rows > 0) {
-				for (idx_t col_idx = 0; col_idx < tuples.number_of_columns; col_idx++) {
-					best_header_row.emplace_back(tuples.GetValue(0, col_idx));
+			if (chunk_size > 0) {
+				for (idx_t col_idx = 0; col_idx < data_chunk.ColumnCount(); col_idx++) {
+					auto &cur_vector = data_chunk.data[col_idx];
+					auto vector_data = FlatVector::GetData<string_t>(cur_vector);
+					auto null_mask = FlatVector::Validity(cur_vector);
+					if (null_mask.RowIsValid(0)) {
+						auto value = HeaderValue(vector_data[0]);
+						best_header_row.push_back(value);
+					} else {
+						best_header_row.push_back({});
+					}
 				}
 			}
 		}