duckdb 0.6.2-dev971.0 → 0.7.1-dev100.0

package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp

@@ -5,6 +5,8 @@
 #include "duckdb/common/string_util.hpp"
 #include "duckdb/common/to_string.hpp"
 #include "duckdb/common/types/cast_helpers.hpp"
+#include "duckdb/common/operator/cast_operators.hpp"
+#include "duckdb/common/operator/decimal_cast_operators.hpp"
 #include "duckdb/common/vector_operations/unary_executor.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/function/scalar/strftime.hpp"
@@ -28,15 +30,10 @@ string BaseCSVReader::GetLineNumberStr(idx_t linenr, bool linenr_estimated) {
 	return to_string(linenr + 1) + estimated;
 }
 
-BaseCSVReader::BaseCSVReader(FileSystem &fs_p, Allocator &allocator, FileOpener *opener_p,
-                             BufferedCSVReaderOptions options_p, const vector<LogicalType> &requested_types)
-    : fs(fs_p), allocator(allocator), opener(opener_p), options(move(options_p)) {
-}
-
-BaseCSVReader::BaseCSVReader(ClientContext &context, BufferedCSVReaderOptions options_p,
+BaseCSVReader::BaseCSVReader(ClientContext &context_p, BufferedCSVReaderOptions options_p,
                              const vector<LogicalType> &requested_types)
-    : BaseCSVReader(FileSystem::GetFileSystem(context), Allocator::Get(context), FileSystem::GetFileOpener(context),
-                    move(options_p), requested_types) {
+    : context(context_p), fs(FileSystem::GetFileSystem(context)), allocator(Allocator::Get(context)),
+      opener(FileSystem::GetFileOpener(context)), options(std::move(options_p)) {
 }
 
 BaseCSVReader::~BaseCSVReader() {
@@ -45,7 +42,7 @@ BaseCSVReader::~BaseCSVReader() {
 unique_ptr<CSVFileHandle> BaseCSVReader::OpenCSV(const BufferedCSVReaderOptions &options_p) {
 	auto file_handle = fs.OpenFile(options_p.file_path.c_str(), FileFlags::FILE_FLAGS_READ, FileLockType::NO_LOCK,
 	                               options_p.compression, this->opener);
-	return make_unique<CSVFileHandle>(move(file_handle));
+	return make_unique<CSVFileHandle>(std::move(file_handle));
 }
 
 void BaseCSVReader::InitParseChunk(idx_t num_cols) {
@@ -77,6 +74,52 @@ void BaseCSVReader::SetDateFormat(const string &format_specifier, const LogicalT
 	StrTimeFormat::ParseFormatSpecifier(date_format.format_specifier, date_format);
 }
 
+struct TryCastDecimalOperator {
+	template <class OP, class T>
+	static bool Operation(string_t input, uint8_t width, uint8_t scale) {
+		T result;
+		string error_message;
+		return OP::Operation(input, result, &error_message, width, scale);
+	}
+};
+
+struct TryCastFloatingOperator {
+	template <class OP, class T>
+	static bool Operation(string_t input) {
+		T result;
+		string error_message;
+		return OP::Operation(input, result, &error_message);
+	}
+};
+
+bool TryCastDecimalValueCommaSeparated(const string_t &value_str, const LogicalType &sql_type) {
+	auto width = DecimalType::GetWidth(sql_type);
+	auto scale = DecimalType::GetScale(sql_type);
+	switch (sql_type.InternalType()) {
+	case PhysicalType::INT16:
+		return TryCastDecimalOperator::Operation<TryCastToDecimalCommaSeparated, int16_t>(value_str, width, scale);
+	case PhysicalType::INT32:
+		return TryCastDecimalOperator::Operation<TryCastToDecimalCommaSeparated, int32_t>(value_str, width, scale);
+	case PhysicalType::INT64:
+		return TryCastDecimalOperator::Operation<TryCastToDecimalCommaSeparated, int64_t>(value_str, width, scale);
+	case PhysicalType::INT128:
+		return TryCastDecimalOperator::Operation<TryCastToDecimalCommaSeparated, hugeint_t>(value_str, width, scale);
+	default:
+		throw InternalException("Unimplemented physical type for decimal");
+	}
+}
+
+bool TryCastFloatingValueCommaSeparated(const string_t &value_str, const LogicalType &sql_type) {
+	switch (sql_type.InternalType()) {
+	case PhysicalType::DOUBLE:
+		return TryCastFloatingOperator::Operation<TryCastErrorMessageCommaSeparated, double>(value_str);
+	case PhysicalType::FLOAT:
+		return TryCastFloatingOperator::Operation<TryCastErrorMessageCommaSeparated, float>(value_str);
+	default:
+		throw InternalException("Unimplemented physical type for floating");
+	}
+}
+
 bool BaseCSVReader::TryCastValue(const Value &value, const LogicalType &sql_type) {
 	if (options.has_format[LogicalTypeId::DATE] && sql_type.id() == LogicalTypeId::DATE) {
 		date_t result;
@@ -88,10 +131,15 @@ bool BaseCSVReader::TryCastValue(const Value &value, const LogicalType &sql_type
 		string error_message;
 		return options.date_format[LogicalTypeId::TIMESTAMP].TryParseTimestamp(string_t(StringValue::Get(value)),
 		                                                                       result, error_message);
+	} else if (options.decimal_separator != "." && sql_type.id() == LogicalTypeId::DECIMAL) {
+		return TryCastDecimalValueCommaSeparated(string_t(StringValue::Get(value)), sql_type);
+	} else if (options.decimal_separator != "." &&
+	           ((sql_type.id() == LogicalTypeId::FLOAT) || (sql_type.id() == LogicalTypeId::DOUBLE))) {
+		return TryCastFloatingValueCommaSeparated(string_t(StringValue::Get(value)), sql_type);
 	} else {
 		Value new_value;
 		string error_message;
-		return value.DefaultTryCastAs(sql_type, new_value, &error_message, true);
+		return value.TryCastAs(context, sql_type, new_value, &error_message, true);
 	}
 }
 
@@ -135,6 +183,36 @@ bool TryCastTimestampVector(BufferedCSVReaderOptions &options, Vector &input_vec
 	                                                                         count, error_message);
 }
 
+template <class OP, class T>
+bool TemplatedTryCastFloatingVector(BufferedCSVReaderOptions &options, Vector &input_vector, Vector &result_vector,
+                                    idx_t count, string &error_message) {
+	D_ASSERT(input_vector.GetType().id() == LogicalTypeId::VARCHAR);
+	bool all_converted = true;
+	UnaryExecutor::Execute<string_t, T>(input_vector, result_vector, count, [&](string_t input) {
+		T result;
+		if (!OP::Operation(input, result, &error_message)) {
+			all_converted = false;
+		}
+		return result;
+	});
+	return all_converted;
+}
+
+template <class OP, class T>
+bool TemplatedTryCastDecimalVector(BufferedCSVReaderOptions &options, Vector &input_vector, Vector &result_vector,
+                                   idx_t count, string &error_message, uint8_t width, uint8_t scale) {
+	D_ASSERT(input_vector.GetType().id() == LogicalTypeId::VARCHAR);
+	bool all_converted = true;
+	UnaryExecutor::Execute<string_t, T>(input_vector, result_vector, count, [&](string_t input) {
+		T result;
+		if (!OP::Operation(input, result, &error_message, width, scale)) {
+			all_converted = false;
+		}
+		return result;
+	});
+	return all_converted;
+}
+
 bool BaseCSVReader::TryCastVector(Vector &parse_chunk_col, idx_t size, const LogicalType &sql_type) {
 	// try vector-cast from string to sql_type
 	Vector dummy_result(sql_type);
@@ -160,7 +238,7 @@ void BaseCSVReader::AddValue(string_t str_val, idx_t &column, vector<idx_t> &esc
 	} else {
 		row_empty = false;
 	}
-	if (!sql_types.empty() && column == sql_types.size() && length == 0) {
+	if (!return_types.empty() && column == return_types.size() && length == 0) {
 		// skip a single trailing delimiter in last column
 		return;
 	}
@@ -168,14 +246,14 @@ void BaseCSVReader::AddValue(string_t str_val, idx_t &column, vector<idx_t> &esc
 		column++;
 		return;
 	}
-	if (column >= sql_types.size()) {
+	if (column >= return_types.size()) {
 		if (options.ignore_errors) {
 			error_column_overflow = true;
 			return;
 		} else {
 			throw InvalidInputException(
 			    "Error in file \"%s\", on line %s: expected %lld values per row, but got more. (%s)", options.file_path,
-			    GetLineNumberStr(linenr, linenr_estimated).c_str(), sql_types.size(), options.ToString());
+			    GetLineNumberStr(linenr, linenr_estimated).c_str(), return_types.size(), options.ToString());
 		}
 	}
 
@@ -183,7 +261,7 @@ void BaseCSVReader::AddValue(string_t str_val, idx_t &column, vector<idx_t> &esc
 	idx_t row_entry = parse_chunk.size();
 
 	// test against null string, but only if the value was not quoted
-	if ((!has_quotes || sql_types[column].id() != LogicalTypeId::VARCHAR) && !options.force_not_null[column] &&
+	if ((!has_quotes || return_types[column].id() != LogicalTypeId::VARCHAR) && !options.force_not_null[column] &&
 	    Equals::Operation(str_val, string_t(options.null_str))) {
 		FlatVector::SetNull(parse_chunk.data[column], row_entry, true);
 	} else {
@@ -221,7 +299,7 @@ bool BaseCSVReader::AddRow(DataChunk &insert_chunk, idx_t &column, string &error
 
 	if (row_empty) {
 		row_empty = false;
-		if (sql_types.size() != 1) {
+		if (return_types.size() != 1) {
 			if (mode == ParserMode::PARSING) {
 				FlatVector::SetNull(parse_chunk.data[0], parse_chunk.size(), false);
 			}
@@ -238,7 +316,7 @@ bool BaseCSVReader::AddRow(DataChunk &insert_chunk, idx_t &column, string &error
 		return false;
 	}
 
-	if (column < sql_types.size() && mode != ParserMode::SNIFFING_DIALECT) {
+	if (column < return_types.size() && mode != ParserMode::SNIFFING_DIALECT) {
 		if (options.ignore_errors) {
 			column = 0;
 			return false;
@@ -249,7 +327,7 @@ bool BaseCSVReader::AddRow(DataChunk &insert_chunk, idx_t &column, string &error
 			} else {
 				throw InvalidInputException(
 				    "Error in file \"%s\" on line %s: expected %lld values per row, but got %d.\nParser options:\n%s",
-				    options.file_path, GetLineNumberStr(linenr, linenr_estimated).c_str(), sql_types.size(), column,
+				    options.file_path, GetLineNumberStr(linenr, linenr_estimated).c_str(), return_types.size(), column,
 				    options.ToString());
 			}
 		}
@@ -282,13 +360,6 @@ bool BaseCSVReader::AddRow(DataChunk &insert_chunk, idx_t &column, string &error
 	return false;
 }
 
-void BaseCSVReader::SetNullUnionCols(DataChunk &insert_chunk) {
-	for (idx_t col = 0; col < insert_nulls_idx.size(); ++col) {
-		insert_chunk.data[insert_nulls_idx[col]].SetVectorType(VectorType::CONSTANT_VECTOR);
-		ConstantVector::SetNull(insert_chunk.data[insert_nulls_idx[col]], true);
-	}
-}
-
 void BaseCSVReader::VerifyUTF8(idx_t col_idx, idx_t row_idx, DataChunk &chunk, int64_t offset) {
 	D_ASSERT(col_idx < chunk.data.size());
 	D_ASSERT(row_idx < chunk.size());
@@ -302,8 +373,8 @@ void BaseCSVReader::VerifyUTF8(idx_t col_idx, idx_t row_idx, DataChunk &chunk, i
 	auto utf_type = Utf8Proc::Analyze(s.GetDataUnsafe(), s.GetSize());
 	if (utf_type == UnicodeType::INVALID) {
 		string col_name = to_string(col_idx);
-		if (col_idx < col_names.size()) {
-			col_name = "\"" + col_names[col_idx] + "\"";
+		if (col_idx < names.size()) {
+			col_name = "\"" + names[col_idx] + "\"";
 		}
 		int64_t error_line = linenr - (chunk.size() - row_idx) + 1 + offset;
 		D_ASSERT(error_line >= 0);
@@ -321,6 +392,48 @@ void BaseCSVReader::VerifyUTF8(idx_t col_idx) {
 	}
 }
 
+bool TryCastDecimalVectorCommaSeparated(BufferedCSVReaderOptions &options, Vector &input_vector, Vector &result_vector,
+                                        idx_t count, string &error_message, LogicalType &result_type) {
+	auto width = DecimalType::GetWidth(result_type);
+	auto scale = DecimalType::GetScale(result_type);
+	switch (result_type.InternalType()) {
+	case PhysicalType::INT16:
+		return TemplatedTryCastDecimalVector<TryCastToDecimalCommaSeparated, int16_t>(
+		    options, input_vector, result_vector, count, error_message, width, scale);
+		break;
+	case PhysicalType::INT32:
+		return TemplatedTryCastDecimalVector<TryCastToDecimalCommaSeparated, int32_t>(
+		    options, input_vector, result_vector, count, error_message, width, scale);
+		break;
+	case PhysicalType::INT64:
+		return TemplatedTryCastDecimalVector<TryCastToDecimalCommaSeparated, int64_t>(
+		    options, input_vector, result_vector, count, error_message, width, scale);
+		break;
+	case PhysicalType::INT128:
+		return TemplatedTryCastDecimalVector<TryCastToDecimalCommaSeparated, hugeint_t>(
+		    options, input_vector, result_vector, count, error_message, width, scale);
+		break;
+	default:
+		throw InternalException("Unimplemented physical type for decimal");
+	}
+}
+
+bool TryCastFloatingVectorCommaSeparated(BufferedCSVReaderOptions &options, Vector &input_vector, Vector &result_vector,
+                                         idx_t count, string &error_message, LogicalType &result_type) {
+	switch (result_type.InternalType()) {
+	case PhysicalType::DOUBLE:
+		return TemplatedTryCastFloatingVector<TryCastErrorMessageCommaSeparated, double>(
+		    options, input_vector, result_vector, count, error_message);
+		break;
+	case PhysicalType::FLOAT:
+		return TemplatedTryCastFloatingVector<TryCastErrorMessageCommaSeparated, float>(
+		    options, input_vector, result_vector, count, error_message);
+		break;
+	default:
+		throw InternalException("Unimplemented physical type for floating");
+	}
+}
+
 bool BaseCSVReader::Flush(DataChunk &insert_chunk, bool try_add_line) {
 	if (parse_chunk.size() == 0) {
 		return true;
@@ -330,14 +443,16 @@ bool BaseCSVReader::Flush(DataChunk &insert_chunk, bool try_add_line) {
 
 	// convert the columns in the parsed chunk to the types of the table
 	insert_chunk.SetCardinality(parse_chunk);
-	for (idx_t col_idx = 0; col_idx < sql_types.size(); col_idx++) {
+	for (idx_t col_idx = 0; col_idx < return_types.size(); col_idx++) {
 		auto insert_idx = insert_cols_idx[col_idx];
-		auto &type = sql_types[col_idx];
+		auto &type = return_types[col_idx];
 		if (type.id() == LogicalTypeId::VARCHAR) {
 			// target type is varchar: no need to convert
 			// just test that all strings are valid utf-8 strings
 			VerifyUTF8(col_idx);
-			insert_chunk.data[insert_idx].Reference(parse_chunk.data[col_idx]);
+			// reinterpret rather than reference so we can deal with user-defined types
+			insert_chunk.data[insert_idx].Reinterpret(parse_chunk.data[col_idx]);
+
 		} else {
 			string error_message;
 			bool success;
@@ -345,14 +460,24 @@ bool BaseCSVReader::Flush(DataChunk &insert_chunk, bool try_add_line) {
 				// use the date format to cast the chunk
 				success = TryCastDateVector(options, parse_chunk.data[col_idx], insert_chunk.data[insert_idx],
 				                            parse_chunk.size(), error_message);
-			} else if (options.has_format[LogicalTypeId::TIMESTAMP] && type.id() == LogicalTypeId::TIMESTAMP) {
+			} else if (options.has_format[LogicalTypeId::TIMESTAMP] &&
+			           return_types[col_idx].id() == LogicalTypeId::TIMESTAMP) {
 				// use the date format to cast the chunk
 				success = TryCastTimestampVector(options, parse_chunk.data[col_idx], insert_chunk.data[insert_idx],
 				                                 parse_chunk.size(), error_message);
+			} else if (options.decimal_separator != "." && (return_types[col_idx].id() == LogicalTypeId::FLOAT ||
+			                                                return_types[col_idx].id() == LogicalTypeId::DOUBLE)) {
+				success = TryCastFloatingVectorCommaSeparated(options, parse_chunk.data[col_idx],
+				                                              insert_chunk.data[insert_idx], parse_chunk.size(),
+				                                              error_message, return_types[col_idx]);
+			} else if (options.decimal_separator != "." && return_types[col_idx].id() == LogicalTypeId::DECIMAL) {
+				success = TryCastDecimalVectorCommaSeparated(options, parse_chunk.data[col_idx],
+				                                             insert_chunk.data[insert_idx], parse_chunk.size(),
+				                                             error_message, return_types[col_idx]);
 			} else {
 				// target type is not varchar: perform a cast
-				success = VectorOperations::DefaultTryCast(parse_chunk.data[col_idx], insert_chunk.data[insert_idx],
-				                                           parse_chunk.size(), &error_message);
+				success = VectorOperations::TryCast(context, parse_chunk.data[col_idx], insert_chunk.data[insert_idx],
+				                                    parse_chunk.size(), &error_message);
 			}
 			if (success) {
 				continue;
@@ -365,8 +490,8 @@ bool BaseCSVReader::Flush(DataChunk &insert_chunk, bool try_add_line) {
 				continue;
 			}
 			string col_name = to_string(col_idx);
-			if (col_idx < col_names.size()) {
-				col_name = "\"" + col_names[col_idx] + "\"";
+			if (col_idx < names.size()) {
+				col_name = "\"" + names[col_idx] + "\"";
 			}
 
 			// figure out the exact line number
@@ -401,7 +526,7 @@ bool BaseCSVReader::Flush(DataChunk &insert_chunk, bool try_add_line) {
 
 		for (idx_t row_idx = 0; row_idx < parse_chunk.size(); row_idx++) {
 			bool failed = false;
-			for (idx_t column_idx = 0; column_idx < sql_types.size(); column_idx++) {
+			for (idx_t column_idx = 0; column_idx < return_types.size(); column_idx++) {
 
 				auto &inserted_column = insert_chunk.data[column_idx];
 				auto &parsed_column = parse_chunk.data[column_idx];
@@ -421,4 +546,32 @@ bool BaseCSVReader::Flush(DataChunk &insert_chunk, bool try_add_line) {
 	parse_chunk.Reset();
 	return true;
 }
+
+void BaseCSVReader::SetNewLineDelimiter(bool carry, bool carry_followed_by_nl) {
+	if ((mode == ParserMode::SNIFFING_DIALECT && !options.has_newline) ||
+	    options.new_line == NewLineIdentifier::NOT_SET) {
+		if (options.new_line == NewLineIdentifier::MIX) {
+			return;
+		}
+		NewLineIdentifier this_line_identifier;
+		if (carry) {
+			if (carry_followed_by_nl) {
+				this_line_identifier = NewLineIdentifier::CARRY_ON;
+			} else {
+				this_line_identifier = NewLineIdentifier::SINGLE;
+			}
+		} else {
+			this_line_identifier = NewLineIdentifier::SINGLE;
+		}
+		if (options.new_line == NewLineIdentifier::NOT_SET) {
+			options.new_line = this_line_identifier;
+			return;
+		}
+		if (options.new_line != this_line_identifier) {
+			options.new_line = NewLineIdentifier::MIX;
+			return;
+		}
+		options.new_line = this_line_identifier;
+	}
+}
 } // namespace duckdb

package/src/duckdb/src/execution/operator/persistent/buffered_csv_reader.cpp

@@ -23,18 +23,19 @@
 
 namespace duckdb {
 
-BufferedCSVReader::BufferedCSVReader(FileSystem &fs_p, Allocator &allocator, FileOpener *opener_p,
-                                     BufferedCSVReaderOptions options_p, const vector<LogicalType> &requested_types)
-    : BaseCSVReader(fs_p, allocator, opener_p, move(options_p), requested_types), buffer_size(0), position(0),
-      start(0) {
+BufferedCSVReader::BufferedCSVReader(ClientContext &context, BufferedCSVReaderOptions options_p,
+                                     const vector<LogicalType> &requested_types)
+    : BaseCSVReader(context, std::move(options_p), requested_types), buffer_size(0), position(0), start(0) {
 	file_handle = OpenCSV(options);
 	Initialize(requested_types);
 }
 
-BufferedCSVReader::BufferedCSVReader(ClientContext &context, BufferedCSVReaderOptions options_p,
+BufferedCSVReader::BufferedCSVReader(ClientContext &context, string filename, BufferedCSVReaderOptions options_p,
                                      const vector<LogicalType> &requested_types)
-    : BufferedCSVReader(FileSystem::GetFileSystem(context), Allocator::Get(context), FileSystem::GetFileOpener(context),
-                        move(options_p), requested_types) {
+    : BaseCSVReader(context, std::move(options_p), requested_types), buffer_size(0), position(0), start(0) {
+	options.file_path = std::move(filename);
+	file_handle = OpenCSV(options);
+	Initialize(requested_types);
 }
 
 BufferedCSVReader::~BufferedCSVReader() {
@@ -118,7 +119,7 @@ TextSearchShiftArray::TextSearchShiftArray() {
 
 TextSearchShiftArray::TextSearchShiftArray(string search_term) : length(search_term.size()) {
 	if (length > 255) {
-		throw Exception("Size of delimiter/quote/escape in CSV reader is limited to 255 bytes");
+		throw InvalidInputException("Size of delimiter/quote/escape in CSV reader is limited to 255 bytes");
 	}
 	// initialize the shifts array
 	shifts = unique_ptr<uint8_t[]>(new uint8_t[length * 255]);
@@ -236,20 +237,20 @@ static string NormalizeColumnName(const string &col_name) {
 void BufferedCSVReader::Initialize(const vector<LogicalType> &requested_types) {
 	PrepareComplexParser();
 	if (options.auto_detect) {
-		sql_types = SniffCSV(requested_types);
-		if (sql_types.empty()) {
-			throw Exception("Failed to detect column types from CSV: is the file a valid CSV file?");
+		return_types = SniffCSV(requested_types);
+		if (return_types.empty()) {
+			throw InvalidInputException("Failed to detect column types from CSV: is the file a valid CSV file?");
 		}
 		if (cached_chunks.empty()) {
 			JumpToBeginning(options.skip_rows, options.header);
 		}
 	} else {
-		sql_types = requested_types;
+		return_types = requested_types;
 		ResetBuffer();
 		SkipRowsAndReadHeader(options.skip_rows, options.header);
 	}
-	InitParseChunk(sql_types.size());
-	InitInsertChunkIdx(sql_types.size());
+	InitParseChunk(return_types.size());
+	InitInsertChunkIdx(return_types.size());
 	// we only need reset support during the automatic CSV type detection
 	// since reset support might require caching (in the case of streams), we disable it for the remainder
 	file_handle->DisableReset();
@@ -297,7 +298,7 @@ void BufferedCSVReader::SkipRowsAndReadHeader(idx_t skip_rows, bool skip_header)
 
 	if (skip_header) {
 		// ignore the first line as a header line
-		InitParseChunk(sql_types.size());
+		InitParseChunk(return_types.size());
 		ParseCSV(ParserMode::PARSING_HEADER);
 	}
 }
@@ -434,13 +435,7 @@ void BufferedCSVReader::DetectDialect(const vector<LogicalType> &requested_types
 
 					JumpToBeginning(original_options.skip_rows);
 					sniffed_column_counts.clear();
-					idx_t num_buffers = 0;
-					bool parsing_success = true;
-					while (num_buffers < options.sample_chunks && !end_of_file_reached) {
-						parsing_success = parsing_success && TryParseCSV(ParserMode::SNIFFING_DIALECT);
-						num_buffers++;
-					}
-					if (!parsing_success) {
+					if (!TryParseCSV(ParserMode::SNIFFING_DIALECT)) {
 						continue;
 					}
 
@@ -472,6 +467,7 @@ void BufferedCSVReader::DetectDialect(const vector<LogicalType> &requested_types
 					} else if ((more_values || single_column_before) && rows_consistent) {
 						sniff_info.skip_rows = start_row;
 						sniff_info.num_cols = num_cols;
+						sniff_info.new_line = options.new_line;
 						best_consistent_rows = consistent_rows;
 						best_num_cols = num_cols;
 
@@ -487,6 +483,7 @@ void BufferedCSVReader::DetectDialect(const vector<LogicalType> &requested_types
 						if (!same_quote_is_candidate) {
 							sniff_info.skip_rows = start_row;
 							sniff_info.num_cols = num_cols;
+							sniff_info.new_line = options.new_line;
 							info_candidates.push_back(sniff_info);
 						}
 					}
@@ -520,14 +517,14 @@ void BufferedCSVReader::DetectCandidateTypes(const vector<LogicalType> &type_can
 			format_candidates[t.first].clear();
 		}
 
-		// set all sql_types to VARCHAR so we can do datatype detection based on VARCHAR values
-		sql_types.clear();
-		sql_types.assign(options.num_cols, LogicalType::VARCHAR);
+		// set all return_types to VARCHAR so we can do datatype detection based on VARCHAR values
+		return_types.clear();
+		return_types.assign(options.num_cols, LogicalType::VARCHAR);
 
 		// jump to beginning and skip potential header
 		JumpToBeginning(options.skip_rows, true);
 		DataChunk header_row;
-		header_row.Initialize(allocator, sql_types);
+		header_row.Initialize(allocator, return_types);
 		parse_chunk.Copy(header_row);
 
 		if (header_row.size() == 0) {
@@ -535,7 +532,7 @@ void BufferedCSVReader::DetectCandidateTypes(const vector<LogicalType> &type_can
 		}
 
 		// init parse chunk and read csv with info candidate
-		InitParseChunk(sql_types.size());
+		InitParseChunk(return_types.size());
 		if (!TryParseCSV(ParserMode::SNIFFING_DATATYPES)) {
 			continue;
 		}
@@ -713,7 +710,7 @@ void BufferedCSVReader::DetectHeader(const vector<vector<LogicalType>> &best_sql
 				col_name = col_name + "_" + to_string(name_collision_count[col_name]);
 			}
 
-			col_names.push_back(col_name);
+			names.push_back(col_name);
 			name_collision_count[col_name] = 0;
 		}
 
@@ -721,7 +718,7 @@ void BufferedCSVReader::DetectHeader(const vector<vector<LogicalType>> &best_sql
 		options.header = false;
 		for (idx_t col = 0; col < options.num_cols; col++) {
 			string column_name = GenerateColumnName(options.num_cols, col);
-			col_names.push_back(column_name);
+			names.push_back(column_name);
 		}
 	}
 }
@@ -731,8 +728,8 @@ vector<LogicalType> BufferedCSVReader::RefineTypeDetection(const vector<LogicalT
                                                            vector<vector<LogicalType>> &best_sql_types_candidates,
                                                            map<LogicalTypeId, vector<string>> &best_format_candidates) {
 	// for the type refine we set the SQL types to VARCHAR for all columns
-	sql_types.clear();
-	sql_types.assign(options.num_cols, LogicalType::VARCHAR);
+	return_types.clear();
+	return_types.assign(options.num_cols, LogicalType::VARCHAR);
 
 	vector<LogicalType> detected_types;
 
@@ -747,11 +744,11 @@ vector<LogicalType> BufferedCSVReader::RefineTypeDetection(const vector<LogicalT
 		}
 	} else if (options.all_varchar) {
 		// return all types varchar
-		detected_types = sql_types;
+		detected_types = return_types;
 	} else {
 		// jump through the rest of the file and continue to refine the sql type guess
 		while (JumpToNextSample()) {
-			InitParseChunk(sql_types.size());
+			InitParseChunk(return_types.size());
 			// if jump ends up a bad line, we just skip this chunk
 			if (!TryParseCSV(ParserMode::SNIFFING_DATATYPES)) {
 				continue;
@@ -800,7 +797,7 @@ vector<LogicalType> BufferedCSVReader::RefineTypeDetection(const vector<LogicalT
 					auto chunk = make_unique<DataChunk>();
 					auto parse_chunk_types = parse_chunk.GetTypes();
 					chunk->Move(parse_chunk);
-					cached_chunks.push(move(chunk));
+					cached_chunks.push(std::move(chunk));
 				} else {
 					while (!cached_chunks.empty()) {
 						cached_chunks.pop();
@@ -822,6 +819,27 @@ vector<LogicalType> BufferedCSVReader::RefineTypeDetection(const vector<LogicalT
 	return detected_types;
 }
 
+string BufferedCSVReader::ColumnTypesError(case_insensitive_map_t<idx_t> sql_types_per_column,
+                                           const vector<string> &names) {
+	for (idx_t i = 0; i < names.size(); i++) {
+		auto it = sql_types_per_column.find(names[i]);
+		if (it != sql_types_per_column.end()) {
+			sql_types_per_column.erase(names[i]);
+			continue;
+		}
+	}
+	if (sql_types_per_column.empty()) {
+		return string();
+	}
+	string exception = "COLUMN_TYPES error: Columns with names: ";
+	for (auto &col : sql_types_per_column) {
+		exception += "\"" + col.first + "\",";
+	}
+	exception.pop_back();
+	exception += " do not exist in the CSV File";
+	return exception;
+}
+
 vector<LogicalType> BufferedCSVReader::SniffCSV(const vector<LogicalType> &requested_types) {
 	for (auto &type : requested_types) {
 		// auto detect for blobs not supported: there may be invalid UTF-8 in the file
@@ -877,23 +895,38 @@ vector<LogicalType> BufferedCSVReader::SniffCSV(const vector<LogicalType> &reque
 	// #######
 	options.num_cols = best_num_cols;
 	DetectHeader(best_sql_types_candidates, best_header_row);
-	auto sql_types_per_column = options.sql_types_per_column;
-	for (idx_t i = 0; i < col_names.size(); i++) {
-		auto it = sql_types_per_column.find(col_names[i]);
-		if (it != sql_types_per_column.end()) {
-			best_sql_types_candidates[i] = {it->second};
-			sql_types_per_column.erase(col_names[i]);
-		}
-	}
-	if (!sql_types_per_column.empty()) {
-		string exception = "COLUMN_TYPES error: Columns with names: ";
-		for (auto &col : sql_types_per_column) {
-			exception += "\"" + col.first + "\",";
+	if (!options.sql_type_list.empty()) {
+		// user-defined types were supplied for certain columns
+		// override the types
+		if (!options.sql_types_per_column.empty()) {
+			// types supplied as name -> value map
+			idx_t found = 0;
+			for (idx_t i = 0; i < names.size(); i++) {
+				auto it = options.sql_types_per_column.find(names[i]);
+				if (it != options.sql_types_per_column.end()) {
+					best_sql_types_candidates[i] = {options.sql_type_list[it->second]};
+					found++;
+					continue;
+				}
+			}
+			if (!options.union_by_name && found < options.sql_types_per_column.size()) {
+				string exception = ColumnTypesError(options.sql_types_per_column, names);
+				if (!exception.empty()) {
+					throw BinderException(exception);
+				}
+			}
+		} else {
+			// types supplied as list
+			if (names.size() < options.sql_type_list.size()) {
+				throw BinderException("read_csv: %d types were provided, but CSV file only has %d columns",
+				                      options.sql_type_list.size(), names.size());
+			}
+			for (idx_t i = 0; i < options.sql_type_list.size(); i++) {
+				best_sql_types_candidates[i] = {options.sql_type_list[i]};
+			}
 		}
-		exception.pop_back();
-		exception += " do not exist in the CSV File";
-		throw BinderException(exception);
 	}
+
 	// #######
 	// ### type detection (refining)
 	// #######
@@ -1218,6 +1251,7 @@ add_row : {
 		// \r newline, go to special state that parses an optional \n afterwards
 		goto carriage_return;
 	} else {
+		SetNewLineDelimiter();
 		// \n newline, move to value start
 		if (finished_chunk) {
 			return true;
@@ -1296,6 +1330,7 @@ carriage_return:
 	/* state: carriage_return */
 	// this stage optionally skips a newline (\n) character, which allows \r\n to be interpreted as a single line
 	if (buffer[position] == '\n') {
+		SetNewLineDelimiter(true, true);
 		// newline after carriage return: skip
 		// increase position by 1 and move start to the new position
 		start = ++position;
@@ -1303,6 +1338,8 @@ carriage_return:
 			// file ends right after delimiter, go to final state
 			goto final_state;
 		}
+	} else {
+		SetNewLineDelimiter(true, false);
 	}
 	if (finished_chunk) {
 		return true;
@@ -1333,7 +1370,7 @@ final_state:
 }
 
 bool BufferedCSVReader::ReadBuffer(idx_t &start) {
-	auto old_buffer = move(buffer);
+	auto old_buffer = std::move(buffer);
 
 	// the remaining part of the last buffer
 	idx_t remaining = buffer_size - start;
@@ -1363,7 +1400,7 @@ bool BufferedCSVReader::ReadBuffer(idx_t &start) {
 	buffer_size = remaining + read_count;
 	buffer[buffer_size] = '\0';
 	if (old_buffer) {
-		cached_buffers.push_back(move(old_buffer));
+		cached_buffers.push_back(std::move(old_buffer));
 	}
 	start = 0;
 	position = remaining;