duckdb 0.6.2-dev971.0 → 0.7.1-dev2.0

package/src/duckdb/src/execution/operator/join/physical_positional_join.cpp

@@ -0,0 +1,195 @@
+#include "duckdb/execution/operator/join/physical_positional_join.hpp"
+
+#include "duckdb/common/types/column_data_collection.hpp"
+#include "duckdb/common/vector_operations/vector_operations.hpp"
+#include "duckdb/execution/operator/join/physical_join.hpp"
+
+namespace duckdb {
+
+PhysicalPositionalJoin::PhysicalPositionalJoin(vector<LogicalType> types, unique_ptr<PhysicalOperator> left,
+                                               unique_ptr<PhysicalOperator> right, idx_t estimated_cardinality)
+    : PhysicalOperator(PhysicalOperatorType::POSITIONAL_JOIN, std::move(types), estimated_cardinality) {
+	children.push_back(std::move(left));
+	children.push_back(std::move(right));
+}
+
+//===--------------------------------------------------------------------===//
+// Sink
+//===--------------------------------------------------------------------===//
+class PositionalJoinGlobalState : public GlobalSinkState {
+public:
+	explicit PositionalJoinGlobalState(ClientContext &context, const PhysicalPositionalJoin &op)
+	    : rhs(context, op.children[1]->GetTypes()), initialized(false), source_offset(0), exhausted(false) {
+		rhs.InitializeAppend(append_state);
+	}
+
+	ColumnDataCollection rhs;
+	ColumnDataAppendState append_state;
+	mutex rhs_lock;
+
+	bool initialized;
+	ColumnDataScanState scan_state;
+	DataChunk source;
+	idx_t source_offset;
+	bool exhausted;
+
+	void InitializeScan();
+	idx_t Refill();
+	idx_t CopyData(DataChunk &output, const idx_t count, const idx_t col_offset);
+	void Execute(DataChunk &input, DataChunk &output);
+	void GetData(DataChunk &output);
+};
+
+unique_ptr<GlobalSinkState> PhysicalPositionalJoin::GetGlobalSinkState(ClientContext &context) const {
+	return make_unique<PositionalJoinGlobalState>(context, *this);
+}
+
+SinkResultType PhysicalPositionalJoin::Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate_p,
+                                            DataChunk &input) const {
+	auto &sink = (PositionalJoinGlobalState &)state;
+	lock_guard<mutex> client_guard(sink.rhs_lock);
+	sink.rhs.Append(sink.append_state, input);
+	return SinkResultType::NEED_MORE_INPUT;
+}
+
+//===--------------------------------------------------------------------===//
+// Operator
+//===--------------------------------------------------------------------===//
+void PositionalJoinGlobalState::InitializeScan() {
+	if (!initialized) {
+		// not initialized yet: initialize the scan
+		initialized = true;
+		rhs.InitializeScanChunk(source);
+		rhs.InitializeScan(scan_state);
+	}
+}
+
+idx_t PositionalJoinGlobalState::Refill() {
+	if (source_offset >= source.size()) {
+		if (!exhausted) {
+			source.Reset();
+			rhs.Scan(scan_state, source);
+		}
+		source_offset = 0;
+	}
+
+	const auto available = source.size() - source_offset;
+	if (!available) {
+		if (!exhausted) {
+			source.Reset();
+			for (idx_t i = 0; i < source.ColumnCount(); ++i) {
+				auto &vec = source.data[i];
+				vec.SetVectorType(VectorType::CONSTANT_VECTOR);
+				ConstantVector::SetNull(vec, true);
+			}
+			exhausted = true;
+		}
+	}
+
+	return available;
+}
+
+idx_t PositionalJoinGlobalState::CopyData(DataChunk &output, const idx_t count, const idx_t col_offset) {
+	if (!source_offset && (source.size() >= count || exhausted)) {
+		//	Fast track: aligned and has enough data
+		for (idx_t i = 0; i < source.ColumnCount(); ++i) {
+			output.data[col_offset + i].Reference(source.data[i]);
+		}
+		source_offset += count;
+	} else {
+		// Copy data
+		for (idx_t target_offset = 0; target_offset < count;) {
+			const auto needed = count - target_offset;
+			const auto available = exhausted ? needed : (source.size() - source_offset);
+			const auto copy_size = MinValue(needed, available);
+			const auto source_count = source_offset + copy_size;
+			for (idx_t i = 0; i < source.ColumnCount(); ++i) {
+				VectorOperations::Copy(source.data[i], output.data[col_offset + i], source_count, source_offset,
+				                       target_offset);
+			}
+			target_offset += copy_size;
+			source_offset += copy_size;
+			Refill();
+		}
+	}
+
+	return source.ColumnCount();
+}
+
+void PositionalJoinGlobalState::Execute(DataChunk &input, DataChunk &output) {
+	lock_guard<mutex> client_guard(rhs_lock);
+
+	// Reference the input and assume it will be full
+	const auto col_offset = input.ColumnCount();
+	for (idx_t i = 0; i < col_offset; ++i) {
+		output.data[i].Reference(input.data[i]);
+	}
+
+	// Copy or reference the RHS columns
+	const auto count = input.size();
+	InitializeScan();
+	Refill();
+	CopyData(output, count, col_offset);
+
+	output.SetCardinality(count);
+}
+
+OperatorResultType PhysicalPositionalJoin::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
+                                                   GlobalOperatorState &gstate, OperatorState &state_p) const {
+	auto &sink = (PositionalJoinGlobalState &)*sink_state;
+	sink.Execute(input, chunk);
+	return OperatorResultType::NEED_MORE_INPUT;
+}
+
+//===--------------------------------------------------------------------===//
+// Source
+//===--------------------------------------------------------------------===//
+void PositionalJoinGlobalState::GetData(DataChunk &output) {
+	lock_guard<mutex> client_guard(rhs_lock);
+
+	InitializeScan();
+	Refill();
+
+	//	LHS exhausted
+	if (exhausted) {
+		//	RHS exhausted too, so we are done
+		output.SetCardinality(0);
+		return;
+	}
+
+	//	LHS is all NULL
+	const auto col_offset = output.ColumnCount() - source.ColumnCount();
+	for (idx_t i = 0; i < col_offset; ++i) {
+		auto &vec = output.data[i];
+		vec.SetVectorType(VectorType::CONSTANT_VECTOR);
+		ConstantVector::SetNull(vec, true);
+	}
+
+	//	RHS still has data, so copy it
+	const auto count = MinValue<idx_t>(STANDARD_VECTOR_SIZE, source.size() - source_offset);
+	CopyData(output, count, col_offset);
+	output.SetCardinality(count);
+}
+
+void PhysicalPositionalJoin::GetData(ExecutionContext &context, DataChunk &result, GlobalSourceState &gstate,
+                                     LocalSourceState &lstate) const {
+	auto &sink = (PositionalJoinGlobalState &)*sink_state;
+	sink.GetData(result);
+}
+
+//===--------------------------------------------------------------------===//
+// Pipeline Construction
+//===--------------------------------------------------------------------===//
+void PhysicalPositionalJoin::BuildPipelines(Pipeline &current, MetaPipeline &meta_pipeline) {
+	PhysicalJoin::BuildJoinPipelines(current, meta_pipeline, *this);
+}
+
+vector<const PhysicalOperator *> PhysicalPositionalJoin::GetSources() const {
+	auto result = children[0]->GetSources();
+	if (IsSource()) {
+		result.push_back(this);
+	}
+	return result;
+}
+
+} // namespace duckdb

package/src/duckdb/src/execution/operator/join/physical_range_join.cpp

@@ -86,7 +86,7 @@ public:
 
 public:
 	RangeJoinMergeTask(shared_ptr<Event> event_p, ClientContext &context, GlobalSortedTable &table)
-	    : ExecutorTask(context), event(move(event_p)), context(context), table(table) {
+	    : ExecutorTask(context), event(std::move(event_p)), context(context), table(table) {
 	}
 
 	TaskExecutionResult ExecuteTask(TaskExecutionMode mode) override {
@@ -128,7 +128,7 @@ public:
 		for (idx_t tnum = 0; tnum < num_threads; tnum++) {
 			iejoin_tasks.push_back(make_unique<RangeJoinMergeTask>(shared_from_this(), context, table));
 		}
-		SetTasks(move(iejoin_tasks));
+		SetTasks(std::move(iejoin_tasks));
 	}
 
 	void FinishEvent() override {
@@ -146,7 +146,7 @@ void PhysicalRangeJoin::GlobalSortedTable::ScheduleMergeTasks(Pipeline &pipeline
 	// Initialize global sort state for a round of merging
 	global_sort_state.InitializeMergeRound();
 	auto new_event = make_shared<RangeJoinMergeEvent>(*this, pipeline);
-	event.InsertEvent(move(new_event));
+	event.InsertEvent(std::move(new_event));
 }
 
 void PhysicalRangeJoin::GlobalSortedTable::Finalize(Pipeline &pipeline, Event &event) {
@@ -162,7 +162,7 @@ void PhysicalRangeJoin::GlobalSortedTable::Finalize(Pipeline &pipeline, Event &e
 PhysicalRangeJoin::PhysicalRangeJoin(LogicalOperator &op, PhysicalOperatorType type, unique_ptr<PhysicalOperator> left,
                                      unique_ptr<PhysicalOperator> right, vector<JoinCondition> cond, JoinType join_type,
                                      idx_t estimated_cardinality)
-    : PhysicalComparisonJoin(op, type, move(cond), join_type, estimated_cardinality) {
+    : PhysicalComparisonJoin(op, type, std::move(cond), join_type, estimated_cardinality) {
 	// Reorder the conditions so that ranges are at the front.
 	// TODO: use stats to improve the choice?
 	// TODO: Prefer fixed length types?
@@ -186,8 +186,8 @@ PhysicalRangeJoin::PhysicalRangeJoin(LogicalOperator &op, PhysicalOperatorType t
 		}
 	}
 
-	children.push_back(move(left));
-	children.push_back(move(right));
+	children.push_back(std::move(left));
+	children.push_back(std::move(right));
 }
 
 idx_t PhysicalRangeJoin::LocalSortedTable::MergeNulls(const vector<JoinCondition> &conditions) {
@@ -315,7 +315,7 @@ BufferHandle PhysicalRangeJoin::SliceSortedPayload(DataChunk &payload, GlobalSor
 		col.Slice(gsel, result_count);
 	}
 
-	return move(read_state.payload_heap_handle);
+	return std::move(read_state.payload_heap_handle);
 }
 
 idx_t PhysicalRangeJoin::SelectJoinTail(const ExpressionType &condition, Vector &left, Vector &right,

package/src/duckdb/src/execution/operator/order/physical_order.cpp

@@ -11,8 +11,8 @@ namespace duckdb {
 
 PhysicalOrder::PhysicalOrder(vector<LogicalType> types, vector<BoundOrderByNode> orders, vector<idx_t> projections,
                              idx_t estimated_cardinality)
-    : PhysicalOperator(PhysicalOperatorType::ORDER_BY, move(types), estimated_cardinality), orders(move(orders)),
-      projections(move(projections)) {
+    : PhysicalOperator(PhysicalOperatorType::ORDER_BY, std::move(types), estimated_cardinality),
+      orders(std::move(orders)), projections(std::move(projections)) {
 }
 
 //===--------------------------------------------------------------------===//
@@ -62,7 +62,7 @@ unique_ptr<GlobalSinkState> PhysicalOrder::GetGlobalSinkState(ClientContext &con
 	// Set external (can be force with the PRAGMA)
 	state->global_sort_state.external = ClientConfig::GetConfig(context).force_external;
 	state->memory_per_thread = GetMaxThreadMemory(context);
-	return move(state);
+	return std::move(state);
 }
 
 unique_ptr<LocalSinkState> PhysicalOrder::GetLocalSinkState(ExecutionContext &context) const {
@@ -111,7 +111,7 @@ void PhysicalOrder::Combine(ExecutionContext &context, GlobalSinkState &gstate_p
 class PhysicalOrderMergeTask : public ExecutorTask {
 public:
 	PhysicalOrderMergeTask(shared_ptr<Event> event_p, ClientContext &context, OrderGlobalSinkState &state)
-	    : ExecutorTask(context), event(move(event_p)), context(context), state(state) {
+	    : ExecutorTask(context), event(std::move(event_p)), context(context), state(state) {
 	}
 
 	TaskExecutionResult ExecuteTask(TaskExecutionMode mode) override {
@@ -149,7 +149,7 @@ public:
 		for (idx_t tnum = 0; tnum < num_threads; tnum++) {
 			merge_tasks.push_back(make_unique<PhysicalOrderMergeTask>(shared_from_this(), context, gstate));
 		}
-		SetTasks(move(merge_tasks));
+		SetTasks(std::move(merge_tasks));
 	}
 
 	void FinishEvent() override {
@@ -187,7 +187,7 @@ void PhysicalOrder::ScheduleMergeTasks(Pipeline &pipeline, Event &event, OrderGl
 	// Initialize global sort state for a round of merging
 	state.global_sort_state.InitializeMergeRound();
 	auto new_event = make_shared<OrderMergeEvent>(state, pipeline);
-	event.InsertEvent(move(new_event));
+	event.InsertEvent(std::move(new_event));
 }
 
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/execution/operator/order/physical_top_n.cpp

@@ -12,7 +12,7 @@ namespace duckdb {
 
 PhysicalTopN::PhysicalTopN(vector<LogicalType> types, vector<BoundOrderByNode> orders, idx_t limit, idx_t offset,
                            idx_t estimated_cardinality)
-    : PhysicalOperator(PhysicalOperatorType::TOP_N, move(types), estimated_cardinality), orders(move(orders)),
+    : PhysicalOperator(PhysicalOperatorType::TOP_N, std::move(types), estimated_cardinality), orders(std::move(orders)),
       limit(limit), offset(offset) {
 }
 
@@ -132,8 +132,8 @@ void TopNSortState::Sink(DataChunk &input) {
 }
 
 void TopNSortState::Move(TopNSortState &other) {
-	local_state = move(other.local_state);
-	global_state = move(other.global_state);
+	local_state = std::move(other.local_state);
+	global_state = std::move(other.global_state);
 	count = other.count;
 	is_sorted = other.is_sorted;
 }

package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp

@@ -5,6 +5,8 @@
 #include "duckdb/common/string_util.hpp"
 #include "duckdb/common/to_string.hpp"
 #include "duckdb/common/types/cast_helpers.hpp"
+#include "duckdb/common/operator/cast_operators.hpp"
+#include "duckdb/common/operator/decimal_cast_operators.hpp"
 #include "duckdb/common/vector_operations/unary_executor.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/function/scalar/strftime.hpp"
@@ -30,13 +32,13 @@ string BaseCSVReader::GetLineNumberStr(idx_t linenr, bool linenr_estimated) {
 
 BaseCSVReader::BaseCSVReader(FileSystem &fs_p, Allocator &allocator, FileOpener *opener_p,
                              BufferedCSVReaderOptions options_p, const vector<LogicalType> &requested_types)
-    : fs(fs_p), allocator(allocator), opener(opener_p), options(move(options_p)) {
+    : fs(fs_p), allocator(allocator), opener(opener_p), options(std::move(options_p)) {
 }
 
 BaseCSVReader::BaseCSVReader(ClientContext &context, BufferedCSVReaderOptions options_p,
                              const vector<LogicalType> &requested_types)
     : BaseCSVReader(FileSystem::GetFileSystem(context), Allocator::Get(context), FileSystem::GetFileOpener(context),
-                    move(options_p), requested_types) {
+                    std::move(options_p), requested_types) {
 }
 
 BaseCSVReader::~BaseCSVReader() {
@@ -45,7 +47,7 @@ BaseCSVReader::~BaseCSVReader() {
 unique_ptr<CSVFileHandle> BaseCSVReader::OpenCSV(const BufferedCSVReaderOptions &options_p) {
 	auto file_handle = fs.OpenFile(options_p.file_path.c_str(), FileFlags::FILE_FLAGS_READ, FileLockType::NO_LOCK,
 	                               options_p.compression, this->opener);
-	return make_unique<CSVFileHandle>(move(file_handle));
+	return make_unique<CSVFileHandle>(std::move(file_handle));
 }
 
 void BaseCSVReader::InitParseChunk(idx_t num_cols) {
@@ -77,6 +79,52 @@ void BaseCSVReader::SetDateFormat(const string &format_specifier, const LogicalT
 	StrTimeFormat::ParseFormatSpecifier(date_format.format_specifier, date_format);
 }
 
+struct TryCastDecimalOperator {
+	template <class OP, class T>
+	static bool Operation(string_t input, uint8_t width, uint8_t scale) {
+		T result;
+		string error_message;
+		return OP::Operation(input, result, &error_message, width, scale);
+	}
+};
+
+struct TryCastFloatingOperator {
+	template <class OP, class T>
+	static bool Operation(string_t input) {
+		T result;
+		string error_message;
+		return OP::Operation(input, result, &error_message);
+	}
+};
+
+bool TryCastDecimalValueCommaSeparated(const string_t &value_str, const LogicalType &sql_type) {
+	auto width = DecimalType::GetWidth(sql_type);
+	auto scale = DecimalType::GetScale(sql_type);
+	switch (sql_type.InternalType()) {
+	case PhysicalType::INT16:
+		return TryCastDecimalOperator::Operation<TryCastToDecimalCommaSeparated, int16_t>(value_str, width, scale);
+	case PhysicalType::INT32:
+		return TryCastDecimalOperator::Operation<TryCastToDecimalCommaSeparated, int32_t>(value_str, width, scale);
+	case PhysicalType::INT64:
+		return TryCastDecimalOperator::Operation<TryCastToDecimalCommaSeparated, int64_t>(value_str, width, scale);
+	case PhysicalType::INT128:
+		return TryCastDecimalOperator::Operation<TryCastToDecimalCommaSeparated, hugeint_t>(value_str, width, scale);
+	default:
+		throw InternalException("Unimplemented physical type for decimal");
+	}
+}
+
+bool TryCastFloatingValueCommaSeparated(const string_t &value_str, const LogicalType &sql_type) {
+	switch (sql_type.InternalType()) {
+	case PhysicalType::DOUBLE:
+		return TryCastFloatingOperator::Operation<TryCastErrorMessageCommaSeparated, double>(value_str);
+	case PhysicalType::FLOAT:
+		return TryCastFloatingOperator::Operation<TryCastErrorMessageCommaSeparated, float>(value_str);
+	default:
+		throw InternalException("Unimplemented physical type for floating");
+	}
+}
+
 bool BaseCSVReader::TryCastValue(const Value &value, const LogicalType &sql_type) {
 	if (options.has_format[LogicalTypeId::DATE] && sql_type.id() == LogicalTypeId::DATE) {
 		date_t result;
@@ -88,6 +136,11 @@ bool BaseCSVReader::TryCastValue(const Value &value, const LogicalType &sql_type
 		string error_message;
 		return options.date_format[LogicalTypeId::TIMESTAMP].TryParseTimestamp(string_t(StringValue::Get(value)),
 		                                                                       result, error_message);
+	} else if (options.decimal_separator != "." && sql_type.id() == LogicalTypeId::DECIMAL) {
+		return TryCastDecimalValueCommaSeparated(string_t(StringValue::Get(value)), sql_type);
+	} else if (options.decimal_separator != "." &&
+	           ((sql_type.id() == LogicalTypeId::FLOAT) || (sql_type.id() == LogicalTypeId::DOUBLE))) {
+		return TryCastFloatingValueCommaSeparated(string_t(StringValue::Get(value)), sql_type);
 	} else {
 		Value new_value;
 		string error_message;
@@ -135,6 +188,36 @@ bool TryCastTimestampVector(BufferedCSVReaderOptions &options, Vector &input_vec
 	                                                                         count, error_message);
 }
 
+template <class OP, class T>
+bool TemplatedTryCastFloatingVector(BufferedCSVReaderOptions &options, Vector &input_vector, Vector &result_vector,
+                                    idx_t count, string &error_message) {
+	D_ASSERT(input_vector.GetType().id() == LogicalTypeId::VARCHAR);
+	bool all_converted = true;
+	UnaryExecutor::Execute<string_t, T>(input_vector, result_vector, count, [&](string_t input) {
+		T result;
+		if (!OP::Operation(input, result, &error_message)) {
+			all_converted = false;
+		}
+		return result;
+	});
+	return all_converted;
+}
+
+template <class OP, class T>
+bool TemplatedTryCastDecimalVector(BufferedCSVReaderOptions &options, Vector &input_vector, Vector &result_vector,
+                                   idx_t count, string &error_message, uint8_t width, uint8_t scale) {
+	D_ASSERT(input_vector.GetType().id() == LogicalTypeId::VARCHAR);
+	bool all_converted = true;
+	UnaryExecutor::Execute<string_t, T>(input_vector, result_vector, count, [&](string_t input) {
+		T result;
+		if (!OP::Operation(input, result, &error_message, width, scale)) {
+			all_converted = false;
+		}
+		return result;
+	});
+	return all_converted;
+}
+
 bool BaseCSVReader::TryCastVector(Vector &parse_chunk_col, idx_t size, const LogicalType &sql_type) {
 	// try vector-cast from string to sql_type
 	Vector dummy_result(sql_type);
@@ -160,7 +243,7 @@ void BaseCSVReader::AddValue(string_t str_val, idx_t &column, vector<idx_t> &esc
 	} else {
 		row_empty = false;
 	}
-	if (!sql_types.empty() && column == sql_types.size() && length == 0) {
+	if (!return_types.empty() && column == return_types.size() && length == 0) {
 		// skip a single trailing delimiter in last column
 		return;
 	}
@@ -168,14 +251,14 @@ void BaseCSVReader::AddValue(string_t str_val, idx_t &column, vector<idx_t> &esc
 		column++;
 		return;
 	}
-	if (column >= sql_types.size()) {
+	if (column >= return_types.size()) {
 		if (options.ignore_errors) {
 			error_column_overflow = true;
 			return;
 		} else {
 			throw InvalidInputException(
 			    "Error in file \"%s\", on line %s: expected %lld values per row, but got more. (%s)", options.file_path,
-			    GetLineNumberStr(linenr, linenr_estimated).c_str(), sql_types.size(), options.ToString());
+			    GetLineNumberStr(linenr, linenr_estimated).c_str(), return_types.size(), options.ToString());
 		}
 	}
 
@@ -183,7 +266,7 @@ void BaseCSVReader::AddValue(string_t str_val, idx_t &column, vector<idx_t> &esc
 	idx_t row_entry = parse_chunk.size();
 
 	// test against null string, but only if the value was not quoted
-	if ((!has_quotes || sql_types[column].id() != LogicalTypeId::VARCHAR) && !options.force_not_null[column] &&
+	if ((!has_quotes || return_types[column].id() != LogicalTypeId::VARCHAR) && !options.force_not_null[column] &&
 	    Equals::Operation(str_val, string_t(options.null_str))) {
 		FlatVector::SetNull(parse_chunk.data[column], row_entry, true);
 	} else {
@@ -221,7 +304,7 @@ bool BaseCSVReader::AddRow(DataChunk &insert_chunk, idx_t &column, string &error
 
 	if (row_empty) {
 		row_empty = false;
-		if (sql_types.size() != 1) {
+		if (return_types.size() != 1) {
 			if (mode == ParserMode::PARSING) {
 				FlatVector::SetNull(parse_chunk.data[0], parse_chunk.size(), false);
 			}
@@ -238,7 +321,7 @@ bool BaseCSVReader::AddRow(DataChunk &insert_chunk, idx_t &column, string &error
 		return false;
 	}
 
-	if (column < sql_types.size() && mode != ParserMode::SNIFFING_DIALECT) {
+	if (column < return_types.size() && mode != ParserMode::SNIFFING_DIALECT) {
 		if (options.ignore_errors) {
 			column = 0;
 			return false;
@@ -249,7 +332,7 @@ bool BaseCSVReader::AddRow(DataChunk &insert_chunk, idx_t &column, string &error
 			} else {
 				throw InvalidInputException(
 				    "Error in file \"%s\" on line %s: expected %lld values per row, but got %d.\nParser options:\n%s",
-				    options.file_path, GetLineNumberStr(linenr, linenr_estimated).c_str(), sql_types.size(), column,
+				    options.file_path, GetLineNumberStr(linenr, linenr_estimated).c_str(), return_types.size(), column,
 				    options.ToString());
 			}
 		}
@@ -282,13 +365,6 @@ bool BaseCSVReader::AddRow(DataChunk &insert_chunk, idx_t &column, string &error
 	return false;
 }
 
-void BaseCSVReader::SetNullUnionCols(DataChunk &insert_chunk) {
-	for (idx_t col = 0; col < insert_nulls_idx.size(); ++col) {
-		insert_chunk.data[insert_nulls_idx[col]].SetVectorType(VectorType::CONSTANT_VECTOR);
-		ConstantVector::SetNull(insert_chunk.data[insert_nulls_idx[col]], true);
-	}
-}
-
 void BaseCSVReader::VerifyUTF8(idx_t col_idx, idx_t row_idx, DataChunk &chunk, int64_t offset) {
 	D_ASSERT(col_idx < chunk.data.size());
 	D_ASSERT(row_idx < chunk.size());
@@ -302,8 +378,8 @@ void BaseCSVReader::VerifyUTF8(idx_t col_idx, idx_t row_idx, DataChunk &chunk, i
 	auto utf_type = Utf8Proc::Analyze(s.GetDataUnsafe(), s.GetSize());
 	if (utf_type == UnicodeType::INVALID) {
 		string col_name = to_string(col_idx);
-		if (col_idx < col_names.size()) {
-			col_name = "\"" + col_names[col_idx] + "\"";
+		if (col_idx < names.size()) {
+			col_name = "\"" + names[col_idx] + "\"";
 		}
 		int64_t error_line = linenr - (chunk.size() - row_idx) + 1 + offset;
 		D_ASSERT(error_line >= 0);
@@ -321,6 +397,48 @@ void BaseCSVReader::VerifyUTF8(idx_t col_idx) {
 	}
 }
 
+bool TryCastDecimalVectorCommaSeparated(BufferedCSVReaderOptions &options, Vector &input_vector, Vector &result_vector,
+                                        idx_t count, string &error_message, LogicalType &result_type) {
+	auto width = DecimalType::GetWidth(result_type);
+	auto scale = DecimalType::GetScale(result_type);
+	switch (result_type.InternalType()) {
+	case PhysicalType::INT16:
+		return TemplatedTryCastDecimalVector<TryCastToDecimalCommaSeparated, int16_t>(
+		    options, input_vector, result_vector, count, error_message, width, scale);
+		break;
+	case PhysicalType::INT32:
+		return TemplatedTryCastDecimalVector<TryCastToDecimalCommaSeparated, int32_t>(
+		    options, input_vector, result_vector, count, error_message, width, scale);
+		break;
+	case PhysicalType::INT64:
+		return TemplatedTryCastDecimalVector<TryCastToDecimalCommaSeparated, int64_t>(
+		    options, input_vector, result_vector, count, error_message, width, scale);
+		break;
+	case PhysicalType::INT128:
+		return TemplatedTryCastDecimalVector<TryCastToDecimalCommaSeparated, hugeint_t>(
+		    options, input_vector, result_vector, count, error_message, width, scale);
+		break;
+	default:
+		throw InternalException("Unimplemented physical type for decimal");
+	}
+}
+
+bool TryCastFloatingVectorCommaSeparated(BufferedCSVReaderOptions &options, Vector &input_vector, Vector &result_vector,
+                                         idx_t count, string &error_message, LogicalType &result_type) {
+	switch (result_type.InternalType()) {
+	case PhysicalType::DOUBLE:
+		return TemplatedTryCastFloatingVector<TryCastErrorMessageCommaSeparated, double>(
+		    options, input_vector, result_vector, count, error_message);
+		break;
+	case PhysicalType::FLOAT:
+		return TemplatedTryCastFloatingVector<TryCastErrorMessageCommaSeparated, float>(
+		    options, input_vector, result_vector, count, error_message);
+		break;
+	default:
+		throw InternalException("Unimplemented physical type for floating");
+	}
+}
+
 bool BaseCSVReader::Flush(DataChunk &insert_chunk, bool try_add_line) {
 	if (parse_chunk.size() == 0) {
 		return true;
@@ -330,14 +448,16 @@ bool BaseCSVReader::Flush(DataChunk &insert_chunk, bool try_add_line) {
 
 	// convert the columns in the parsed chunk to the types of the table
 	insert_chunk.SetCardinality(parse_chunk);
-	for (idx_t col_idx = 0; col_idx < sql_types.size(); col_idx++) {
+	for (idx_t col_idx = 0; col_idx < return_types.size(); col_idx++) {
 		auto insert_idx = insert_cols_idx[col_idx];
-		auto &type = sql_types[col_idx];
+		auto &type = return_types[col_idx];
 		if (type.id() == LogicalTypeId::VARCHAR) {
 			// target type is varchar: no need to convert
 			// just test that all strings are valid utf-8 strings
 			VerifyUTF8(col_idx);
-			insert_chunk.data[insert_idx].Reference(parse_chunk.data[col_idx]);
+			// reinterpret rather than reference so we can deal with user-defined types
+			insert_chunk.data[insert_idx].Reinterpret(parse_chunk.data[col_idx]);
+
 		} else {
 			string error_message;
 			bool success;
@@ -345,10 +465,20 @@ bool BaseCSVReader::Flush(DataChunk &insert_chunk, bool try_add_line) {
 				// use the date format to cast the chunk
 				success = TryCastDateVector(options, parse_chunk.data[col_idx], insert_chunk.data[insert_idx],
 				                            parse_chunk.size(), error_message);
-			} else if (options.has_format[LogicalTypeId::TIMESTAMP] && type.id() == LogicalTypeId::TIMESTAMP) {
+			} else if (options.has_format[LogicalTypeId::TIMESTAMP] &&
+			           return_types[col_idx].id() == LogicalTypeId::TIMESTAMP) {
 				// use the date format to cast the chunk
 				success = TryCastTimestampVector(options, parse_chunk.data[col_idx], insert_chunk.data[insert_idx],
 				                                 parse_chunk.size(), error_message);
+			} else if (options.decimal_separator != "." && (return_types[col_idx].id() == LogicalTypeId::FLOAT ||
+			                                                return_types[col_idx].id() == LogicalTypeId::DOUBLE)) {
+				success = TryCastFloatingVectorCommaSeparated(options, parse_chunk.data[col_idx],
+				                                              insert_chunk.data[insert_idx], parse_chunk.size(),
+				                                              error_message, return_types[col_idx]);
+			} else if (options.decimal_separator != "." && return_types[col_idx].id() == LogicalTypeId::DECIMAL) {
+				success = TryCastDecimalVectorCommaSeparated(options, parse_chunk.data[col_idx],
+				                                             insert_chunk.data[insert_idx], parse_chunk.size(),
+				                                             error_message, return_types[col_idx]);
 			} else {
 				// target type is not varchar: perform a cast
 				success = VectorOperations::DefaultTryCast(parse_chunk.data[col_idx], insert_chunk.data[insert_idx],
@@ -365,8 +495,8 @@ bool BaseCSVReader::Flush(DataChunk &insert_chunk, bool try_add_line) {
 				continue;
 			}
 			string col_name = to_string(col_idx);
-			if (col_idx < col_names.size()) {
-				col_name = "\"" + col_names[col_idx] + "\"";
+			if (col_idx < names.size()) {
+				col_name = "\"" + names[col_idx] + "\"";
 			}
 
 			// figure out the exact line number
@@ -401,7 +531,7 @@ bool BaseCSVReader::Flush(DataChunk &insert_chunk, bool try_add_line) {
 
 		for (idx_t row_idx = 0; row_idx < parse_chunk.size(); row_idx++) {
 			bool failed = false;
-			for (idx_t column_idx = 0; column_idx < sql_types.size(); column_idx++) {
+			for (idx_t column_idx = 0; column_idx < return_types.size(); column_idx++) {
 
 				auto &inserted_column = insert_chunk.data[column_idx];
 				auto &parsed_column = parse_chunk.data[column_idx];
@@ -421,4 +551,32 @@ bool BaseCSVReader::Flush(DataChunk &insert_chunk, bool try_add_line) {
 	parse_chunk.Reset();
 	return true;
 }
+
+void BaseCSVReader::SetNewLineDelimiter(bool carry, bool carry_followed_by_nl) {
+	if ((mode == ParserMode::SNIFFING_DIALECT && !options.has_newline) ||
+	    options.new_line == NewLineIdentifier::NOT_SET) {
+		if (options.new_line == NewLineIdentifier::MIX) {
+			return;
+		}
+		NewLineIdentifier this_line_identifier;
+		if (carry) {
+			if (carry_followed_by_nl) {
+				this_line_identifier = NewLineIdentifier::CARRY_ON;
+			} else {
+				this_line_identifier = NewLineIdentifier::SINGLE;
+			}
+		} else {
+			this_line_identifier = NewLineIdentifier::SINGLE;
+		}
+		if (options.new_line == NewLineIdentifier::NOT_SET) {
+			options.new_line = this_line_identifier;
+			return;
+		}
+		if (options.new_line != this_line_identifier) {
+			options.new_line = NewLineIdentifier::MIX;
+			return;
+		}
+		options.new_line = this_line_identifier;
+	}
+}
 } // namespace duckdb