duckdb 0.6.2-dev971.0 → 0.7.1-dev100.0

package/src/duckdb/src/execution/operator/persistent/csv_buffer.cpp

@@ -3,12 +3,15 @@
 
 namespace duckdb {
 
-CSVBuffer::CSVBuffer(ClientContext &context, idx_t buffer_size_p, CSVFileHandle &file_handle)
+CSVBuffer::CSVBuffer(ClientContext &context, idx_t buffer_size_p, CSVFileHandle &file_handle,
+                     idx_t &global_csv_current_position)
     : context(context), first_buffer(true) {
 	this->handle = AllocateBuffer(buffer_size_p);
 
 	auto buffer = Ptr();
 	actual_size = file_handle.Read(buffer, buffer_size_p);
+	global_csv_start = global_csv_current_position;
+	global_csv_current_position += actual_size;
 	if (actual_size >= 3 && buffer[0] == '\xEF' && buffer[1] == '\xBB' && buffer[2] == '\xBF') {
 		start_position += 3;
 	}
@@ -16,21 +19,25 @@ CSVBuffer::CSVBuffer(ClientContext &context, idx_t buffer_size_p, CSVFileHandle
 }
 
 CSVBuffer::CSVBuffer(ClientContext &context, BufferHandle buffer_p, idx_t buffer_size_p, idx_t actual_size_p,
-                     bool final_buffer)
-    : context(context), handle(move(buffer_p)), actual_size(actual_size_p), last_buffer(final_buffer) {
+                     bool final_buffer, idx_t global_csv_current_position)
+    : context(context), handle(std::move(buffer_p)), actual_size(actual_size_p), last_buffer(final_buffer),
+      global_csv_start(global_csv_current_position) {
 }
 
-unique_ptr<CSVBuffer> CSVBuffer::Next(CSVFileHandle &file_handle, idx_t set_buffer_size) {
+unique_ptr<CSVBuffer> CSVBuffer::Next(CSVFileHandle &file_handle, idx_t buffer_size,
+                                      idx_t &global_csv_current_position) {
 	if (file_handle.FinishedReading()) {
 		// this was the last buffer
 		return nullptr;
 	}
 
-	auto next_buffer = AllocateBuffer(set_buffer_size);
-	idx_t next_buffer_actual_size = file_handle.Read(next_buffer.Ptr(), set_buffer_size);
+	auto next_buffer = AllocateBuffer(buffer_size);
+	idx_t next_buffer_actual_size = file_handle.Read(next_buffer.Ptr(), buffer_size);
 
-	return make_unique<CSVBuffer>(context, move(next_buffer), set_buffer_size, next_buffer_actual_size,
-	                              file_handle.FinishedReading());
+	auto next_csv_buffer = make_unique<CSVBuffer>(context, std::move(next_buffer), buffer_size, next_buffer_actual_size,
+	                                              file_handle.FinishedReading(), global_csv_current_position);
+	global_csv_current_position += next_buffer_actual_size;
+	return next_csv_buffer;
 }
 
 BufferHandle CSVBuffer::AllocateBuffer(idx_t buffer_size) {
@@ -54,4 +61,8 @@ bool CSVBuffer::IsCSVFileFirstBuffer() {
 	return first_buffer;
 }
 
+idx_t CSVBuffer::GetCSVGlobalStart() {
+	return global_csv_start;
+}
+
 } // namespace duckdb

package/src/duckdb/src/execution/operator/persistent/csv_reader_options.cpp

@@ -1,4 +1,5 @@
 #include "duckdb/execution/operator/persistent/csv_reader_options.hpp"
+#include "duckdb/common/bind_helpers.hpp"
 #include "duckdb/common/vector_size.hpp"
 #include "duckdb/common/string_util.hpp"
 
@@ -59,54 +60,23 @@ static int64_t ParseInteger(const Value &value, const string &loption) {
 	return value.GetValue<int64_t>();
 }
 
-static vector<bool> ParseColumnList(const vector<Value> &set, vector<string> &names, const string &loption) {
-	vector<bool> result;
+void BufferedCSVReaderOptions::SetHeader(bool input) {
+	this->header = input;
+	this->has_header = true;
+}
 
-	if (set.empty()) {
-		throw BinderException("\"%s\" expects a column list or * as parameter", loption);
-	}
-	// list of options: parse the list
-	unordered_map<string, bool> option_map;
-	for (idx_t i = 0; i < set.size(); i++) {
-		option_map[set[i].ToString()] = false;
-	}
-	result.resize(names.size(), false);
-	for (idx_t i = 0; i < names.size(); i++) {
-		auto entry = option_map.find(names[i]);
-		if (entry != option_map.end()) {
-			result[i] = true;
-			entry->second = true;
-		}
-	}
-	for (auto &entry : option_map) {
-		if (!entry.second) {
-			throw BinderException("\"%s\" expected to find %s, but it was not found in the table", loption,
-			                      entry.first.c_str());
-		}
-	}
-	return result;
+void BufferedCSVReaderOptions::SetCompression(const string &compression) {
+	this->compression = FileCompressionTypeFromString(compression);
 }
 
-static vector<bool> ParseColumnList(const Value &value, vector<string> &names, const string &loption) {
-	vector<bool> result;
+void BufferedCSVReaderOptions::SetEscape(const string &input) {
+	this->escape = input;
+	this->has_escape = true;
+}
 
-	// Only accept a list of arguments
-	if (value.type().id() != LogicalTypeId::LIST) {
-		// Support a single argument if it's '*'
-		if (value.type().id() == LogicalTypeId::VARCHAR && value.GetValue<string>() == "*") {
-			result.resize(names.size(), true);
-			return result;
-		}
-		throw BinderException("\"%s\" expects a column list or * as parameter", loption);
-	}
-	auto &children = ListValue::GetChildren(value);
-	// accept '*' as single argument
-	if (children.size() == 1 && children[0].type().id() == LogicalTypeId::VARCHAR &&
-	    children[0].GetValue<string>() == "*") {
-		result.resize(names.size(), true);
-		return result;
-	}
-	return ParseColumnList(children, names, loption);
+void BufferedCSVReaderOptions::SetParallel(bool use_parallel) {
+	this->has_parallel = true;
+	this->use_parallel = use_parallel;
 }
 
 void BufferedCSVReaderOptions::SetDelimiter(const string &input) {
@@ -117,6 +87,22 @@ void BufferedCSVReaderOptions::SetDelimiter(const string &input) {
 	}
 }
 
+void BufferedCSVReaderOptions::SetQuote(const string &quote) {
+	this->quote = quote;
+	this->has_quote = true;
+}
+
+void BufferedCSVReaderOptions::SetNewline(const string &input) {
+	if (input == "\\n" || input == "\\r") {
+		new_line = NewLineIdentifier::SINGLE;
+	} else if (input == "\\r\\n") {
+		new_line = NewLineIdentifier::CARRY_ON;
+	} else {
+		throw InvalidInputException("This is not accepted as a newline: " + input);
+	}
+	has_newline = true;
+}
+
 void BufferedCSVReaderOptions::SetDateFormat(LogicalTypeId type, const string &format, bool read_format) {
 	string error;
 	if (read_format) {
@@ -140,6 +126,8 @@ void BufferedCSVReaderOptions::SetReadOption(const string &loption, const Value
 	}
 	if (loption == "auto_detect") {
 		auto_detect = ParseBoolean(value, loption);
+	} else if (loption == "parallel") {
+		SetParallel(ParseBoolean(value, loption));
 	} else if (loption == "sample_size") {
 		int64_t sample_size = ParseInteger(value, loption);
 		if (sample_size < 1 && sample_size != -1) {
@@ -181,9 +169,6 @@ void BufferedCSVReaderOptions::SetReadOption(const string &loption, const Value
 	} else if (loption == "timestamp_format" || loption == "timestampformat") {
 		string format = ParseString(value, loption);
 		SetDateFormat(LogicalTypeId::TIMESTAMP, format, true);
-	} else if (loption == "escape") {
-		escape = ParseString(value, loption);
-		has_escape = true;
 	} else if (loption == "ignore_errors") {
 		ignore_errors = ParseBoolean(value, loption);
 	} else if (loption == "union_by_name") {
@@ -193,6 +178,11 @@ void BufferedCSVReaderOptions::SetReadOption(const string &loption, const Value
 		if (buffer_size == 0) {
 			throw InvalidInputException("Buffer Size option must be higher than 0");
 		}
+	} else if (loption == "decimal_separator") {
+		decimal_separator = ParseString(value, loption);
+		if (decimal_separator != "." && decimal_separator != ",") {
+			throw BinderException("Unsupported parameter for DECIMAL_SEPARATOR: should be '.' or ','");
+		}
 	} else {
 		throw BinderException("Unrecognized option for CSV reader \"%s\"", loption);
 	}
@@ -226,14 +216,13 @@ bool BufferedCSVReaderOptions::SetBaseOption(const string &loption, const Value
 	if (StringUtil::StartsWith(loption, "delim") || StringUtil::StartsWith(loption, "sep")) {
 		SetDelimiter(ParseString(value, loption));
 	} else if (loption == "quote") {
-		quote = ParseString(value, loption);
-		has_quote = true;
+		SetQuote(ParseString(value, loption));
+	} else if (loption == "new_line") {
+		SetNewline(ParseString(value, loption));
 	} else if (loption == "escape") {
-		escape = ParseString(value, loption);
-		has_escape = true;
+		SetEscape(ParseString(value, loption));
 	} else if (loption == "header") {
-		header = ParseBoolean(value, loption);
-		has_header = true;
+		SetHeader(ParseBoolean(value, loption));
 	} else if (loption == "null" || loption == "nullstr") {
 		null_str = ParseString(value, loption);
 	} else if (loption == "encoding") {
@@ -242,7 +231,7 @@ bool BufferedCSVReaderOptions::SetBaseOption(const string &loption, const Value
 			throw BinderException("Copy is only supported for UTF-8 encoded files, ENCODING 'UTF-8'");
 		}
 	} else if (loption == "compression") {
-		compression = FileCompressionTypeFromString(ParseString(value, loption));
+		SetCompression(ParseString(value, loption));
 	} else {
 		// unrecognized option in base CSV
 		return false;
@@ -258,7 +247,7 @@ std::string BufferedCSVReaderOptions::ToString() const {
 	       "\n  header=" + std::to_string(header) +
 	       (has_header ? "" : (auto_detect ? " (auto detected)" : "' (default)")) +
 	       "\n  sample_size=" + std::to_string(sample_chunk_size * sample_chunks) +
-	       "\n  ignore_erros=" + std::to_string(ignore_errors) + "\n  all_varchar=" + std::to_string(all_varchar);
+	       "\n  ignore_errors=" + std::to_string(ignore_errors) + "\n  all_varchar=" + std::to_string(all_varchar);
 }
 
 } // namespace duckdb

package/src/duckdb/src/execution/operator/persistent/parallel_csv_reader.cpp

@@ -26,9 +26,9 @@ namespace duckdb {
 
 ParallelCSVReader::ParallelCSVReader(ClientContext &context, BufferedCSVReaderOptions options_p,
                                      unique_ptr<CSVBufferRead> buffer_p, const vector<LogicalType> &requested_types)
-    : BaseCSVReader(context, move(options_p), requested_types) {
+    : BaseCSVReader(context, std::move(options_p), requested_types) {
 	Initialize(requested_types);
-	SetBufferRead(move(buffer_p));
+	SetBufferRead(std::move(buffer_p));
 	if (options.delimiter.size() > 1 || options.escape.size() > 1 || options.quote.size() > 1) {
 		throw InternalException("Parallel CSV reader cannot handle CSVs with multi-byte delimiters/escapes/quotes");
 	}
@@ -38,20 +38,39 @@ ParallelCSVReader::~ParallelCSVReader() {
 }
 
 void ParallelCSVReader::Initialize(const vector<LogicalType> &requested_types) {
-	sql_types = requested_types;
-	InitParseChunk(sql_types.size());
-	InitInsertChunkIdx(sql_types.size());
+	return_types = requested_types;
+	InitParseChunk(return_types.size());
+	InitInsertChunkIdx(return_types.size());
+}
+
+bool ParallelCSVReader::NewLineDelimiter(bool carry, bool carry_followed_by_nl, bool first_char) {
+	// Set the delimiter if not set yet.
+	SetNewLineDelimiter(carry, carry_followed_by_nl);
+	D_ASSERT(options.new_line == NewLineIdentifier::SINGLE || options.new_line == NewLineIdentifier::CARRY_ON);
+	if (options.new_line == NewLineIdentifier::SINGLE) {
+		return (!carry) || (carry && !carry_followed_by_nl);
+	}
+	return (carry && carry_followed_by_nl) || (!carry && first_char);
 }
 
 bool ParallelCSVReader::SetPosition(DataChunk &insert_chunk) {
 	if (buffer->buffer->IsCSVFileFirstBuffer() && start_buffer == position_buffer &&
 	    start_buffer == buffer->buffer->GetStart()) {
+		verification_positions.beginning_of_first_line = position_buffer;
+		verification_positions.end_of_last_line = position_buffer;
 		// First buffer doesn't need any setting
 		// Unless we have a header
-		if (options.header && options.auto_detect) {
+		if (options.header) {
 			for (; position_buffer < end_buffer; position_buffer++) {
 				if (StringUtil::CharacterIsNewline((*buffer)[position_buffer])) {
+					bool carrier_return = (*buffer)[position_buffer] == '\r';
 					position_buffer++;
+					if (carrier_return && position_buffer < buffer_size && (*buffer)[position_buffer] == '\n') {
+						position_buffer++;
+					}
+					if (position_buffer > end_buffer) {
+						return false;
+					}
 					return true;
 				}
 			}
@@ -70,26 +89,50 @@ bool ParallelCSVReader::SetPosition(DataChunk &insert_chunk) {
 		first_line_chunk.Initialize(allocator, insert_chunk.GetTypes());
 		for (; position_buffer < end_buffer; position_buffer++) {
 			if (StringUtil::CharacterIsNewline((*buffer)[position_buffer])) {
+				bool carriage_return = (*buffer)[position_buffer] == '\r';
+				bool carriage_return_followed = false;
 				position_buffer++;
-				break;
+				if (position_buffer < end_buffer) {
+					if (carriage_return && (*buffer)[position_buffer] == '\n') {
+						carriage_return_followed = true;
+						position_buffer++;
+					}
+				}
+				if (NewLineDelimiter(carriage_return, carriage_return_followed, position_buffer - 1 == start_buffer)) {
+					break;
+				}
 			}
 		}
-		D_ASSERT(position_buffer <= end_buffer);
-		if (position_buffer == end_buffer && !StringUtil::CharacterIsNewline((*buffer)[position_buffer - 1])) {
+
+		if (position_buffer >= end_buffer && !StringUtil::CharacterIsNewline((*buffer)[position_buffer - 1])) {
+			break;
+		}
+
+		if (position_buffer > end_buffer && options.new_line == NewLineIdentifier::CARRY_ON &&
+		    (*buffer)[position_buffer - 1] == '\n') {
 			break;
 		}
 		idx_t position_set = position_buffer;
 		start_buffer = position_buffer;
 		// We check if we can add this line
 		successfully_read_first_line = TryParseSimpleCSV(first_line_chunk, error_message, true);
-		start_buffer = position_set;
+
 		end_buffer = end_buffer_real;
-		position_buffer = position_set;
-		if (end_buffer == position_buffer) {
+		start_buffer = position_set;
+		if (position_buffer >= end_buffer) {
+			if (successfully_read_first_line) {
+				position_buffer = position_set;
+			}
 			break;
 		}
+		position_buffer = position_set;
+	}
+	if (verification_positions.beginning_of_first_line == 0) {
+		verification_positions.beginning_of_first_line = position_buffer;
 	}
 
+	verification_positions.end_of_last_line = position_buffer;
+	finished = false;
 	return successfully_read_first_line;
 }
 
@@ -106,13 +149,22 @@ void ParallelCSVReader::SetBufferRead(unique_ptr<CSVBufferRead> buffer_read_p) {
 		buffer_size = buffer_read_p->buffer->GetBufferSize();
 	}
 	linenr = buffer_read_p->estimated_linenr;
-	buffer = move(buffer_read_p);
+	buffer = std::move(buffer_read_p);
 
 	linenr_estimated = true;
 	reached_remainder_state = false;
+	verification_positions.beginning_of_first_line = 0;
+	verification_positions.end_of_last_line = 0;
+	finished = false;
 	D_ASSERT(end_buffer <= buffer_size);
 }
 
+VerificationPositions ParallelCSVReader::GetVerificationPositions() {
+	verification_positions.beginning_of_first_line += buffer->buffer->GetCSVGlobalStart();
+	verification_positions.end_of_last_line += buffer->buffer->GetCSVGlobalStart();
+	return verification_positions;
+}
+
 // If BufferRemainder returns false, it means we are done scanning this buffer and should go to the end_state
 bool ParallelCSVReader::BufferRemainder() {
 	if (position_buffer >= end_buffer && !reached_remainder_state) {
@@ -131,7 +183,6 @@ bool ParallelCSVReader::BufferRemainder() {
 }
 
 bool ParallelCSVReader::TryParseSimpleCSV(DataChunk &insert_chunk, string &error_message, bool try_add_line) {
-
 	// used for parsing algorithm
 	D_ASSERT(end_buffer <= buffer_size);
 	bool finished_chunk = false;
@@ -139,10 +190,15 @@ bool ParallelCSVReader::TryParseSimpleCSV(DataChunk &insert_chunk, string &error
 	idx_t offset = 0;
 	bool has_quotes = false;
 	vector<idx_t> escape_positions;
-	if (start_buffer == buffer->buffer_start && !try_add_line) {
+	if ((start_buffer == buffer->buffer_start || start_buffer == buffer->buffer_end) && !try_add_line) {
 		// First time reading this buffer piece
 		if (!SetPosition(insert_chunk)) {
 			// This means the buffer size does not contain a new line
+			if (position_buffer - start_buffer == options.buffer_size) {
+				error_message = "Line does not fit in one buffer. Increase the buffer size.";
+				return false;
+			}
+			finished = true;
 			return true;
 		}
 	}
@@ -180,7 +236,7 @@ normal : {
 			goto add_value;
 		} else if (StringUtil::CharacterIsNewline(c)) {
 			// newline: add row
-			if (column > 0 || try_add_line) {
+			if (column > 0 || try_add_line || insert_chunk.data.size() == 1) {
 				goto add_row;
 			}
 		}
@@ -227,7 +283,8 @@ add_row : {
 	offset = 0;
 	has_quotes = false;
 	start_buffer = ++position_buffer;
-	if (reached_remainder_state || finished_chunk) {
+	verification_positions.end_of_last_line = position_buffer;
+	if (reached_remainder_state) {
 		goto final_state;
 	}
 	if (!BufferRemainder()) {
@@ -235,8 +292,37 @@ add_row : {
 	}
 	if (carriage_return) {
 		// \r newline, go to special state that parses an optional \n afterwards
-		goto carriage_return;
+		// optionally skips a newline (\n) character, which allows \r\n to be interpreted as a single line
+		if ((*buffer)[position_buffer] == '\n') {
+			if (options.new_line == NewLineIdentifier::SINGLE) {
+				error_message = "Wrong NewLine Identifier. Expecting \\r\\n";
+				return false;
+			}
+			// newline after carriage return: skip
+			// increase position by 1 and move start to the new position
+			start_buffer = ++position_buffer;
+			verification_positions.end_of_last_line = position_buffer;
+			if (reached_remainder_state) {
+				goto final_state;
+			}
+		} else {
+			if (options.new_line == NewLineIdentifier::CARRY_ON) {
+				error_message = "Wrong NewLine Identifier. Expecting \\r or \\n";
+				return false;
+			}
+		}
+		if (!BufferRemainder()) {
+			goto final_state;
+		}
+		if (reached_remainder_state || finished_chunk) {
+			goto final_state;
+		}
+		goto value_start;
 	} else {
+		if (options.new_line == NewLineIdentifier::CARRY_ON) {
+			error_message = "Wrong NewLine Identifier. Expecting \\r or \\n";
+			return false;
+		}
 		// \n newline, move to value start
 		if (finished_chunk) {
 			goto final_state;
@@ -332,35 +418,33 @@ handle_escape : {
 	// escape was followed by quote or escape, go back to quoted state
 	goto in_quotes;
 }
-
-carriage_return : {
-	/* state: carriage_return */
-	// this stage optionally skips a newline (\n) character, which allows \r\n to be interpreted as a single line
-	if ((*buffer)[position_buffer] == '\n') {
-		// newline after carriage return: skip
-		// increase position by 1 and move start to the new position
-		start_buffer = ++position_buffer;
-		if (position_buffer >= buffer_size) {
-			// file ends right after delimiter, go to final state
-			goto final_state;
-		}
-	}
-	goto value_start;
-}
 final_state : {
 	/* state: final_stage reached after we finished reading the end_buffer of the csv buffer */
 	// reset end buffer
 	end_buffer = buffer->buffer_end;
+	if (position_buffer == end_buffer) {
+		reached_remainder_state = false;
+	}
 	if (finished_chunk) {
+		if (position_buffer >= end_buffer) {
+			if (position_buffer == end_buffer && StringUtil::CharacterIsNewline((*buffer)[position_buffer - 1]) &&
+			    position_buffer < buffer_size) {
+				// last position is a new line, we still have to go through one more line of this buffer
+				finished = false;
+			} else {
+				finished = true;
+			}
+		}
 		return true;
 	}
 	// If this is the last buffer, we have to read the last value
-	if (buffer->buffer->IsCSVFileLastBuffer() || (buffer->next_buffer->IsCSVFileLastBuffer())) {
-		if (column > 0 || try_add_line) {
+	if (buffer->buffer->IsCSVFileLastBuffer() || (buffer->next_buffer && buffer->next_buffer->IsCSVFileLastBuffer())) {
+		if (column > 0 || try_add_line || (insert_chunk.data.size() == 1 && start_buffer != position_buffer)) {
 			// remaining values to be added to the chunk
-			AddValue(buffer->GetValue(start_buffer, position_buffer, offset), column, escape_positions, has_quotes);
+			auto str_value = buffer->GetValue(start_buffer, position_buffer, offset);
+			AddValue(str_value, column, escape_positions, has_quotes);
 			if (try_add_line) {
-				bool success = column == sql_types.size();
+				bool success = column == return_types.size();
 				if (success) {
 					AddRow(insert_chunk, column, error_message);
 					success = Flush(insert_chunk);
@@ -370,6 +454,7 @@ final_state : {
 				return success;
 			} else {
 				AddRow(insert_chunk, column, error_message);
+				verification_positions.end_of_last_line = position_buffer;
 			}
 		}
 	}
@@ -377,6 +462,22 @@ final_state : {
 	if (mode == ParserMode::PARSING) {
 		Flush(insert_chunk);
 	}
+	if (position_buffer != verification_positions.end_of_last_line &&
+	    !StringUtil::CharacterIsNewline((*buffer)[position_buffer - 1])) {
+		error_message = "Line does not fit in one buffer. Increase the buffer size.";
+		return false;
+	}
+	if (position_buffer >= end_buffer) {
+		if (position_buffer >= end_buffer) {
+			if (position_buffer == end_buffer && StringUtil::CharacterIsNewline((*buffer)[position_buffer - 1]) &&
+			    position_buffer < buffer_size) {
+				// last position is a new line, we still have to go through one more line of this buffer
+				finished = false;
+			} else {
+				finished = true;
+			}
+		}
+	}
 	return true;
 };
 }