duckdb 0.6.2-dev971.0 → 0.7.1-dev2.0

package/src/duckdb/src/execution/operator/persistent/parallel_csv_reader.cpp

@@ -26,9 +26,9 @@ namespace duckdb {
 
 ParallelCSVReader::ParallelCSVReader(ClientContext &context, BufferedCSVReaderOptions options_p,
                                      unique_ptr<CSVBufferRead> buffer_p, const vector<LogicalType> &requested_types)
-    : BaseCSVReader(context, move(options_p), requested_types) {
+    : BaseCSVReader(context, std::move(options_p), requested_types) {
 	Initialize(requested_types);
-	SetBufferRead(move(buffer_p));
+	SetBufferRead(std::move(buffer_p));
 	if (options.delimiter.size() > 1 || options.escape.size() > 1 || options.quote.size() > 1) {
 		throw InternalException("Parallel CSV reader cannot handle CSVs with multi-byte delimiters/escapes/quotes");
 	}
@@ -38,20 +38,39 @@ ParallelCSVReader::~ParallelCSVReader() {
 }
 
 void ParallelCSVReader::Initialize(const vector<LogicalType> &requested_types) {
-	sql_types = requested_types;
-	InitParseChunk(sql_types.size());
-	InitInsertChunkIdx(sql_types.size());
+	return_types = requested_types;
+	InitParseChunk(return_types.size());
+	InitInsertChunkIdx(return_types.size());
+}
+
+bool ParallelCSVReader::NewLineDelimiter(bool carry, bool carry_followed_by_nl, bool first_char) {
+	// Set the delimiter if not set yet.
+	SetNewLineDelimiter(carry, carry_followed_by_nl);
+	D_ASSERT(options.new_line == NewLineIdentifier::SINGLE || options.new_line == NewLineIdentifier::CARRY_ON);
+	if (options.new_line == NewLineIdentifier::SINGLE) {
+		return (!carry) || (carry && !carry_followed_by_nl);
+	}
+	return (carry && carry_followed_by_nl) || (!carry && first_char);
 }
 
 bool ParallelCSVReader::SetPosition(DataChunk &insert_chunk) {
 	if (buffer->buffer->IsCSVFileFirstBuffer() && start_buffer == position_buffer &&
 	    start_buffer == buffer->buffer->GetStart()) {
+		verification_positions.beginning_of_first_line = position_buffer;
+		verification_positions.end_of_last_line = position_buffer;
 		// First buffer doesn't need any setting
 		// Unless we have a header
 		if (options.header && options.auto_detect) {
 			for (; position_buffer < end_buffer; position_buffer++) {
 				if (StringUtil::CharacterIsNewline((*buffer)[position_buffer])) {
+					bool carrier_return = (*buffer)[position_buffer] == '\r';
 					position_buffer++;
+					if (carrier_return && position_buffer < buffer_size && (*buffer)[position_buffer] == '\n') {
+						position_buffer++;
+					}
+					if (position_buffer > end_buffer) {
+						return false;
+					}
 					return true;
 				}
 			}
@@ -70,26 +89,50 @@ bool ParallelCSVReader::SetPosition(DataChunk &insert_chunk) {
 		first_line_chunk.Initialize(allocator, insert_chunk.GetTypes());
 		for (; position_buffer < end_buffer; position_buffer++) {
 			if (StringUtil::CharacterIsNewline((*buffer)[position_buffer])) {
+				bool carriage_return = (*buffer)[position_buffer] == '\r';
+				bool carriage_return_followed = false;
 				position_buffer++;
-				break;
+				if (position_buffer < end_buffer) {
+					if (carriage_return && (*buffer)[position_buffer] == '\n') {
+						carriage_return_followed = true;
+						position_buffer++;
+					}
+				}
+				if (NewLineDelimiter(carriage_return, carriage_return_followed, position_buffer - 1 == start_buffer)) {
+					break;
+				}
 			}
 		}
-		D_ASSERT(position_buffer <= end_buffer);
-		if (position_buffer == end_buffer && !StringUtil::CharacterIsNewline((*buffer)[position_buffer - 1])) {
+
+		if (position_buffer >= end_buffer && !StringUtil::CharacterIsNewline((*buffer)[position_buffer - 1])) {
+			break;
+		}
+
+		if (position_buffer > end_buffer && options.new_line == NewLineIdentifier::CARRY_ON &&
+		    (*buffer)[position_buffer - 1] == '\n') {
 			break;
 		}
 		idx_t position_set = position_buffer;
 		start_buffer = position_buffer;
 		// We check if we can add this line
 		successfully_read_first_line = TryParseSimpleCSV(first_line_chunk, error_message, true);
-		start_buffer = position_set;
+
 		end_buffer = end_buffer_real;
-		position_buffer = position_set;
-		if (end_buffer == position_buffer) {
+		start_buffer = position_set;
+		if (position_buffer >= end_buffer) {
+			if (successfully_read_first_line) {
+				position_buffer = position_set;
+			}
 			break;
 		}
+		position_buffer = position_set;
+	}
+	if (verification_positions.beginning_of_first_line == 0) {
+		verification_positions.beginning_of_first_line = position_buffer;
 	}
 
+	verification_positions.end_of_last_line = position_buffer;
+	finished = false;
 	return successfully_read_first_line;
 }
 
@@ -106,13 +149,22 @@ void ParallelCSVReader::SetBufferRead(unique_ptr<CSVBufferRead> buffer_read_p) {
 		buffer_size = buffer_read_p->buffer->GetBufferSize();
 	}
 	linenr = buffer_read_p->estimated_linenr;
-	buffer = move(buffer_read_p);
+	buffer = std::move(buffer_read_p);
 
 	linenr_estimated = true;
 	reached_remainder_state = false;
+	verification_positions.beginning_of_first_line = 0;
+	verification_positions.end_of_last_line = 0;
+	finished = false;
 	D_ASSERT(end_buffer <= buffer_size);
 }
 
+VerificationPositions ParallelCSVReader::GetVerificationPositions() {
+	verification_positions.beginning_of_first_line += buffer->buffer->GetCSVGlobalStart();
+	verification_positions.end_of_last_line += buffer->buffer->GetCSVGlobalStart();
+	return verification_positions;
+}
+
 // If BufferRemainder returns false, it means we are done scanning this buffer and should go to the end_state
 bool ParallelCSVReader::BufferRemainder() {
 	if (position_buffer >= end_buffer && !reached_remainder_state) {
@@ -131,7 +183,6 @@ bool ParallelCSVReader::BufferRemainder() {
 }
 
 bool ParallelCSVReader::TryParseSimpleCSV(DataChunk &insert_chunk, string &error_message, bool try_add_line) {
-
 	// used for parsing algorithm
 	D_ASSERT(end_buffer <= buffer_size);
 	bool finished_chunk = false;
@@ -139,10 +190,15 @@ bool ParallelCSVReader::TryParseSimpleCSV(DataChunk &insert_chunk, string &error
 	idx_t offset = 0;
 	bool has_quotes = false;
 	vector<idx_t> escape_positions;
-	if (start_buffer == buffer->buffer_start && !try_add_line) {
+	if ((start_buffer == buffer->buffer_start || start_buffer == buffer->buffer_end) && !try_add_line) {
 		// First time reading this buffer piece
 		if (!SetPosition(insert_chunk)) {
 			// This means the buffer size does not contain a new line
+			if (position_buffer - start_buffer == options.buffer_size) {
+				error_message = "Line does not fit in one buffer. Increase the buffer size.";
+				return false;
+			}
+			finished = true;
 			return true;
 		}
 	}
@@ -180,7 +236,7 @@ normal : {
 			goto add_value;
 		} else if (StringUtil::CharacterIsNewline(c)) {
 			// newline: add row
-			if (column > 0 || try_add_line) {
+			if (column > 0 || try_add_line || insert_chunk.data.size() == 1) {
 				goto add_row;
 			}
 		}
@@ -227,7 +283,8 @@ add_row : {
 	offset = 0;
 	has_quotes = false;
 	start_buffer = ++position_buffer;
-	if (reached_remainder_state || finished_chunk) {
+	verification_positions.end_of_last_line = position_buffer;
+	if (reached_remainder_state) {
 		goto final_state;
 	}
 	if (!BufferRemainder()) {
@@ -235,8 +292,37 @@ add_row : {
 	}
 	if (carriage_return) {
 		// \r newline, go to special state that parses an optional \n afterwards
-		goto carriage_return;
+		// optionally skips a newline (\n) character, which allows \r\n to be interpreted as a single line
+		if ((*buffer)[position_buffer] == '\n') {
+			if (options.new_line == NewLineIdentifier::SINGLE) {
+				error_message = "Wrong NewLine Identifier. Expecting \\r\\n";
+				return false;
+			}
+			// newline after carriage return: skip
+			// increase position by 1 and move start to the new position
+			start_buffer = ++position_buffer;
+			verification_positions.end_of_last_line = position_buffer;
+			if (reached_remainder_state) {
+				goto final_state;
+			}
+		} else {
+			if (options.new_line == NewLineIdentifier::CARRY_ON) {
+				error_message = "Wrong NewLine Identifier. Expecting \\r or \\n";
+				return false;
+			}
+		}
+		if (!BufferRemainder()) {
+			goto final_state;
+		}
+		if (reached_remainder_state || finished_chunk) {
+			goto final_state;
+		}
+		goto value_start;
 	} else {
+		if (options.new_line == NewLineIdentifier::CARRY_ON) {
+			error_message = "Wrong NewLine Identifier. Expecting \\r or \\n";
+			return false;
+		}
 		// \n newline, move to value start
 		if (finished_chunk) {
 			goto final_state;
@@ -332,35 +418,33 @@ handle_escape : {
 	// escape was followed by quote or escape, go back to quoted state
 	goto in_quotes;
 }
-
-carriage_return : {
-	/* state: carriage_return */
-	// this stage optionally skips a newline (\n) character, which allows \r\n to be interpreted as a single line
-	if ((*buffer)[position_buffer] == '\n') {
-		// newline after carriage return: skip
-		// increase position by 1 and move start to the new position
-		start_buffer = ++position_buffer;
-		if (position_buffer >= buffer_size) {
-			// file ends right after delimiter, go to final state
-			goto final_state;
-		}
-	}
-	goto value_start;
-}
 final_state : {
 	/* state: final_stage reached after we finished reading the end_buffer of the csv buffer */
 	// reset end buffer
 	end_buffer = buffer->buffer_end;
+	if (position_buffer == end_buffer) {
+		reached_remainder_state = false;
+	}
 	if (finished_chunk) {
+		if (position_buffer >= end_buffer) {
+			if (position_buffer == end_buffer && StringUtil::CharacterIsNewline((*buffer)[position_buffer - 1]) &&
+			    position_buffer < buffer_size) {
+				// last position is a new line, we still have to go through one more line of this buffer
+				finished = false;
+			} else {
+				finished = true;
+			}
+		}
 		return true;
 	}
 	// If this is the last buffer, we have to read the last value
-	if (buffer->buffer->IsCSVFileLastBuffer() || (buffer->next_buffer->IsCSVFileLastBuffer())) {
-		if (column > 0 || try_add_line) {
+	if (buffer->buffer->IsCSVFileLastBuffer() || (buffer->next_buffer && buffer->next_buffer->IsCSVFileLastBuffer())) {
+		if (column > 0 || try_add_line || (insert_chunk.data.size() == 1 && start_buffer != position_buffer)) {
 			// remaining values to be added to the chunk
-			AddValue(buffer->GetValue(start_buffer, position_buffer, offset), column, escape_positions, has_quotes);
+			auto str_value = buffer->GetValue(start_buffer, position_buffer, offset);
+			AddValue(str_value, column, escape_positions, has_quotes);
 			if (try_add_line) {
-				bool success = column == sql_types.size();
+				bool success = column == return_types.size();
 				if (success) {
 					AddRow(insert_chunk, column, error_message);
 					success = Flush(insert_chunk);
@@ -370,6 +454,7 @@ final_state : {
 				return success;
 			} else {
 				AddRow(insert_chunk, column, error_message);
+				verification_positions.end_of_last_line = position_buffer;
 			}
 		}
 	}
@@ -377,6 +462,22 @@ final_state : {
 	if (mode == ParserMode::PARSING) {
 		Flush(insert_chunk);
 	}
+	if (position_buffer != verification_positions.end_of_last_line &&
+	    !StringUtil::CharacterIsNewline((*buffer)[position_buffer - 1])) {
+		error_message = "Line does not fit in one buffer. Increase the buffer size.";
+		return false;
+	}
+	if (position_buffer >= end_buffer) {
+		if (position_buffer >= end_buffer) {
+			if (position_buffer == end_buffer && StringUtil::CharacterIsNewline((*buffer)[position_buffer - 1]) &&
+			    position_buffer < buffer_size) {
+				// last position is a new line, we still have to go through one more line of this buffer
+				finished = false;
+			} else {
+				finished = true;
+			}
+		}
+	}
 	return true;
 };
 }

package/src/duckdb/src/execution/operator/persistent/physical_batch_insert.cpp

@@ -6,21 +6,22 @@
 #include "duckdb/storage/table/row_group_collection.hpp"
 #include "duckdb/storage/table_io_manager.hpp"
 #include "duckdb/transaction/local_storage.hpp"
+#include "duckdb/catalog/catalog_entry/duck_table_entry.hpp"
 
 namespace duckdb {
 
 PhysicalBatchInsert::PhysicalBatchInsert(vector<LogicalType> types, TableCatalogEntry *table,
                                          physical_index_vector_t<idx_t> column_index_map,
                                          vector<unique_ptr<Expression>> bound_defaults, idx_t estimated_cardinality)
-    : PhysicalOperator(PhysicalOperatorType::BATCH_INSERT, move(types), estimated_cardinality),
+    : PhysicalOperator(PhysicalOperatorType::BATCH_INSERT, std::move(types), estimated_cardinality),
       column_index_map(std::move(column_index_map)), insert_table(table), insert_types(table->GetTypes()),
-      bound_defaults(move(bound_defaults)) {
+      bound_defaults(std::move(bound_defaults)) {
 }
 
 PhysicalBatchInsert::PhysicalBatchInsert(LogicalOperator &op, SchemaCatalogEntry *schema,
                                          unique_ptr<BoundCreateTableInfo> info_p, idx_t estimated_cardinality)
     : PhysicalOperator(PhysicalOperatorType::BATCH_CREATE_TABLE_AS, op.types, estimated_cardinality),
-      insert_table(nullptr), schema(schema), info(move(info_p)) {
+      insert_table(nullptr), schema(schema), info(std::move(info_p)) {
 	PhysicalInsert::GetInsertInfo(*info, insert_types, bound_defaults);
 }
 
@@ -38,7 +39,7 @@ public:
 
 public:
 	void AddCollection(unique_ptr<RowGroupCollection> collection) {
-		current_collections.push_back(move(collection));
+		current_collections.push_back(std::move(collection));
 	}
 
 	bool Empty() {
@@ -49,7 +50,7 @@ public:
 		if (Empty()) {
 			return nullptr;
 		}
-		unique_ptr<RowGroupCollection> new_collection = move(current_collections[0]);
+		unique_ptr<RowGroupCollection> new_collection = std::move(current_collections[0]);
 		if (current_collections.size() > 1) {
 			// we have gathered multiple collections: create one big collection and merge that
 			auto &types = new_collection->GetTypes();
@@ -98,7 +99,7 @@ public:
 	}
 
 	mutex lock;
-	TableCatalogEntry *table;
+	DuckTableEntry *table;
 	idx_t insert_count;
 	map<idx_t, unique_ptr<RowGroupCollection>> collections;
 
@@ -121,7 +122,7 @@ public:
 		} else {
 			// add the
 			D_ASSERT(result);
-			result->push_back(move(entry->second));
+			result->push_back(std::move(entry->second));
 			collections.erase(batch_index);
 		}
 		return true;
@@ -139,7 +140,7 @@ public:
 	                                                OptimisticDataWriter &writer) {
 		CollectionMerger merger(context);
 		for (auto &collection : merge_collections) {
-			merger.AddCollection(move(collection));
+			merger.AddCollection(std::move(collection));
 		}
 		return merger.Flush(writer);
 	}
@@ -189,7 +190,7 @@ public:
 					// note that we need to gather them in order of batch index
 					for (idx_t i = start_batch_index; i <= end_batch_index; i++) {
 						if (i == batch_index) {
-							merge_collections.push_back(move(current_collection));
+							merge_collections.push_back(std::move(current_collection));
 							continue;
 						}
 						auto can_merge = CheckMerge(i, merge_collections);
@@ -201,13 +202,13 @@ public:
 			}
 			if (merge_collections.empty()) {
 				// no collections to merge together - add the collection to the batch index
-				collections[batch_index] = move(current_collection);
+				collections[batch_index] = std::move(current_collection);
 			}
 		}
 		if (!merge_collections.empty()) {
 			// merge together the collections
 			D_ASSERT(writer);
-			auto final_collection = MergeCollections(context, move(merge_collections), *writer);
+			auto final_collection = MergeCollections(context, std::move(merge_collections), *writer);
 			D_ASSERT(final_collection->GetTotalRows() == merge_count);
 			D_ASSERT(final_collection->GetTotalRows() >= RowGroup::ROW_GROUP_SIZE);
 			if (written_to_disk) {
@@ -217,7 +218,7 @@ public:
 			{
 				lock_guard<mutex> l(lock);
 				VerifyUniqueBatch(batch_index);
-				collections[batch_index] = move(final_collection);
+				collections[batch_index] = std::move(final_collection);
 			}
 		}
 	}
@@ -249,9 +250,9 @@ public:
 		writer->FlushToDisk(*current_collection, true);
 	}
 
-	void CreateNewCollection(TableCatalogEntry *table, const vector<LogicalType> &insert_types) {
-		auto &table_info = table->storage->info;
-		auto &block_manager = TableIOManager::Get(*table->storage).GetBlockManagerForRowData();
+	void CreateNewCollection(DuckTableEntry *table, const vector<LogicalType> &insert_types) {
+		auto &table_info = table->GetStorage().info;
+		auto &block_manager = TableIOManager::Get(table->GetStorage()).GetBlockManagerForRowData();
 		current_collection = make_unique<RowGroupCollection>(table_info, block_manager, insert_types, MAX_ROW_ID);
 		current_collection->InitializeEmpty();
 		current_collection->InitializeAppend(current_append_state);
@@ -266,12 +267,13 @@ unique_ptr<GlobalSinkState> PhysicalBatchInsert::GetGlobalSinkState(ClientContex
 		D_ASSERT(!insert_table);
 		auto &catalog = *schema->catalog;
 		result->table =
-		    (TableCatalogEntry *)catalog.CreateTable(catalog.GetCatalogTransaction(context), schema, info.get());
+		    (DuckTableEntry *)catalog.CreateTable(catalog.GetCatalogTransaction(context), schema, info.get());
 	} else {
 		D_ASSERT(insert_table);
-		result->table = insert_table;
+		D_ASSERT(insert_table->IsDuckTable());
+		result->table = (DuckTableEntry *)insert_table;
 	}
-	return move(result);
+	return std::move(result);
 }
 
 unique_ptr<LocalSinkState> PhysicalBatchInsert::GetLocalSinkState(ExecutionContext &context) const {
@@ -290,18 +292,20 @@ SinkResultType PhysicalBatchInsert::Sink(ExecutionContext &context, GlobalSinkSt
 		lock_guard<mutex> l(gstate.lock);
 		// no collection yet: create a new one
 		lstate.CreateNewCollection(table, insert_types);
-		lstate.writer = gstate.table->storage->CreateOptimisticWriter(context.client);
+		lstate.writer = gstate.table->GetStorage().CreateOptimisticWriter(context.client);
 	} else if (lstate.current_index != lstate.batch_index) {
 		// batch index has changed: move the old collection to the global state and create a new collection
 		TransactionData tdata(0, 0);
 		lstate.current_collection->FinalizeAppend(tdata, lstate.current_append_state);
 		lstate.FlushToDisk();
-		gstate.AddCollection(context.client, lstate.current_index, move(lstate.current_collection), lstate.writer,
+		gstate.AddCollection(context.client, lstate.current_index, std::move(lstate.current_collection), lstate.writer,
 		                     &lstate.written_to_disk);
 		lstate.CreateNewCollection(table, insert_types);
 	}
 	lstate.current_index = lstate.batch_index;
-	table->storage->VerifyAppendConstraints(*table, context.client, lstate.insert_chunk);
+
+	table->GetStorage().VerifyAppendConstraints(*table, context.client, lstate.insert_chunk);
+
 	auto new_row_group = lstate.current_collection->Append(lstate.insert_chunk, lstate.current_append_state);
 	if (new_row_group) {
 		lstate.writer->CheckFlushToDisk(*lstate.current_collection);
@@ -326,7 +330,7 @@ void PhysicalBatchInsert::Combine(ExecutionContext &context, GlobalSinkState &gs
 
 	TransactionData tdata(0, 0);
 	lstate.current_collection->FinalizeAppend(tdata, lstate.current_append_state);
-	gstate.AddCollection(context.client, lstate.current_index, move(lstate.current_collection));
+	gstate.AddCollection(context.client, lstate.current_index, std::move(lstate.current_collection));
 }
 
 SinkFinalizeType PhysicalBatchInsert::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
@@ -338,29 +342,29 @@ SinkFinalizeType PhysicalBatchInsert::Finalize(Pipeline &pipeline, Event &event,
 	vector<unique_ptr<CollectionMerger>> mergers;
 	unique_ptr<CollectionMerger> current_merger;
 
-	auto &storage = *gstate.table->storage;
+	auto &storage = gstate.table->GetStorage();
 	for (auto &collection : gstate.collections) {
 		if (collection.second->GetTotalRows() < LocalStorage::MERGE_THRESHOLD) {
 			// this collection has very few rows: add it to the merge set
 			if (!current_merger) {
 				current_merger = make_unique<CollectionMerger>(context);
 			}
-			current_merger->AddCollection(move(collection.second));
+			current_merger->AddCollection(std::move(collection.second));
 		} else {
 			// this collection has a lot of rows: it does not need to be merged
 			// create a separate collection merger only for this entry
 			if (current_merger) {
 				// we have small collections remaining: flush them
-				mergers.push_back(move(current_merger));
+				mergers.push_back(std::move(current_merger));
 				current_merger.reset();
 			}
 			auto larger_merger = make_unique<CollectionMerger>(context);
-			larger_merger->AddCollection(move(collection.second));
-			mergers.push_back(move(larger_merger));
+			larger_merger->AddCollection(std::move(collection.second));
+			mergers.push_back(std::move(larger_merger));
 		}
 	}
 	if (current_merger) {
-		mergers.push_back(move(current_merger));
+		mergers.push_back(std::move(current_merger));
 	}
 
 	// now that we have created all of the mergers, perform the actual merging