duckdb 0.7.2-dev1901.0 → 0.7.2-dev2233.0

package/src/duckdb/src/execution/operator/join/physical_index_join.cpp

@@ -60,7 +60,7 @@ public:
 PhysicalIndexJoin::PhysicalIndexJoin(LogicalOperator &op, unique_ptr<PhysicalOperator> left,
                                      unique_ptr<PhysicalOperator> right, vector<JoinCondition> cond, JoinType join_type,
                                      const vector<idx_t> &left_projection_map_p, vector<idx_t> right_projection_map_p,
-                                     vector<column_t> column_ids_p, Index *index_p, bool lhs_first,
+                                     vector<column_t> column_ids_p, Index &index_p, bool lhs_first,
                                      idx_t estimated_cardinality)
     : CachingPhysicalOperator(PhysicalOperatorType::INDEX_JOIN, std::move(op.types), estimated_cardinality),
       left_projection_map(left_projection_map_p), right_projection_map(std::move(right_projection_map_p)),
@@ -74,7 +74,7 @@ PhysicalIndexJoin::PhysicalIndexJoin(LogicalOperator &op, unique_ptr<PhysicalOpe
 		condition_types.push_back(condition.left->return_type);
 	}
 	//! Only add to fetch_ids columns that are not indexed
-	for (auto &index_id : index->column_ids) {
+	for (auto &index_id : index.column_ids) {
 		index_ids.insert(index_id);
 	}
 
@@ -165,7 +165,7 @@ void PhysicalIndexJoin::Output(ExecutionContext &context, DataChunk &input, Data
 void PhysicalIndexJoin::GetRHSMatches(ExecutionContext &context, DataChunk &input, OperatorState &state_p) const {
 
 	auto &state = state_p.Cast<IndexJoinOperatorState>();
-	auto &art = index->Cast<ART>();
+	auto &art = index.Cast<ART>();
 	;
 
 	// generate the keys for this chunk
@@ -177,11 +177,11 @@ void PhysicalIndexJoin::GetRHSMatches(ExecutionContext &context, DataChunk &inpu
 		if (!state.keys[i].Empty()) {
 			if (fetch_types.empty()) {
 				IndexLock lock;
-				index->InitializeLock(lock);
+				index.InitializeLock(lock);
 				art.SearchEqualJoinNoFetch(state.keys[i], state.result_sizes[i]);
 			} else {
 				IndexLock lock;
-				index->InitializeLock(lock);
+				index.InitializeLock(lock);
 				art.SearchEqual(state.keys[i], (idx_t)-1, state.rhs_rows[i]);
 				state.result_sizes[i] = state.rhs_rows[i].size();
 			}

package/src/duckdb/src/execution/operator/join/physical_join.cpp

@@ -41,7 +41,7 @@ void PhysicalJoin::BuildJoinPipelines(Pipeline &current, MetaPipeline &meta_pipe
 
 	// on the RHS (build side), we construct a child MetaPipeline with this operator as its sink
 	auto child_meta_pipeline = meta_pipeline.CreateChildMetaPipeline(current, &op);
-	child_meta_pipeline->Build(op.children[1].get());
+	child_meta_pipeline->Build(*op.children[1]);
 
 	// continue building the current pipeline on the LHS (probe side)
 	op.children[0]->BuildPipelines(current, meta_pipeline);
@@ -60,18 +60,10 @@ void PhysicalJoin::BuildJoinPipelines(Pipeline &current, MetaPipeline &meta_pipe
 	// Join can become a source operator if it's RIGHT/OUTER, or if the hash join goes out-of-core
 	bool add_child_pipeline = false;
 	auto &join_op = op.Cast<PhysicalJoin>();
-	if (IsRightOuterJoin(join_op.join_type)) {
+	if (IsRightOuterJoin(join_op.join_type) || join_op.type == PhysicalOperatorType::HASH_JOIN) {
 		add_child_pipeline = true;
 	}
 
-	if (join_op.type == PhysicalOperatorType::HASH_JOIN) {
-		auto &hash_join_op = join_op.Cast<PhysicalHashJoin>();
-		hash_join_op.can_go_external = !meta_pipeline.HasRecursiveCTE();
-		if (hash_join_op.can_go_external) {
-			add_child_pipeline = true;
-		}
-	}
-
 	if (add_child_pipeline) {
 		meta_pipeline.CreateChildPipeline(current, &op, last_pipeline);
 	}

package/src/duckdb/src/execution/operator/join/physical_positional_join.cpp

@@ -1,6 +1,5 @@
 #include "duckdb/execution/operator/join/physical_positional_join.hpp"
 
-#include "duckdb/common/types/column_data_collection.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/execution/operator/join/physical_join.hpp"
 

package/src/duckdb/src/execution/operator/order/physical_top_n.cpp

@@ -1,12 +1,12 @@
 #include "duckdb/execution/operator/order/physical_top_n.hpp"
 
 #include "duckdb/common/assert.hpp"
+#include "duckdb/common/sort/sort.hpp"
+#include "duckdb/common/types/row/row_layout.hpp"
 #include "duckdb/common/value_operations/value_operations.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/storage/data_table.hpp"
-#include "duckdb/common/sort/sort.hpp"
-#include "duckdb/common/types/row_layout.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp

@@ -42,6 +42,9 @@ BaseCSVReader::~BaseCSVReader() {
 unique_ptr<CSVFileHandle> BaseCSVReader::OpenCSV(const BufferedCSVReaderOptions &options_p) {
 	auto file_handle = fs.OpenFile(options_p.file_path.c_str(), FileFlags::FILE_FLAGS_READ, FileLockType::NO_LOCK,
 	                               options_p.compression, this->opener);
+	if (file_handle->CanSeek()) {
+		file_handle->Reset();
+	}
 	return make_uniq<CSVFileHandle>(std::move(file_handle));
 }
 

package/src/duckdb/src/execution/operator/persistent/buffered_csv_reader.cpp

@@ -15,6 +15,7 @@
 #include "utf8proc.hpp"
 #include "duckdb/parser/keyword_helper.hpp"
 #include "duckdb/main/error_manager.hpp"
+#include "duckdb/main/client_data.hpp"
 
 #include <algorithm>
 #include <cctype>
@@ -954,10 +955,10 @@ bool BufferedCSVReader::TryParseComplexCSV(DataChunk &insert_chunk, string &erro
 	bool has_quotes = false;
 	uint8_t delimiter_pos = 0, escape_pos = 0, quote_pos = 0;
 	idx_t offset = 0;
-
+	idx_t line_start = 0;
 	// read values into the buffer (if any)
 	if (position >= buffer_size) {
-		if (!ReadBuffer(start)) {
+		if (!ReadBuffer(start, line_start)) {
 			return true;
 		}
 	}
@@ -994,7 +995,7 @@ value_start:
 				goto in_quotes;
 			}
 		}
-	} while (ReadBuffer(start));
+	} while (ReadBuffer(start, line_start));
 	// file ends while scanning for quote/delimiter, go to final state
 	goto final_state;
 normal:
@@ -1011,7 +1012,7 @@ normal:
 				goto add_row;
 			}
 		}
-	} while (ReadBuffer(start));
+	} while (ReadBuffer(start, line_start));
 	goto final_state;
 add_value:
 	AddValue(string_t(buffer.get() + start, position - start - offset), column, escape_positions, has_quotes);
@@ -1019,7 +1020,7 @@ add_value:
 	offset = 0;
 	has_quotes = false;
 	start = ++position;
-	if (position >= buffer_size && !ReadBuffer(start)) {
+	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
 		// file ends right after delimiter, go to final state
 		goto final_state;
 	}
@@ -1029,14 +1030,17 @@ add_row : {
 	bool carriage_return = buffer[position] == '\r';
 	AddValue(string_t(buffer.get() + start, position - start - offset), column, escape_positions, has_quotes);
 	finished_chunk = AddRow(insert_chunk, column, error_message);
+
 	if (!error_message.empty()) {
 		return false;
 	}
 	// increase position by 1 and move start to the new position
 	offset = 0;
 	has_quotes = false;
-	start = ++position;
-	if (position >= buffer_size && !ReadBuffer(start)) {
+	position++;
+	SkipEmptyLines();
+	start = position;
+	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
 		// file ends right after newline, go to final state
 		goto final_state;
 	}
@@ -1069,7 +1073,7 @@ in_quotes:
 				goto handle_escape;
 			}
 		}
-	} while (ReadBuffer(start));
+	} while (ReadBuffer(start, line_start));
 	// still in quoted state at the end of the file, error:
 	error_message = StringUtil::Format("Error in file \"%s\" on line %s: unterminated quotes. (%s)", options.file_path,
 	                                   GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
@@ -1082,7 +1086,7 @@ unquote:
 	delimiter_pos = 0;
 	quote_pos = 0;
 	position++;
-	if (position >= buffer_size && !ReadBuffer(start)) {
+	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
 		// file ends right after unquote, go to final state
 		offset = options.quote.size();
 		goto final_state;
@@ -1116,7 +1120,7 @@ unquote:
 				goto in_quotes;
 			}
 		}
-	} while (ReadBuffer(start));
+	} while (ReadBuffer(start, line_start));
 	error_message = StringUtil::Format(
 	    "Error in file \"%s\" on line %s: quote should be followed by end of value, end of row or another quote. (%s)",
 	    options.file_path, GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
@@ -1142,7 +1146,7 @@ handle_escape:
 				goto in_quotes;
 			}
 		}
-	} while (ReadBuffer(start));
+	} while (ReadBuffer(start, line_start));
 	error_message =
 	    StringUtil::Format("Error in file \"%s\" on line %s: neither QUOTE nor ESCAPE is proceeded by ESCAPE. (%s)",
 	                       options.file_path, GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
@@ -1153,7 +1157,7 @@ carriage_return:
 	if (buffer[position] == '\n') {
 		// newline after carriage return: skip
 		start = ++position;
-		if (position >= buffer_size && !ReadBuffer(start)) {
+		if (position >= buffer_size && !ReadBuffer(start, line_start)) {
 			// file ends right after newline, go to final state
 			goto final_state;
 		}
@@ -1170,6 +1174,7 @@ final_state:
 		// remaining values to be added to the chunk
 		AddValue(string_t(buffer.get() + start, position - start - offset), column, escape_positions, has_quotes);
 		finished_chunk = AddRow(insert_chunk, column, error_message);
+		SkipEmptyLines();
 		if (!error_message.empty()) {
 			return false;
 		}
@@ -1184,6 +1189,18 @@ final_state:
 	return true;
 }
 
+void BufferedCSVReader::SkipEmptyLines() {
+	if (parse_chunk.data.size() == 1) {
+		// Empty lines are null data.
+		return;
+	}
+	for (; position < buffer_size; position++) {
+		if (!StringUtil::CharacterIsNewline(buffer[position])) {
+			return;
+		}
+	}
+}
+
 bool BufferedCSVReader::TryParseSimpleCSV(DataChunk &insert_chunk, string &error_message) {
 	// used for parsing algorithm
 	bool finished_chunk = false;
@@ -1192,12 +1209,14 @@ bool BufferedCSVReader::TryParseSimpleCSV(DataChunk &insert_chunk, string &error
 	bool has_quotes = false;
 	vector<idx_t> escape_positions;
 
+	idx_t line_start = position;
 	// read values into the buffer (if any)
 	if (position >= buffer_size) {
-		if (!ReadBuffer(start)) {
+		if (!ReadBuffer(start, line_start)) {
 			return true;
 		}
 	}
+
 	// start parsing the first value
 	goto value_start;
 value_start:
@@ -1227,7 +1246,7 @@ normal:
 				goto add_row;
 			}
 		}
-	} while (ReadBuffer(start));
+	} while (ReadBuffer(start, line_start));
 	// file ends during normal scan: go to end state
 	goto final_state;
 add_value:
@@ -1236,7 +1255,7 @@ add_value:
 	offset = 0;
 	has_quotes = false;
 	start = ++position;
-	if (position >= buffer_size && !ReadBuffer(start)) {
+	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
 		// file ends right after delimiter, go to final state
 		goto final_state;
 	}
@@ -1249,14 +1268,19 @@ add_row : {
 		return false;
 	}
 	finished_chunk = AddRow(insert_chunk, column, error_message);
+	if (context.client_data->max_line_length < position - line_start) {
+		context.client_data->max_line_length = position - line_start;
+	}
 	if (!error_message.empty()) {
 		return false;
 	}
 	// increase position by 1 and move start to the new position
 	offset = 0;
 	has_quotes = false;
-	start = ++position;
-	if (position >= buffer_size && !ReadBuffer(start)) {
+	position++;
+	start = position;
+	line_start = position;
+	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
 		// file ends right after delimiter, go to final state
 		goto final_state;
 	}
@@ -1265,6 +1289,14 @@ add_row : {
 		goto carriage_return;
 	} else {
 		SetNewLineDelimiter();
+		SkipEmptyLines();
+
+		start = position;
+		line_start = position;
+		if (position >= buffer_size && !ReadBuffer(start, line_start)) {
+			// file ends right after delimiter, go to final state
+			goto final_state;
+		}
 		// \n newline, move to value start
 		if (finished_chunk) {
 			return true;
@@ -1288,7 +1320,7 @@ in_quotes:
 				goto handle_escape;
 			}
 		}
-	} while (ReadBuffer(start));
+	} while (ReadBuffer(start, line_start));
 	// still in quoted state at the end of the file, error:
 	throw InvalidInputException("Error in file \"%s\" on line %s: unterminated quotes. (%s)", options.file_path,
 	                            GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
@@ -1298,7 +1330,7 @@ unquote:
 	// in this state we expect either another quote (entering the quoted state again, and escaping the quote)
 	// or a delimiter/newline, ending the current value and moving on to the next value
 	position++;
-	if (position >= buffer_size && !ReadBuffer(start)) {
+	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
 		// file ends right after unquote, go to final state
 		offset = 1;
 		goto final_state;
@@ -1325,7 +1357,7 @@ handle_escape:
 	/* state: handle_escape */
 	// escape should be followed by a quote or another escape character
 	position++;
-	if (position >= buffer_size && !ReadBuffer(start)) {
+	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
 		error_message = StringUtil::Format(
 		    "Error in file \"%s\" on line %s: neither QUOTE nor ESCAPE is proceeded by ESCAPE. (%s)", options.file_path,
 		    GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
@@ -1347,7 +1379,7 @@ carriage_return:
 		// newline after carriage return: skip
 		// increase position by 1 and move start to the new position
 		start = ++position;
-		if (position >= buffer_size && !ReadBuffer(start)) {
+		if (position >= buffer_size && !ReadBuffer(start, line_start)) {
 			// file ends right after delimiter, go to final state
 			goto final_state;
 		}
@@ -1357,6 +1389,14 @@ carriage_return:
 	if (finished_chunk) {
 		return true;
 	}
+	SkipEmptyLines();
+	start = position;
+	line_start = position;
+	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
+		// file ends right after delimiter, go to final state
+		goto final_state;
+	}
+
 	goto value_start;
 final_state:
 	if (finished_chunk) {
@@ -1367,6 +1407,10 @@ final_state:
 		// remaining values to be added to the chunk
 		AddValue(string_t(buffer.get() + start, position - start - offset), column, escape_positions, has_quotes);
 		finished_chunk = AddRow(insert_chunk, column, error_message);
+		SkipEmptyLines();
+		if (context.client_data->max_line_length < position - line_start) {
+			context.client_data->max_line_length = position - line_start;
+		}
 		if (!error_message.empty()) {
 			return false;
 		}
@@ -1382,7 +1426,10 @@ final_state:
 	return true;
 }
 
-bool BufferedCSVReader::ReadBuffer(idx_t &start) {
+bool BufferedCSVReader::ReadBuffer(idx_t &start, idx_t &line_start) {
+	if (start > buffer_size) {
+		return false;
+	}
 	auto old_buffer = std::move(buffer);
 
 	// the remaining part of the last buffer
@@ -1420,9 +1467,11 @@ bool BufferedCSVReader::ReadBuffer(idx_t &start) {
 	if (!bom_checked) {
 		bom_checked = true;
 		if (read_count >= 3 && buffer[0] == '\xEF' && buffer[1] == '\xBB' && buffer[2] == '\xBF') {
+			start += 3;
 			position += 3;
 		}
 	}
+	line_start = start;
 
 	return read_count > 0;
 }

package/src/duckdb/src/execution/operator/persistent/csv_buffer.cpp

@@ -4,8 +4,8 @@
 namespace duckdb {
 
 CSVBuffer::CSVBuffer(ClientContext &context, idx_t buffer_size_p, CSVFileHandle &file_handle,
-                     idx_t &global_csv_current_position)
-    : context(context), first_buffer(true) {
+                     idx_t &global_csv_current_position, idx_t file_number_p)
+    : context(context), first_buffer(true), file_number(file_number_p) {
 	this->handle = AllocateBuffer(buffer_size_p);
 
 	auto buffer = Ptr();
@@ -19,23 +19,23 @@ CSVBuffer::CSVBuffer(ClientContext &context, idx_t buffer_size_p, CSVFileHandle
 }
 
 CSVBuffer::CSVBuffer(ClientContext &context, BufferHandle buffer_p, idx_t buffer_size_p, idx_t actual_size_p,
-                     bool final_buffer, idx_t global_csv_current_position)
+                     bool final_buffer, idx_t global_csv_current_position, idx_t file_number_p)
     : context(context), handle(std::move(buffer_p)), actual_size(actual_size_p), last_buffer(final_buffer),
-      global_csv_start(global_csv_current_position) {
+      global_csv_start(global_csv_current_position), file_number(file_number_p) {
 }
 
-unique_ptr<CSVBuffer> CSVBuffer::Next(CSVFileHandle &file_handle, idx_t buffer_size,
-                                      idx_t &global_csv_current_position) {
-	if (file_handle.FinishedReading()) {
-		// this was the last buffer
-		return nullptr;
-	}
-
+unique_ptr<CSVBuffer> CSVBuffer::Next(CSVFileHandle &file_handle, idx_t buffer_size, idx_t &global_csv_current_position,
+                                      idx_t file_number_p) {
 	auto next_buffer = AllocateBuffer(buffer_size);
 	idx_t next_buffer_actual_size = file_handle.Read(next_buffer.Ptr(), buffer_size);
+	if (next_buffer_actual_size == 0) {
+		// We are done reading
+		return nullptr;
+	}
 
-	auto next_csv_buffer = make_uniq<CSVBuffer>(context, std::move(next_buffer), buffer_size, next_buffer_actual_size,
-	                                            file_handle.FinishedReading(), global_csv_current_position);
+	auto next_csv_buffer =
+	    make_uniq<CSVBuffer>(context, std::move(next_buffer), buffer_size, next_buffer_actual_size,
+	                         file_handle.FinishedReading(), global_csv_current_position, file_number_p);
 	global_csv_current_position += next_buffer_actual_size;
 	return next_csv_buffer;
 }
@@ -65,4 +65,8 @@ idx_t CSVBuffer::GetCSVGlobalStart() {
 	return global_csv_start;
 }
 
+idx_t CSVBuffer::GetFileNumber() {
+	return file_number;
+}
+
 } // namespace duckdb

package/src/duckdb/src/execution/operator/persistent/csv_reader_options.cpp

@@ -74,11 +74,6 @@ void BufferedCSVReaderOptions::SetEscape(const string &input) {
 	this->has_escape = true;
 }
 
-void BufferedCSVReaderOptions::SetParallel(bool use_parallel) {
-	this->has_parallel = true;
-	this->use_parallel = use_parallel;
-}
-
 void BufferedCSVReaderOptions::SetDelimiter(const string &input) {
 	this->delimiter = StringUtil::Replace(input, "\\t", "\t");
 	this->has_delimiter = true;
@@ -126,8 +121,6 @@ void BufferedCSVReaderOptions::SetReadOption(const string &loption, const Value
 	}
 	if (loption == "auto_detect") {
 		auto_detect = ParseBoolean(value, loption);
-	} else if (loption == "parallel") {
-		SetParallel(ParseBoolean(value, loption));
 	} else if (loption == "sample_size") {
 		int64_t sample_size = ParseInteger(value, loption);
 		if (sample_size < 1 && sample_size != -1) {