duckdb 0.7.2-dev2144.0 → 0.7.2-dev2233.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.7.2-dev2144.0",
+  "version": "0.7.2-dev2233.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/extension/parquet/column_reader.cpp

@@ -647,6 +647,7 @@ void StringColumnReader::PrepareDeltaLengthByteArray(ResizeableBuffer &buffer) {
 	auto length_data = (uint32_t *)length_buffer->ptr;
 	byte_array_data = make_uniq<Vector>(LogicalType::VARCHAR, value_count);
 	byte_array_count = value_count;
+	delta_offset = 0;
 	auto string_data = FlatVector::GetData<string_t>(*byte_array_data);
 	for (idx_t i = 0; i < value_count; i++) {
 		auto str_len = length_data[i];
@@ -674,6 +675,7 @@ void StringColumnReader::PrepareDeltaByteArray(ResizeableBuffer &buffer) {
 	auto suffix_data = (uint32_t *)suffix_buffer->ptr;
 	byte_array_data = make_uniq<Vector>(LogicalType::VARCHAR, prefix_count);
 	byte_array_count = prefix_count;
+	delta_offset = 0;
 	auto string_data = FlatVector::GetData<string_t>(*byte_array_data);
 	for (idx_t i = 0; i < prefix_count; i++) {
 		auto str_len = prefix_data[i] + suffix_data[i];
@@ -715,6 +717,7 @@ void StringColumnReader::DeltaByteArray(uint8_t *defines, idx_t num_values, parq
 			delta_offset++;
 		}
 	}
+	StringVector::AddHeapReference(result, *byte_array_data);
 }
 
 class ParquetStringVectorBuffer : public VectorBuffer {

package/src/duckdb/src/common/types/column/column_data_collection.cpp

@@ -970,7 +970,7 @@ struct ValueResultEquals {
 };
 
 bool ColumnDataCollection::ResultEquals(const ColumnDataCollection &left, const ColumnDataCollection &right,
-                                        string &error_message) {
+                                        string &error_message, bool ordered) {
 	if (left.ColumnCount() != right.ColumnCount()) {
 		error_message = "Column count mismatch";
 		return false;
@@ -985,6 +985,7 @@ bool ColumnDataCollection::ResultEquals(const ColumnDataCollection &left, const
 		for (idx_t c = 0; c < left.ColumnCount(); c++) {
 			auto lvalue = left_rows.GetValue(c, r);
 			auto rvalue = right_rows.GetValue(c, r);
+
 			if (!Value::DefaultValuesAreEqual(lvalue, rvalue)) {
 				error_message =
 				    StringUtil::Format("%s <> %s (row: %lld, col: %lld)\n", lvalue.ToString(), rvalue.ToString(), r, c);
@@ -992,7 +993,11 @@ bool ColumnDataCollection::ResultEquals(const ColumnDataCollection &left, const
 			}
 		}
 		if (!error_message.empty()) {
-			break;
+			if (ordered) {
+				return false;
+			} else {
+				break;
+			}
 		}
 	}
 	if (!error_message.empty()) {

package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp

@@ -42,6 +42,9 @@ BaseCSVReader::~BaseCSVReader() {
 unique_ptr<CSVFileHandle> BaseCSVReader::OpenCSV(const BufferedCSVReaderOptions &options_p) {
 	auto file_handle = fs.OpenFile(options_p.file_path.c_str(), FileFlags::FILE_FLAGS_READ, FileLockType::NO_LOCK,
 	                               options_p.compression, this->opener);
+	if (file_handle->CanSeek()) {
+		file_handle->Reset();
+	}
 	return make_uniq<CSVFileHandle>(std::move(file_handle));
 }
 

package/src/duckdb/src/execution/operator/persistent/buffered_csv_reader.cpp

@@ -15,6 +15,7 @@
 #include "utf8proc.hpp"
 #include "duckdb/parser/keyword_helper.hpp"
 #include "duckdb/main/error_manager.hpp"
+#include "duckdb/main/client_data.hpp"
 
 #include <algorithm>
 #include <cctype>
@@ -954,10 +955,10 @@ bool BufferedCSVReader::TryParseComplexCSV(DataChunk &insert_chunk, string &erro
 	bool has_quotes = false;
 	uint8_t delimiter_pos = 0, escape_pos = 0, quote_pos = 0;
 	idx_t offset = 0;
-
+	idx_t line_start = 0;
 	// read values into the buffer (if any)
 	if (position >= buffer_size) {
-		if (!ReadBuffer(start)) {
+		if (!ReadBuffer(start, line_start)) {
 			return true;
 		}
 	}
@@ -994,7 +995,7 @@ value_start:
 				goto in_quotes;
 			}
 		}
-	} while (ReadBuffer(start));
+	} while (ReadBuffer(start, line_start));
 	// file ends while scanning for quote/delimiter, go to final state
 	goto final_state;
 normal:
@@ -1011,7 +1012,7 @@ normal:
 				goto add_row;
 			}
 		}
-	} while (ReadBuffer(start));
+	} while (ReadBuffer(start, line_start));
 	goto final_state;
 add_value:
 	AddValue(string_t(buffer.get() + start, position - start - offset), column, escape_positions, has_quotes);
@@ -1019,7 +1020,7 @@ add_value:
 	offset = 0;
 	has_quotes = false;
 	start = ++position;
-	if (position >= buffer_size && !ReadBuffer(start)) {
+	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
 		// file ends right after delimiter, go to final state
 		goto final_state;
 	}
@@ -1029,14 +1030,17 @@ add_row : {
 	bool carriage_return = buffer[position] == '\r';
 	AddValue(string_t(buffer.get() + start, position - start - offset), column, escape_positions, has_quotes);
 	finished_chunk = AddRow(insert_chunk, column, error_message);
+
 	if (!error_message.empty()) {
 		return false;
 	}
 	// increase position by 1 and move start to the new position
 	offset = 0;
 	has_quotes = false;
-	start = ++position;
-	if (position >= buffer_size && !ReadBuffer(start)) {
+	position++;
+	SkipEmptyLines();
+	start = position;
+	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
 		// file ends right after newline, go to final state
 		goto final_state;
 	}
@@ -1069,7 +1073,7 @@ in_quotes:
 				goto handle_escape;
 			}
 		}
-	} while (ReadBuffer(start));
+	} while (ReadBuffer(start, line_start));
 	// still in quoted state at the end of the file, error:
 	error_message = StringUtil::Format("Error in file \"%s\" on line %s: unterminated quotes. (%s)", options.file_path,
 	                                   GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
@@ -1082,7 +1086,7 @@ unquote:
 	delimiter_pos = 0;
 	quote_pos = 0;
 	position++;
-	if (position >= buffer_size && !ReadBuffer(start)) {
+	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
 		// file ends right after unquote, go to final state
 		offset = options.quote.size();
 		goto final_state;
@@ -1116,7 +1120,7 @@ unquote:
 				goto in_quotes;
 			}
 		}
-	} while (ReadBuffer(start));
+	} while (ReadBuffer(start, line_start));
 	error_message = StringUtil::Format(
 	    "Error in file \"%s\" on line %s: quote should be followed by end of value, end of row or another quote. (%s)",
 	    options.file_path, GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
@@ -1142,7 +1146,7 @@ handle_escape:
 				goto in_quotes;
 			}
 		}
-	} while (ReadBuffer(start));
+	} while (ReadBuffer(start, line_start));
 	error_message =
 	    StringUtil::Format("Error in file \"%s\" on line %s: neither QUOTE nor ESCAPE is proceeded by ESCAPE. (%s)",
 	                       options.file_path, GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
@@ -1153,7 +1157,7 @@ carriage_return:
 	if (buffer[position] == '\n') {
 		// newline after carriage return: skip
 		start = ++position;
-		if (position >= buffer_size && !ReadBuffer(start)) {
+		if (position >= buffer_size && !ReadBuffer(start, line_start)) {
 			// file ends right after newline, go to final state
 			goto final_state;
 		}
@@ -1170,6 +1174,7 @@ final_state:
 		// remaining values to be added to the chunk
 		AddValue(string_t(buffer.get() + start, position - start - offset), column, escape_positions, has_quotes);
 		finished_chunk = AddRow(insert_chunk, column, error_message);
+		SkipEmptyLines();
 		if (!error_message.empty()) {
 			return false;
 		}
@@ -1184,6 +1189,18 @@ final_state:
 	return true;
 }
 
+void BufferedCSVReader::SkipEmptyLines() {
+	if (parse_chunk.data.size() == 1) {
+		// Empty lines are null data.
+		return;
+	}
+	for (; position < buffer_size; position++) {
+		if (!StringUtil::CharacterIsNewline(buffer[position])) {
+			return;
+		}
+	}
+}
+
 bool BufferedCSVReader::TryParseSimpleCSV(DataChunk &insert_chunk, string &error_message) {
 	// used for parsing algorithm
 	bool finished_chunk = false;
@@ -1192,12 +1209,14 @@ bool BufferedCSVReader::TryParseSimpleCSV(DataChunk &insert_chunk, string &error
 	bool has_quotes = false;
 	vector<idx_t> escape_positions;
 
+	idx_t line_start = position;
 	// read values into the buffer (if any)
 	if (position >= buffer_size) {
-		if (!ReadBuffer(start)) {
+		if (!ReadBuffer(start, line_start)) {
 			return true;
 		}
 	}
+
 	// start parsing the first value
 	goto value_start;
 value_start:
@@ -1227,7 +1246,7 @@ normal:
 				goto add_row;
 			}
 		}
-	} while (ReadBuffer(start));
+	} while (ReadBuffer(start, line_start));
 	// file ends during normal scan: go to end state
 	goto final_state;
 add_value:
@@ -1236,7 +1255,7 @@ add_value:
 	offset = 0;
 	has_quotes = false;
 	start = ++position;
-	if (position >= buffer_size && !ReadBuffer(start)) {
+	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
 		// file ends right after delimiter, go to final state
 		goto final_state;
 	}
@@ -1249,14 +1268,19 @@ add_row : {
 		return false;
 	}
 	finished_chunk = AddRow(insert_chunk, column, error_message);
+	if (context.client_data->max_line_length < position - line_start) {
+		context.client_data->max_line_length = position - line_start;
+	}
 	if (!error_message.empty()) {
 		return false;
 	}
 	// increase position by 1 and move start to the new position
 	offset = 0;
 	has_quotes = false;
-	start = ++position;
-	if (position >= buffer_size && !ReadBuffer(start)) {
+	position++;
+	start = position;
+	line_start = position;
+	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
 		// file ends right after delimiter, go to final state
 		goto final_state;
 	}
@@ -1265,6 +1289,14 @@ add_row : {
 		goto carriage_return;
 	} else {
 		SetNewLineDelimiter();
+		SkipEmptyLines();
+
+		start = position;
+		line_start = position;
+		if (position >= buffer_size && !ReadBuffer(start, line_start)) {
+			// file ends right after delimiter, go to final state
+			goto final_state;
+		}
 		// \n newline, move to value start
 		if (finished_chunk) {
 			return true;
@@ -1288,7 +1320,7 @@ in_quotes:
 				goto handle_escape;
 			}
 		}
-	} while (ReadBuffer(start));
+	} while (ReadBuffer(start, line_start));
 	// still in quoted state at the end of the file, error:
 	throw InvalidInputException("Error in file \"%s\" on line %s: unterminated quotes. (%s)", options.file_path,
 	                            GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
@@ -1298,7 +1330,7 @@ unquote:
 	// in this state we expect either another quote (entering the quoted state again, and escaping the quote)
 	// or a delimiter/newline, ending the current value and moving on to the next value
 	position++;
-	if (position >= buffer_size && !ReadBuffer(start)) {
+	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
 		// file ends right after unquote, go to final state
 		offset = 1;
 		goto final_state;
@@ -1325,7 +1357,7 @@ handle_escape:
 	/* state: handle_escape */
 	// escape should be followed by a quote or another escape character
 	position++;
-	if (position >= buffer_size && !ReadBuffer(start)) {
+	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
 		error_message = StringUtil::Format(
 		    "Error in file \"%s\" on line %s: neither QUOTE nor ESCAPE is proceeded by ESCAPE. (%s)", options.file_path,
 		    GetLineNumberStr(linenr, linenr_estimated).c_str(), options.ToString());
@@ -1347,7 +1379,7 @@ carriage_return:
 		// newline after carriage return: skip
 		// increase position by 1 and move start to the new position
 		start = ++position;
-		if (position >= buffer_size && !ReadBuffer(start)) {
+		if (position >= buffer_size && !ReadBuffer(start, line_start)) {
 			// file ends right after delimiter, go to final state
 			goto final_state;
 		}
@@ -1357,6 +1389,14 @@ carriage_return:
 	if (finished_chunk) {
 		return true;
 	}
+	SkipEmptyLines();
+	start = position;
+	line_start = position;
+	if (position >= buffer_size && !ReadBuffer(start, line_start)) {
+		// file ends right after delimiter, go to final state
+		goto final_state;
+	}
+
 	goto value_start;
 final_state:
 	if (finished_chunk) {
@@ -1367,6 +1407,10 @@ final_state:
 		// remaining values to be added to the chunk
 		AddValue(string_t(buffer.get() + start, position - start - offset), column, escape_positions, has_quotes);
 		finished_chunk = AddRow(insert_chunk, column, error_message);
+		SkipEmptyLines();
+		if (context.client_data->max_line_length < position - line_start) {
+			context.client_data->max_line_length = position - line_start;
+		}
 		if (!error_message.empty()) {
 			return false;
 		}
@@ -1382,7 +1426,10 @@ final_state:
 	return true;
 }
 
-bool BufferedCSVReader::ReadBuffer(idx_t &start) {
+bool BufferedCSVReader::ReadBuffer(idx_t &start, idx_t &line_start) {
+	if (start > buffer_size) {
+		return false;
+	}
 	auto old_buffer = std::move(buffer);
 
 	// the remaining part of the last buffer
@@ -1420,9 +1467,11 @@ bool BufferedCSVReader::ReadBuffer(idx_t &start) {
 	if (!bom_checked) {
 		bom_checked = true;
 		if (read_count >= 3 && buffer[0] == '\xEF' && buffer[1] == '\xBB' && buffer[2] == '\xBF') {
+			start += 3;
 			position += 3;
 		}
 	}
+	line_start = start;
 
 	return read_count > 0;
 }

package/src/duckdb/src/execution/operator/persistent/csv_buffer.cpp

@@ -4,8 +4,8 @@
 namespace duckdb {
 
 CSVBuffer::CSVBuffer(ClientContext &context, idx_t buffer_size_p, CSVFileHandle &file_handle,
-                     idx_t &global_csv_current_position)
-    : context(context), first_buffer(true) {
+                     idx_t &global_csv_current_position, idx_t file_number_p)
+    : context(context), first_buffer(true), file_number(file_number_p) {
 	this->handle = AllocateBuffer(buffer_size_p);
 
 	auto buffer = Ptr();
@@ -19,23 +19,23 @@ CSVBuffer::CSVBuffer(ClientContext &context, idx_t buffer_size_p, CSVFileHandle
 }
 
 CSVBuffer::CSVBuffer(ClientContext &context, BufferHandle buffer_p, idx_t buffer_size_p, idx_t actual_size_p,
-                     bool final_buffer, idx_t global_csv_current_position)
+                     bool final_buffer, idx_t global_csv_current_position, idx_t file_number_p)
     : context(context), handle(std::move(buffer_p)), actual_size(actual_size_p), last_buffer(final_buffer),
-      global_csv_start(global_csv_current_position) {
+      global_csv_start(global_csv_current_position), file_number(file_number_p) {
 }
 
-unique_ptr<CSVBuffer> CSVBuffer::Next(CSVFileHandle &file_handle, idx_t buffer_size,
-                                      idx_t &global_csv_current_position) {
-	if (file_handle.FinishedReading()) {
-		// this was the last buffer
-		return nullptr;
-	}
-
+unique_ptr<CSVBuffer> CSVBuffer::Next(CSVFileHandle &file_handle, idx_t buffer_size, idx_t &global_csv_current_position,
+                                      idx_t file_number_p) {
 	auto next_buffer = AllocateBuffer(buffer_size);
 	idx_t next_buffer_actual_size = file_handle.Read(next_buffer.Ptr(), buffer_size);
+	if (next_buffer_actual_size == 0) {
+		// We are done reading
+		return nullptr;
+	}
 
-	auto next_csv_buffer = make_uniq<CSVBuffer>(context, std::move(next_buffer), buffer_size, next_buffer_actual_size,
-	                                            file_handle.FinishedReading(), global_csv_current_position);
+	auto next_csv_buffer =
+	    make_uniq<CSVBuffer>(context, std::move(next_buffer), buffer_size, next_buffer_actual_size,
+	                         file_handle.FinishedReading(), global_csv_current_position, file_number_p);
 	global_csv_current_position += next_buffer_actual_size;
 	return next_csv_buffer;
 }
@@ -65,4 +65,8 @@ idx_t CSVBuffer::GetCSVGlobalStart() {
 	return global_csv_start;
 }
 
+idx_t CSVBuffer::GetFileNumber() {
+	return file_number;
+}
+
 } // namespace duckdb

package/src/duckdb/src/execution/operator/persistent/csv_reader_options.cpp

@@ -74,11 +74,6 @@ void BufferedCSVReaderOptions::SetEscape(const string &input) {
 	this->has_escape = true;
 }
 
-void BufferedCSVReaderOptions::SetParallel(bool use_parallel) {
-	this->has_parallel = true;
-	this->use_parallel = use_parallel;
-}
-
 void BufferedCSVReaderOptions::SetDelimiter(const string &input) {
 	this->delimiter = StringUtil::Replace(input, "\\t", "\t");
 	this->has_delimiter = true;
@@ -126,8 +121,6 @@ void BufferedCSVReaderOptions::SetReadOption(const string &loption, const Value
 	}
 	if (loption == "auto_detect") {
 		auto_detect = ParseBoolean(value, loption);
-	} else if (loption == "parallel") {
-		SetParallel(ParseBoolean(value, loption));
 	} else if (loption == "sample_size") {
 		int64_t sample_size = ParseInteger(value, loption);
 		if (sample_size < 1 && sample_size != -1) {