duckdb 0.3.5-dev1125.0 → 0.3.5-dev1167.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.3.5-dev1125.0",
+  "version": "0.3.5-dev1167.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -69189,6 +69189,10 @@ public:
 		return plain_file_source;
 	}
 
+	bool OnDiskFile() {
+		return file_handle->OnDiskFile();
+	}
+
 	idx_t FileSize() {
 		return file_size;
 	}
@@ -70771,13 +70775,19 @@ bool BufferedCSVReader::ReadBuffer(idx_t &start) {
 
 	// the remaining part of the last buffer
 	idx_t remaining = buffer_size - start;
-	idx_t buffer_read_size = INITIAL_BUFFER_SIZE;
+
+	bool large_buffers = mode == ParserMode::PARSING && !file_handle->OnDiskFile() && file_handle->CanSeek();
+	idx_t buffer_read_size = large_buffers ? INITIAL_BUFFER_SIZE_LARGE : INITIAL_BUFFER_SIZE;
+
 	while (remaining > buffer_read_size) {
 		buffer_read_size *= 2;
 	}
-	if (remaining + buffer_read_size > options.maximum_line_size) {
+
+	// Check line length
+	if (remaining > options.maximum_line_size) {
 		throw InvalidInputException("Maximum line size of %llu bytes exceeded!", options.maximum_line_size);
 	}
+
 	buffer = unique_ptr<char[]>(new char[buffer_read_size + remaining + 1]);
 	buffer_size = remaining + buffer_read_size;
 	if (remaining > 0) {

package/src/duckdb.hpp

@@ -11,8 +11,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 #pragma once
 #define DUCKDB_AMALGAMATION 1
 #define DUCKDB_AMALGAMATION_EXTENDED 1
-#define DUCKDB_SOURCE_ID "e2e7520ec"
-#define DUCKDB_VERSION "v0.3.5-dev1125"
+#define DUCKDB_SOURCE_ID "9c978fb3c"
+#define DUCKDB_VERSION "v0.3.5-dev1167"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -22591,6 +22591,7 @@ struct BufferedCSVReaderOptions {
 	//! How many leading rows to skip
 	idx_t skip_rows = 0;
 	//! Maximum CSV line size: specified because if we reach this amount, we likely have wrong delimiters (default: 2MB)
+	//! note that this is the guaranteed line length that will succeed, longer lines may be accepted if slightly above
 	idx_t maximum_line_size = 2097152;
 	//! Whether or not header names shall be normalized
 	bool normalize_names = false;
@@ -22644,6 +22645,8 @@ enum class ParserMode : uint8_t { PARSING = 0, SNIFFING_DIALECT = 1, SNIFFING_DA
 class BufferedCSVReader {
 	//! Initial buffer read size; can be extended for long lines
 	static constexpr idx_t INITIAL_BUFFER_SIZE = 16384;
+	//! Larger buffer size for non disk files
+	static constexpr idx_t INITIAL_BUFFER_SIZE_LARGE = 10000000; // 10MB
 	ParserMode mode;
 
 public: