duckdb 0.8.2-dev3989.0 → 0.8.2-dev4126.0

package/binding.gyp

@@ -61,6 +61,7 @@
                 "src/duckdb/ub_src_execution.cpp", 
                 "src/duckdb/ub_src_execution_expression_executor.cpp", 
                 "src/duckdb/ub_src_execution_index_art.cpp", 
+                "src/duckdb/ub_src_execution_index.cpp", 
                 "src/duckdb/ub_src_execution_nested_loop_join.cpp", 
                 "src/duckdb/ub_src_execution_operator_aggregate.cpp", 
                 "src/duckdb/ub_src_execution_operator_csv_scanner.cpp", 
@@ -249,18 +250,18 @@
                 "src/duckdb/third_party/zstd/compress/zstd_lazy.cpp", 
                 "src/duckdb/third_party/zstd/compress/zstd_ldm.cpp", 
                 "src/duckdb/third_party/zstd/compress/zstd_opt.cpp", 
-                "src/duckdb/extension/icu/./icu-strptime.cpp", 
+                "src/duckdb/extension/icu/./icu-timezone.cpp", 
                 "src/duckdb/extension/icu/./icu-datepart.cpp", 
-                "src/duckdb/extension/icu/./icu-datesub.cpp", 
-                "src/duckdb/extension/icu/./icu-table-range.cpp", 
-                "src/duckdb/extension/icu/./icu-datetrunc.cpp", 
                 "src/duckdb/extension/icu/./icu-timebucket.cpp", 
-                "src/duckdb/extension/icu/./icu-dateadd.cpp", 
+                "src/duckdb/extension/icu/./icu-datesub.cpp", 
                 "src/duckdb/extension/icu/./icu-list-range.cpp", 
-                "src/duckdb/extension/icu/./icu-timezone.cpp", 
+                "src/duckdb/extension/icu/./icu-makedate.cpp", 
                 "src/duckdb/extension/icu/./icu-datefunc.cpp", 
+                "src/duckdb/extension/icu/./icu-datetrunc.cpp", 
+                "src/duckdb/extension/icu/./icu-dateadd.cpp", 
+                "src/duckdb/extension/icu/./icu-table-range.cpp", 
                 "src/duckdb/extension/icu/./icu_extension.cpp", 
-                "src/duckdb/extension/icu/./icu-makedate.cpp", 
+                "src/duckdb/extension/icu/./icu-strptime.cpp", 
                 "src/duckdb/ub_extension_icu_third_party_icu_common.cpp", 
                 "src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp", 
                 "src/duckdb/extension/icu/third_party/icu/stubdata/stubdata.cpp", 

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.8.2-dev3989.0",
+  "version": "0.8.2-dev4126.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/extension/json/buffered_json_reader.cpp

@@ -2,9 +2,10 @@
 
 #include "duckdb/common/field_writer.hpp"
 #include "duckdb/common/file_opener.hpp"
-#include "duckdb/common/printer.hpp"
-#include "duckdb/common/serializer/format_serializer.hpp"
 #include "duckdb/common/serializer/format_deserializer.hpp"
+#include "duckdb/common/serializer/format_serializer.hpp"
+
+#include <utility>
 
 namespace duckdb {
 
@@ -37,11 +38,24 @@ bool JSONFileHandle::IsOpen() const {
 }
 
 void JSONFileHandle::Close() {
-	if (file_handle) {
+	if (IsOpen()) {
 		file_handle->Close();
 		file_handle = nullptr;
 	}
-	cached_buffers.clear();
+}
+
+void JSONFileHandle::Reset() {
+	D_ASSERT(RequestedReadsComplete());
+	read_position = 0;
+	requested_reads = 0;
+	actual_reads = 0;
+	if (IsOpen() && plain_file_source) {
+		file_handle->Reset();
+	}
+}
+
+bool JSONFileHandle::RequestedReadsComplete() {
+	return requested_reads == actual_reads;
 }
 
 idx_t JSONFileHandle::FileSize() const {
@@ -56,12 +70,9 @@ bool JSONFileHandle::CanSeek() const {
 	return can_seek;
 }
 
-void JSONFileHandle::Seek(idx_t position) {
-	file_handle->Seek(position);
-}
-
 idx_t JSONFileHandle::GetPositionAndSize(idx_t &position, idx_t requested_size) {
 	D_ASSERT(requested_size != 0);
+
 	position = read_position;
 	auto actual_size = MinValue<idx_t>(requested_size, Remaining());
 	read_position += actual_size;
@@ -77,15 +88,18 @@ void JSONFileHandle::ReadAtPosition(char *pointer, idx_t size, idx_t position, b
 	if (plain_file_source) {
 		file_handle->Read(pointer, size, position);
 		actual_reads++;
+
 		return;
 	}
 
 	if (sample_run) { // Cache the buffer
 		file_handle->Read(pointer, size, position);
 		actual_reads++;
+
 		cached_buffers.emplace_back(allocator.Allocate(size));
 		memcpy(cached_buffers.back().get(), pointer, size);
 		cached_size += size;
+
 		return;
 	}
 
@@ -93,6 +107,7 @@ void JSONFileHandle::ReadAtPosition(char *pointer, idx_t size, idx_t position, b
 		ReadFromCache(pointer, size, position);
 		actual_reads++;
 	}
+
 	if (size != 0) {
 		file_handle->Read(pointer, size, position);
 		actual_reads++;
@@ -128,6 +143,19 @@ idx_t JSONFileHandle::Read(char *pointer, idx_t requested_size, bool sample_run)
 	return actual_size;
 }
 
+idx_t JSONFileHandle::ReadInternal(char *pointer, const idx_t requested_size) {
+	// Deal with reading from pipes
+	idx_t total_read_size = 0;
+	while (total_read_size < requested_size) {
+		auto read_size = file_handle->Read(pointer + total_read_size, requested_size - total_read_size);
+		if (read_size == 0) {
+			break;
+		}
+		total_read_size += read_size;
+	}
+	return total_read_size;
+}
+
 idx_t JSONFileHandle::ReadFromCache(char *&pointer, idx_t &size, idx_t &position) {
 	idx_t read_size = 0;
 	idx_t total_offset = 0;
@@ -154,35 +182,27 @@ idx_t JSONFileHandle::ReadFromCache(char *&pointer, idx_t &size, idx_t &position
 	return read_size;
 }
 
-idx_t JSONFileHandle::ReadInternal(char *pointer, const idx_t requested_size) {
-	// Deal with reading from pipes
-	idx_t total_read_size = 0;
-	while (total_read_size < requested_size) {
-		auto read_size = file_handle->Read(pointer + total_read_size, requested_size - total_read_size);
-		if (read_size == 0) {
-			break;
-		}
-		total_read_size += read_size;
-	}
-	return total_read_size;
-}
-
 BufferedJSONReader::BufferedJSONReader(ClientContext &context, BufferedJSONReaderOptions options_p, string file_name_p)
-    : context(context), options(options_p), file_name(std::move(file_name_p)), buffer_index(0) {
+    : context(context), options(std::move(options_p)), file_name(std::move(file_name_p)), buffer_index(0),
+      thrown(false) {
 }
 
 void BufferedJSONReader::OpenJSONFile() {
-	D_ASSERT(!IsDone());
+	D_ASSERT(!IsOpen());
 	lock_guard<mutex> guard(lock);
 	auto &file_system = FileSystem::GetFileSystem(context);
 	auto regular_file_handle =
 	    file_system.OpenFile(file_name.c_str(), FileFlags::FILE_FLAGS_READ, FileLockType::NO_LOCK, options.compression);
 	file_handle = make_uniq<JSONFileHandle>(std::move(regular_file_handle), BufferAllocator::Get(context));
+	Reset();
 }
 
 void BufferedJSONReader::CloseJSONFile() {
 	while (true) {
 		lock_guard<mutex> guard(lock);
+		if (!file_handle->IsOpen()) {
+			return; // Already closed
+		}
 		if (file_handle->RequestedReadsComplete()) {
 			file_handle->Close();
 			break;
@@ -190,13 +210,22 @@ void BufferedJSONReader::CloseJSONFile() {
 	}
 }
 
-bool BufferedJSONReader::IsOpen() const {
+void BufferedJSONReader::Reset() {
+	buffer_index = 0;
+	buffer_map.clear();
+	buffer_line_or_object_counts.clear();
+	if (HasFileHandle()) {
+		file_handle->Reset();
+	}
+}
+
+bool BufferedJSONReader::HasFileHandle() const {
 	return file_handle != nullptr;
 }
 
-bool BufferedJSONReader::IsDone() const {
-	if (file_handle) {
-		return !file_handle->IsOpen();
+bool BufferedJSONReader::IsOpen() const {
+	if (HasFileHandle()) {
+		return file_handle->IsOpen();
 	}
 	return false;
 }
@@ -205,10 +234,6 @@ BufferedJSONReaderOptions &BufferedJSONReader::GetOptions() {
 	return options;
 }
 
-const BufferedJSONReaderOptions &BufferedJSONReader::GetOptions() const {
-	return options;
-}
-
 JSONFormat BufferedJSONReader::GetFormat() const {
 	return options.format;
 }
@@ -232,6 +257,7 @@ const string &BufferedJSONReader::GetFileName() const {
 }
 
 JSONFileHandle &BufferedJSONReader::GetFileHandle() const {
+	D_ASSERT(HasFileHandle());
 	return *file_handle;
 }
 
@@ -240,7 +266,7 @@ void BufferedJSONReader::InsertBuffer(idx_t buffer_idx, unique_ptr<JSONBufferHan
 	buffer_map.insert(make_pair(buffer_idx, std::move(buffer)));
 }
 
-JSONBufferHandle *BufferedJSONReader::GetBuffer(idx_t buffer_idx) {
+optional_ptr<JSONBufferHandle> BufferedJSONReader::GetBuffer(idx_t buffer_idx) {
 	lock_guard<mutex> guard(lock);
 	auto it = buffer_map.find(buffer_idx);
 	return it == buffer_map.end() ? nullptr : it->second.get();
@@ -268,22 +294,28 @@ void BufferedJSONReader::SetBufferLineOrObjectCount(idx_t index, idx_t count) {
 idx_t BufferedJSONReader::GetLineNumber(idx_t buf_index, idx_t line_or_object_in_buf) {
 	D_ASSERT(options.format != JSONFormat::AUTO_DETECT);
 	while (true) {
-		lock_guard<mutex> guard(lock);
 		idx_t line = line_or_object_in_buf;
 		bool can_throw = true;
-		for (idx_t b_idx = 0; b_idx < buf_index; b_idx++) {
-			if (buffer_line_or_object_counts[b_idx] == -1) {
-				can_throw = false;
-				break;
-			} else {
-				line += buffer_line_or_object_counts[b_idx];
+		{
+			lock_guard<mutex> guard(lock);
+			if (thrown) {
+				return DConstants::INVALID_INDEX;
+			}
+			for (idx_t b_idx = 0; b_idx < buf_index; b_idx++) {
+				if (buffer_line_or_object_counts[b_idx] == -1) {
+					can_throw = false;
+					break;
+				} else {
+					line += buffer_line_or_object_counts[b_idx];
+					thrown = true;
+				}
 			}
 		}
-		if (!can_throw) {
-			continue;
+		if (can_throw) {
+			// SQL uses 1-based indexing so I guess we will do that in our exception here as well
+			return line + 1;
 		}
-		// SQL uses 1-based indexing so I guess we will do that in our exception here as well
-		return line + 1;
+		TaskScheduler::YieldThread();
 	}
 }
 
@@ -304,41 +336,11 @@ void BufferedJSONReader::ThrowTransformError(idx_t buf_index, idx_t line_or_obje
 }
 
 double BufferedJSONReader::GetProgress() const {
-	if (IsOpen()) {
+	if (HasFileHandle()) {
 		return 100.0 - 100.0 * double(file_handle->Remaining()) / double(file_handle->FileSize());
 	} else {
 		return 0;
 	}
 }
 
-void BufferedJSONReader::Reset() {
-	buffer_index = 0;
-	buffer_map.clear();
-	buffer_line_or_object_counts.clear();
-
-	if (!file_handle) {
-		return;
-	}
-
-	if (file_handle->CanSeek()) {
-		file_handle->Seek(0);
-	} else {
-		file_handle->Reset();
-	}
-	file_handle->Reset();
-}
-
-void JSONFileHandle::Reset() {
-	read_position = 0;
-	requested_reads = 0;
-	actual_reads = 0;
-	if (plain_file_source) {
-		file_handle->Reset();
-	}
-}
-
-bool JSONFileHandle::RequestedReadsComplete() {
-	return requested_reads == actual_reads;
-}
-
 } // namespace duckdb

package/src/duckdb/extension/json/include/buffered_json_reader.hpp

@@ -9,13 +9,13 @@
 #pragma once
 
 #include "duckdb/common/atomic.hpp"
+#include "duckdb/common/enum_util.hpp"
 #include "duckdb/common/enums/file_compression_type.hpp"
 #include "duckdb/common/file_system.hpp"
 #include "duckdb/common/multi_file_reader.hpp"
 #include "duckdb/common/mutex.hpp"
 #include "json_common.hpp"
 #include "json_enums.hpp"
-#include "duckdb/common/enum_util.hpp"
 
 namespace duckdb {
 
@@ -57,25 +57,25 @@ public:
 struct JSONFileHandle {
 public:
 	JSONFileHandle(unique_ptr<FileHandle> file_handle, Allocator &allocator);
+
 	bool IsOpen() const;
 	void Close();
 
+	void Reset();
+	bool RequestedReadsComplete();
+
 	idx_t FileSize() const;
 	idx_t Remaining() const;
 
 	bool CanSeek() const;
-	void Seek(idx_t position);
 
 	idx_t GetPositionAndSize(idx_t &position, idx_t requested_size);
 	void ReadAtPosition(char *pointer, idx_t size, idx_t position, bool sample_run);
 	idx_t Read(char *pointer, idx_t requested_size, bool sample_run);
 
-	void Reset();
-	bool RequestedReadsComplete();
-
 private:
-	idx_t ReadFromCache(char *&pointer, idx_t &size, idx_t &position);
 	idx_t ReadInternal(char *pointer, const idx_t requested_size);
+	idx_t ReadFromCache(char *&pointer, idx_t &size, idx_t &position);
 
 private:
 	//! The JSON file handle
@@ -101,38 +101,18 @@ class BufferedJSONReader {
 public:
 	BufferedJSONReader(ClientContext &context, BufferedJSONReaderOptions options, string file_name);
 
-private:
-	ClientContext &context;
-	BufferedJSONReaderOptions options;
-
-	//! File name
-	const string file_name;
-	//! File handle
-	unique_ptr<JSONFileHandle> file_handle;
-
-	//! Next buffer index within the file
-	idx_t buffer_index;
-	//! Mapping from batch index to currently held buffers
-	unordered_map<idx_t, unique_ptr<JSONBufferHandle>> buffer_map;
-
-	//! Line count per buffer
-	vector<int64_t> buffer_line_or_object_counts;
-
-public:
-	mutex lock;
-	MultiFileReaderData reader_data;
-
-public:
 	void OpenJSONFile();
 	void CloseJSONFile();
+	void Reset();
+
+	bool HasFileHandle() const;
 	bool IsOpen() const;
-	bool IsDone() const;
 
 	BufferedJSONReaderOptions &GetOptions();
-	const BufferedJSONReaderOptions &GetOptions() const;
 
 	JSONFormat GetFormat() const;
 	void SetFormat(JSONFormat format);
+
 	JSONRecordType GetRecordType() const;
 	void SetRecordType(JSONRecordType type);
 
@@ -142,7 +122,7 @@ public:
 public:
 	//! Insert/get/remove buffer (grabs the lock)
 	void InsertBuffer(idx_t buffer_idx, unique_ptr<JSONBufferHandle> &&buffer);
-	JSONBufferHandle *GetBuffer(idx_t buffer_idx);
+	optional_ptr<JSONBufferHandle> GetBuffer(idx_t buffer_idx);
 	AllocatedData RemoveBuffer(idx_t buffer_idx);
 
 	//! Get a new buffer index (must hold the lock)
@@ -154,11 +134,34 @@ public:
 	//! Throws a transform error that mentions the file name and line number
 	void ThrowTransformError(idx_t buf_index, idx_t line_or_object_in_buf, const string &error_message);
 
+	//! Scan progress
 	double GetProgress() const;
-	void Reset();
 
 private:
 	idx_t GetLineNumber(idx_t buf_index, idx_t line_or_object_in_buf);
+
+private:
+	ClientContext &context;
+	BufferedJSONReaderOptions options;
+
+	//! File name
+	const string file_name;
+	//! File handle
+	unique_ptr<JSONFileHandle> file_handle;
+
+	//! Next buffer index within the file
+	idx_t buffer_index;
+	//! Mapping from batch index to currently held buffers
+	unordered_map<idx_t, unique_ptr<JSONBufferHandle>> buffer_map;
+
+	//! Line count per buffer
+	vector<int64_t> buffer_line_or_object_counts;
+	//! Whether any of the reading threads has thrown an error
+	bool thrown;
+
+public:
+	mutex lock;
+	MultiFileReaderData reader_data;
 };
 
 } // namespace duckdb

package/src/duckdb/extension/json/include/json_scan.hpp

@@ -182,11 +182,13 @@ public:
 	//! One JSON reader per file
 	vector<optional_ptr<BufferedJSONReader>> json_readers;
 	//! Current file/batch index
-	idx_t file_index;
+	atomic<idx_t> file_index;
 	atomic<idx_t> batch_index;
 
 	//! Current number of threads active
 	idx_t system_threads;
+	//! Whether we enable parallel scans (only if less files than threads)
+	bool enable_parallel_scans;
 };
 
 struct JSONScanLocalState {
@@ -219,19 +221,20 @@ public:
 
 private:
 	bool ReadNextBuffer(JSONScanGlobalState &gstate);
-	void ReadNextBufferInternal(JSONScanGlobalState &gstate, idx_t &buffer_index);
-	void ReadNextBufferSeek(JSONScanGlobalState &gstate, idx_t &buffer_index);
-	void ReadNextBufferNoSeek(JSONScanGlobalState &gstate, idx_t &buffer_index);
+	void ReadNextBufferInternal(JSONScanGlobalState &gstate, optional_idx &buffer_index);
+	void ReadNextBufferSeek(JSONScanGlobalState &gstate, optional_idx &buffer_index);
+	void ReadNextBufferNoSeek(JSONScanGlobalState &gstate, optional_idx &buffer_index);
 	void SkipOverArrayStart();
 
-	bool ReadAndAutoDetect(JSONScanGlobalState &gstate, idx_t &buffer_index, const bool already_incremented_file_idx);
-	void ReconstructFirstObject(JSONScanGlobalState &gstate);
+	void ReadAndAutoDetect(JSONScanGlobalState &gstate, optional_idx &buffer_index);
+	void ReconstructFirstObject();
 	void ParseNextChunk();
 
 	void ParseJSON(char *const json_start, const idx_t json_size, const idx_t remaining);
 	void ThrowObjectSizeError(const idx_t object_size);
 	void ThrowInvalidAtEndError();
 
+	void TryIncrementFileIndex(JSONScanGlobalState &gstate) const;
 	bool IsParallel(JSONScanGlobalState &gstate) const;
 
 private: