duckdb 0.7.2-dev3515.0 → 0.7.2-dev3666.0

package/configure.py

@@ -63,6 +63,8 @@ elif 'DUCKDB_NODE_BINDIR' in os.environ:
             cflags += ['-g']
         if '-O0' in os.environ['DUCKDB_NODE_CFLAGS']:
             cflags += ['-O0']
+        if '-DNDEBUG' in os.environ['DUCKDB_NODE_CFLAGS']:
+            defines += ['NDEBUG']
 
     if 'DUCKDB_NODE_BUILD_CACHE' in os.environ:
         cache = {

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.7.2-dev3515.0",
+  "version": "0.7.2-dev3666.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/database.cpp

@@ -328,6 +328,7 @@ ScanReplacement(duckdb::ClientContext &context, const std::string &table_name, d
 			children.push_back(duckdb::make_uniq<duckdb::ConstantExpression>(std::move(param)));
 		}
 		table_function->function = duckdb::make_uniq<duckdb::FunctionExpression>(jsargs.function, std::move(children));
+		table_function->alias = table_name;
 		return std::move(table_function);
 	}
 	return nullptr;

package/src/duckdb/extension/json/buffered_json_reader.cpp

@@ -6,30 +6,35 @@
 
 namespace duckdb {
 
-void BufferedJSONReaderOptions::Serialize(FieldWriter &writer) {
-	writer.WriteString(file_path);
+void BufferedJSONReaderOptions::Serialize(FieldWriter &writer) const {
 	writer.WriteField<JSONFormat>(format);
+	writer.WriteField<JSONRecordType>(record_type);
 	writer.WriteField<FileCompressionType>(compression);
+	writer.WriteSerializable(file_options);
 }
 
 void BufferedJSONReaderOptions::Deserialize(FieldReader &reader) {
-	file_path = reader.ReadRequired<string>();
 	format = reader.ReadRequired<JSONFormat>();
+	record_type = reader.ReadRequired<JSONRecordType>();
 	compression = reader.ReadRequired<FileCompressionType>();
+	file_options = reader.ReadRequiredSerializable<MultiFileReaderOptions, MultiFileReaderOptions>();
 }
 
 JSONBufferHandle::JSONBufferHandle(idx_t buffer_index_p, idx_t readers_p, AllocatedData &&buffer_p, idx_t buffer_size_p)
     : buffer_index(buffer_index_p), readers(readers_p), buffer(std::move(buffer_p)), buffer_size(buffer_size_p) {
 }
 
-JSONFileHandle::JSONFileHandle(duckdb::unique_ptr<FileHandle> file_handle_p, Allocator &allocator_p)
+JSONFileHandle::JSONFileHandle(unique_ptr<FileHandle> file_handle_p, Allocator &allocator_p)
     : file_handle(std::move(file_handle_p)), allocator(allocator_p), can_seek(file_handle->CanSeek()),
       plain_file_source(file_handle->OnDiskFile() && can_seek), file_size(file_handle->GetFileSize()), read_position(0),
       requested_reads(0), actual_reads(0), cached_size(0) {
 }
 
 void JSONFileHandle::Close() {
-	file_handle->Close();
+	if (file_handle) {
+		file_handle->Close();
+		file_handle = nullptr;
+	}
 	cached_buffers.clear();
 }
 
@@ -155,15 +160,15 @@ idx_t JSONFileHandle::ReadInternal(const char *pointer, const idx_t requested_si
 	return total_read_size;
 }
 
-BufferedJSONReader::BufferedJSONReader(ClientContext &context, BufferedJSONReaderOptions options_p, string file_path_p)
-    : file_path(std::move(file_path_p)), context(context), options(std::move(options_p)), buffer_index(0) {
+BufferedJSONReader::BufferedJSONReader(ClientContext &context, BufferedJSONReaderOptions options_p, string file_name_p)
+    : context(context), options(options_p), file_name(std::move(file_name_p)), buffer_index(0) {
 }
 
 void BufferedJSONReader::OpenJSONFile() {
 	lock_guard<mutex> guard(lock);
 	auto &file_system = FileSystem::GetFileSystem(context);
 	auto regular_file_handle =
-	    file_system.OpenFile(file_path.c_str(), FileFlags::FILE_FLAGS_READ, FileLockType::NO_LOCK, options.compression);
+	    file_system.OpenFile(file_name.c_str(), FileFlags::FILE_FLAGS_READ, FileLockType::NO_LOCK, options.compression);
 	file_handle = make_uniq<JSONFileHandle>(std::move(regular_file_handle), BufferAllocator::Get(context));
 }
 
@@ -177,7 +182,7 @@ void BufferedJSONReader::CloseJSONFile() {
 	}
 }
 
-bool BufferedJSONReader::IsOpen() {
+bool BufferedJSONReader::IsOpen() const {
 	return file_handle != nullptr;
 }
 
@@ -185,11 +190,41 @@ BufferedJSONReaderOptions &BufferedJSONReader::GetOptions() {
 	return options;
 }
 
+const BufferedJSONReaderOptions &BufferedJSONReader::GetOptions() const {
+	return options;
+}
+
+JSONFormat BufferedJSONReader::GetFormat() const {
+	return options.format;
+}
+
+void BufferedJSONReader::SetFormat(JSONFormat format) {
+	D_ASSERT(options.format == JSONFormat::AUTO_DETECT);
+	options.format = format;
+}
+
+JSONRecordType BufferedJSONReader::GetRecordType() const {
+	return options.record_type;
+}
+
+void BufferedJSONReader::SetRecordType(duckdb::JSONRecordType type) {
+	D_ASSERT(options.record_type == JSONRecordType::AUTO_DETECT);
+	options.record_type = type;
+}
+
+bool BufferedJSONReader::IsParallel() const {
+	return options.format == JSONFormat::NEWLINE_DELIMITED && file_handle->CanSeek();
+}
+
+const string &BufferedJSONReader::GetFileName() const {
+	return file_name;
+}
+
 JSONFileHandle &BufferedJSONReader::GetFileHandle() const {
 	return *file_handle;
 }
 
-void BufferedJSONReader::InsertBuffer(idx_t buffer_idx, duckdb::unique_ptr<JSONBufferHandle> &&buffer) {
+void BufferedJSONReader::InsertBuffer(idx_t buffer_idx, unique_ptr<JSONBufferHandle> &&buffer) {
 	lock_guard<mutex> guard(lock);
 	buffer_map.insert(make_pair(buffer_idx, std::move(buffer)));
 }
@@ -220,7 +255,7 @@ void BufferedJSONReader::SetBufferLineOrObjectCount(idx_t index, idx_t count) {
 }
 
 idx_t BufferedJSONReader::GetLineNumber(idx_t buf_index, idx_t line_or_object_in_buf) {
-	D_ASSERT(options.format == JSONFormat::UNSTRUCTURED || options.format == JSONFormat::NEWLINE_DELIMITED);
+	D_ASSERT(options.format != JSONFormat::AUTO_DETECT);
 	while (true) {
 		lock_guard<mutex> guard(lock);
 		idx_t line = line_or_object_in_buf;
@@ -243,23 +278,23 @@ idx_t BufferedJSONReader::GetLineNumber(idx_t buf_index, idx_t line_or_object_in
 
 void BufferedJSONReader::ThrowParseError(idx_t buf_index, idx_t line_or_object_in_buf, yyjson_read_err &err,
                                          const string &extra) {
-	string unit = options.format == JSONFormat::NEWLINE_DELIMITED ? "line" : "object";
+	string unit = options.format == JSONFormat::NEWLINE_DELIMITED ? "line" : "record/value";
 	auto line = GetLineNumber(buf_index, line_or_object_in_buf);
-	throw InvalidInputException("Malformed JSON in file \"%s\", at byte %llu in %s %llu: %s. %s", file_path,
+	throw InvalidInputException("Malformed JSON in file \"%s\", at byte %llu in %s %llu: %s. %s", file_name,
 	                            err.pos + 1, unit, line + 1, err.msg, extra);
 }
 
 void BufferedJSONReader::ThrowTransformError(idx_t buf_index, idx_t line_or_object_in_buf,
                                              const string &error_message) {
-	string unit = options.format == JSONFormat::NEWLINE_DELIMITED ? "line" : "object";
+	string unit = options.format == JSONFormat::NEWLINE_DELIMITED ? "line" : "record/value";
 	auto line = GetLineNumber(buf_index, line_or_object_in_buf);
-	throw InvalidInputException("JSON transform error in file \"%s\", in %s %llu: %s", file_path, unit, line,
+	throw InvalidInputException("JSON transform error in file \"%s\", in %s %llu: %s.", file_name, unit, line,
 	                            error_message);
 }
 
 double BufferedJSONReader::GetProgress() const {
-	if (file_handle) {
-		return 100.0 * double(file_handle->Remaining()) / double(file_handle->FileSize());
+	if (IsOpen()) {
+		return 100.0 - 100.0 * double(file_handle->Remaining()) / double(file_handle->FileSize());
 	} else {
 		return 0;
 	}
@@ -270,6 +305,10 @@ void BufferedJSONReader::Reset() {
 	buffer_map.clear();
 	buffer_line_or_object_counts.clear();
 
+	if (!file_handle) {
+		return;
+	}
+
 	if (file_handle->CanSeek()) {
 		file_handle->Seek(0);
 	} else {

package/src/duckdb/extension/json/include/buffered_json_reader.hpp

@@ -11,6 +11,7 @@
 #include "duckdb/common/atomic.hpp"
 #include "duckdb/common/enums/file_compression_type.hpp"
 #include "duckdb/common/file_system.hpp"
+#include "duckdb/common/multi_file_reader.hpp"
 #include "duckdb/common/mutex.hpp"
 #include "json_common.hpp"
 
@@ -19,23 +20,35 @@ namespace duckdb {
 enum class JSONFormat : uint8_t {
 	//! Auto-detect format (UNSTRUCTURED / NEWLINE_DELIMITED)
 	AUTO_DETECT = 0,
-	//! One object after another, newlines can be anywhere
+	//! One unit after another, newlines can be anywhere
 	UNSTRUCTURED = 1,
-	//! Objects are separated by newlines, newlines do not occur within values (NDJSON)
+	//! Units are separated by newlines, newlines do not occur within Units (NDJSON)
 	NEWLINE_DELIMITED = 2,
+	//! File is one big array of units
+	ARRAY = 3,
+};
+
+enum class JSONRecordType : uint8_t {
+	AUTO_DETECT = 0,
+	//! Sequential objects that are unpacked
+	RECORDS = 1,
+	//! Any other JSON type, e.g., ARRAY
+	VALUES = 2,
 };
 
 struct BufferedJSONReaderOptions {
 public:
-	//! The file path of the JSON file to read
-	string file_path;
 	//! The format of the JSON
 	JSONFormat format = JSONFormat::AUTO_DETECT;
+	//! Whether record types in the JSON
+	JSONRecordType record_type = JSONRecordType::AUTO_DETECT;
 	//! Whether file is compressed or not, and if so which compression type
 	FileCompressionType compression = FileCompressionType::AUTO_DETECT;
+	//! Multi-file reader options
+	MultiFileReaderOptions file_options;
 
 public:
-	void Serialize(FieldWriter &writer);
+	void Serialize(FieldWriter &writer) const;
 	void Deserialize(FieldReader &reader);
 };
 
@@ -57,7 +70,7 @@ public:
 
 struct JSONFileHandle {
 public:
-	JSONFileHandle(duckdb::unique_ptr<FileHandle> file_handle, Allocator &allocator);
+	JSONFileHandle(unique_ptr<FileHandle> file_handle, Allocator &allocator);
 	void Close();
 
 	idx_t FileSize() const;
@@ -79,7 +92,7 @@ private:
 
 private:
 	//! The JSON file handle
-	duckdb::unique_ptr<FileHandle> file_handle;
+	unique_ptr<FileHandle> file_handle;
 	Allocator &allocator;
 
 	//! File properties
@@ -99,17 +112,50 @@ private:
 
 class BufferedJSONReader {
 public:
-	BufferedJSONReader(ClientContext &context, BufferedJSONReaderOptions options, string file_path);
+	BufferedJSONReader(ClientContext &context, BufferedJSONReaderOptions options, string file_name);
 
+private:
+	ClientContext &context;
+	BufferedJSONReaderOptions options;
+
+	//! File name
+	const string file_name;
+	//! File handle
+	unique_ptr<JSONFileHandle> file_handle;
+
+	//! Next buffer index within the file
+	idx_t buffer_index;
+	//! Mapping from batch index to currently held buffers
+	unordered_map<idx_t, unique_ptr<JSONBufferHandle>> buffer_map;
+
+	//! Line count per buffer
+	vector<int64_t> buffer_line_or_object_counts;
+
+public:
+	mutex lock;
+	MultiFileReaderData reader_data;
+
+public:
 	void OpenJSONFile();
 	void CloseJSONFile();
-	bool IsOpen();
+	bool IsOpen() const;
 
 	BufferedJSONReaderOptions &GetOptions();
+	const BufferedJSONReaderOptions &GetOptions() const;
+
+	JSONFormat GetFormat() const;
+	void SetFormat(JSONFormat format);
+	JSONRecordType GetRecordType() const;
+	void SetRecordType(JSONRecordType type);
+
+	bool IsParallel() const;
+
+	const string &GetFileName() const;
 	JSONFileHandle &GetFileHandle() const;
 
+public:
 	//! Insert/get/remove buffer (grabs the lock)
-	void InsertBuffer(idx_t buffer_idx, duckdb::unique_ptr<JSONBufferHandle> &&buffer);
+	void InsertBuffer(idx_t buffer_idx, unique_ptr<JSONBufferHandle> &&buffer);
 	JSONBufferHandle *GetBuffer(idx_t buffer_idx);
 	AllocatedData RemoveBuffer(idx_t buffer_idx);
 
@@ -127,27 +173,6 @@ public:
 
 private:
 	idx_t GetLineNumber(idx_t buf_index, idx_t line_or_object_in_buf);
-
-public:
-	mutex lock;
-
-	//! File path
-	const string file_path;
-
-private:
-	ClientContext &context;
-	BufferedJSONReaderOptions options;
-
-	//! File handle
-	duckdb::unique_ptr<JSONFileHandle> file_handle;
-
-	//! Next buffer index within the file
-	idx_t buffer_index;
-	//! Mapping from batch index to currently held buffers
-	unordered_map<idx_t, duckdb::unique_ptr<JSONBufferHandle>> buffer_map;
-
-	//! Line count per buffer
-	vector<int64_t> buffer_line_or_object_counts;
 };
 
 } // namespace duckdb

package/src/duckdb/extension/json/include/json_common.hpp

@@ -22,7 +22,7 @@ public:
 	    : arena_allocator(allocator), yyjson_allocator({Allocate, Reallocate, Free, &arena_allocator}) {
 	}
 
-	inline yyjson_alc *GetYYJSONAllocator() {
+	inline yyjson_alc *GetYYAlc() {
 		return &yyjson_allocator;
 	}
 
@@ -62,7 +62,7 @@ struct JSONKeyHash {
 			memcpy(&result, k.ptr + k.len - sizeof(size_t), sizeof(size_t));
 		} else {
 			result = 0;
-			duckdb::FastMemcpy(&result, k.ptr, k.len);
+			FastMemcpy(&result, k.ptr, k.len);
 		}
 		return result;
 	}
@@ -73,7 +73,7 @@ struct JSONKeyEquality {
 		if (a.len != b.len) {
 			return false;
 		}
-		return duckdb::FastMemcmp(a.ptr, b.ptr, a.len) == 0;
+		return FastMemcmp(a.ptr, b.ptr, a.len) == 0;
 	}
 };
 
@@ -98,7 +98,8 @@ public:
 public:
 	//! Read/Write flags
 	static constexpr auto READ_FLAG = YYJSON_READ_ALLOW_INF_AND_NAN | YYJSON_READ_ALLOW_TRAILING_COMMAS;
-	static constexpr auto STOP_READ_FLAG = READ_FLAG | YYJSON_READ_STOP_WHEN_DONE | YYJSON_READ_INSITU;
+	static constexpr auto READ_STOP_FLAG = READ_FLAG | YYJSON_READ_STOP_WHEN_DONE;
+	static constexpr auto READ_INSITU_FLAG = READ_STOP_FLAG | YYJSON_READ_INSITU;
 	static constexpr auto WRITE_FLAG = YYJSON_WRITE_ALLOW_INF_AND_NAN;
 	static constexpr auto WRITE_PRETTY_FLAG = YYJSON_WRITE_ALLOW_INF_AND_NAN | YYJSON_WRITE_PRETTY;
 

package/src/duckdb/extension/json/include/json_executors.hpp

@@ -20,7 +20,7 @@ public:
 	static void UnaryExecute(DataChunk &args, ExpressionState &state, Vector &result,
 	                         std::function<T(yyjson_val *, yyjson_alc *, Vector &)> fun) {
 		auto &lstate = JSONFunctionLocalState::ResetAndGet(state);
-		auto alc = lstate.json_allocator.GetYYJSONAllocator();
+		auto alc = lstate.json_allocator.GetYYAlc();
 
 		auto &inputs = args.data[0];
 		UnaryExecutor::Execute<string_t, T>(inputs, result, args.size(), [&](string_t input) {
@@ -34,36 +34,32 @@ public:
 	static void BinaryExecute(DataChunk &args, ExpressionState &state, Vector &result,
 	                          std::function<T(yyjson_val *, yyjson_alc *, Vector &)> fun) {
 		auto &func_expr = state.expr.Cast<BoundFunctionExpression>();
-		const auto &info = (JSONReadFunctionData &)*func_expr.bind_info;
+		const auto &info = func_expr.bind_info->Cast<JSONReadFunctionData>();
 		auto &lstate = JSONFunctionLocalState::ResetAndGet(state);
-		auto alc = lstate.json_allocator.GetYYJSONAllocator();
+		auto alc = lstate.json_allocator.GetYYAlc();
 
 		auto &inputs = args.data[0];
-		if (info.constant) {
-			// Constant path
+		if (info.constant) { // Constant path
 			const char *ptr = info.ptr;
 			const idx_t &len = info.len;
 			UnaryExecutor::ExecuteWithNulls<string_t, T>(
 			    inputs, result, args.size(), [&](string_t input, ValidityMask &mask, idx_t idx) {
-				    auto doc = JSONCommon::ReadDocument(input, JSONCommon::READ_FLAG,
-				                                        lstate.json_allocator.GetYYJSONAllocator());
+				    auto doc = JSONCommon::ReadDocument(input, JSONCommon::READ_FLAG, lstate.json_allocator.GetYYAlc());
 				    auto val = JSONCommon::GetPointerUnsafe<yyjson_val>(doc->root, ptr, len);
-				    if (!val) {
+				    if (!val || unsafe_yyjson_is_null(val)) {
 					    mask.SetInvalid(idx);
 					    return T {};
 				    } else {
 					    return fun(val, alc, result);
 				    }
 			    });
-		} else {
-			// Columnref path
+		} else { // Columnref path
 			auto &paths = args.data[1];
 			BinaryExecutor::ExecuteWithNulls<string_t, string_t, T>(
 			    inputs, paths, result, args.size(), [&](string_t input, string_t path, ValidityMask &mask, idx_t idx) {
-				    auto doc = JSONCommon::ReadDocument(input, JSONCommon::READ_FLAG,
-				                                        lstate.json_allocator.GetYYJSONAllocator());
+				    auto doc = JSONCommon::ReadDocument(input, JSONCommon::READ_FLAG, lstate.json_allocator.GetYYAlc());
 				    auto val = JSONCommon::GetPointer<yyjson_val>(doc->root, path);
-				    if (!val) {
+				    if (!val || unsafe_yyjson_is_null(val)) {
 					    mask.SetInvalid(idx);
 					    return T {};
 				    } else {
@@ -81,9 +77,9 @@ public:
 	static void ExecuteMany(DataChunk &args, ExpressionState &state, Vector &result,
 	                        std::function<T(yyjson_val *, yyjson_alc *, Vector &)> fun) {
 		auto &func_expr = state.expr.Cast<BoundFunctionExpression>();
-		const auto &info = (JSONReadManyFunctionData &)*func_expr.bind_info;
+		const auto &info = func_expr.bind_info->Cast<JSONReadManyFunctionData>();
 		auto &lstate = JSONFunctionLocalState::ResetAndGet(state);
-		auto alc = lstate.json_allocator.GetYYJSONAllocator();
+		auto alc = lstate.json_allocator.GetYYAlc();
 		D_ASSERT(info.ptrs.size() == info.lens.size());
 
 		const auto count = args.size();
@@ -112,12 +108,11 @@ public:
 				continue;
 			}
 
-			auto doc = JSONCommon::ReadDocument(inputs[idx], JSONCommon::READ_FLAG,
-			                                    lstate.json_allocator.GetYYJSONAllocator());
+			auto doc = JSONCommon::ReadDocument(inputs[idx], JSONCommon::READ_FLAG, lstate.json_allocator.GetYYAlc());
 			for (idx_t path_i = 0; path_i < num_paths; path_i++) {
 				auto child_idx = offset + path_i;
 				val = JSONCommon::GetPointerUnsafe<yyjson_val>(doc->root, info.ptrs[path_i], info.lens[path_i]);
-				if (!val) {
+				if (!val || unsafe_yyjson_is_null(val)) {
 					child_validity.SetInvalid(child_idx);
 				} else {
 					child_data[child_idx] = fun(val, alc, child);

package/src/duckdb/extension/json/include/json_functions.hpp

@@ -115,10 +115,13 @@ private:
 	// Table functions
 	static TableFunctionSet GetReadJSONObjectsFunction();
 	static TableFunctionSet GetReadNDJSONObjectsFunction();
+	static TableFunctionSet GetReadJSONObjectsAutoFunction();
+
 	static TableFunctionSet GetReadJSONFunction();
 	static TableFunctionSet GetReadNDJSONFunction();
 	static TableFunctionSet GetReadJSONAutoFunction();
 	static TableFunctionSet GetReadNDJSONAutoFunction();
+
 	static TableFunctionSet GetExecuteJsonSerializedSqlFunction();
 };