duckdb 0.6.2-dev2115.0 → 0.6.2-dev2226.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.6.2-dev2115.0",
+  "version": "0.6.2-dev2226.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/extension/json/buffered_json_reader.cpp

@@ -185,8 +185,7 @@ void BufferedJSONReader::SetBufferLineOrObjectCount(idx_t index, idx_t count) {
 	buffer_line_or_object_counts[index] = count;
 }
 
-void BufferedJSONReader::ThrowParseError(idx_t buf_index, idx_t line_or_object_in_buf, yyjson_read_err &err,
-                                         const string &extra) {
+idx_t BufferedJSONReader::GetLineNumber(idx_t buf_index, idx_t line_or_object_in_buf) {
 	D_ASSERT(options.format == JSONFormat::UNSTRUCTURED || options.format == JSONFormat::NEWLINE_DELIMITED);
 	while (true) {
 		lock_guard<mutex> guard(lock);
@@ -203,13 +202,27 @@ void BufferedJSONReader::ThrowParseError(idx_t buf_index, idx_t line_or_object_i
 		if (!can_throw) {
 			continue;
 		}
-		string unit = options.format == JSONFormat::NEWLINE_DELIMITED ? "line" : "object";
 		// SQL uses 1-based indexing so I guess we will do that in our exception here as well
-		throw InvalidInputException("Malformed JSON in file \"%s\", at byte %llu in %s %llu: %s. %s", file_path,
-		                            err.pos + 1, unit, line + 1, err.msg, extra);
+		return line + 1;
 	}
 }
 
+void BufferedJSONReader::ThrowParseError(idx_t buf_index, idx_t line_or_object_in_buf, yyjson_read_err &err,
+                                         const string &extra) {
+	string unit = options.format == JSONFormat::NEWLINE_DELIMITED ? "line" : "object";
+	auto line = GetLineNumber(buf_index, line_or_object_in_buf);
+	throw InvalidInputException("Malformed JSON in file \"%s\", at byte %llu in %s %llu: %s. %s", file_path,
+	                            err.pos + 1, unit, line + 1, err.msg, extra);
+}
+
+void BufferedJSONReader::ThrowTransformError(idx_t buf_index, idx_t line_or_object_in_buf,
+                                             const string &error_message) {
+	string unit = options.format == JSONFormat::NEWLINE_DELIMITED ? "line" : "object";
+	auto line = GetLineNumber(buf_index, line_or_object_in_buf);
+	throw InvalidInputException("JSON transform error in file \"%s\", in %s %llu: %s", file_path, unit, line,
+	                            error_message);
+}
+
 double BufferedJSONReader::GetProgress() const {
 	if (file_handle) {
 		return 100.0 * double(file_handle->Remaining()) / double(file_handle->FileSize());

package/src/duckdb/extension/json/include/buffered_json_reader.hpp

@@ -112,12 +112,17 @@ public:
 	idx_t GetBufferIndex();
 	//! Set line count for a buffer that is done (grabs the lock)
 	void SetBufferLineOrObjectCount(idx_t index, idx_t count);
-	//! Throws an error that mentions the file name and line number
+	//! Throws a parse error that mentions the file name and line number
 	void ThrowParseError(idx_t buf_index, idx_t line_or_object_in_buf, yyjson_read_err &err, const string &extra = "");
+	//! Throws a transform error that mentions the file name and line number
+	void ThrowTransformError(idx_t buf_index, idx_t line_or_object_in_buf, const string &error_message);
 
 	double GetProgress() const;
 	void Reset();
 
+private:
+	idx_t GetLineNumber(idx_t buf_index, idx_t line_or_object_in_buf);
+
 public:
 	mutex lock;
 

package/src/duckdb/extension/json/include/json_common.hpp

@@ -216,6 +216,7 @@ public:
 		auto data = WriteVal<YYJSON_VAL_T>(val, alc, len);
 		return string_t(data, len);
 	}
+	static string ValToString(yyjson_val *val);
 	//! Throw an error with the printed yyjson_val
 	static void ThrowValFormatError(string error_string, yyjson_val *val);
 

package/src/duckdb/extension/json/include/json_scan.hpp

@@ -97,6 +97,8 @@ public:
 	vector<string> names;
 	//! Max depth we go to detect nested JSON schema (defaults to unlimited)
 	idx_t max_depth = NumericLimits<idx_t>::Maximum();
+	//! Whether we're parsing objects (usually), or something else like arrays
+	bool objects = true;
 
 	//! Stored readers for when we're detecting the schema
 	vector<unique_ptr<BufferedJSONReader>> stored_readers;
@@ -167,12 +169,17 @@ public:
 public:
 	idx_t ReadNext(JSONScanGlobalState &gstate);
 	yyjson_alc *GetAllocator();
+	void ThrowTransformError(idx_t count, idx_t object_index, const string &error_message);
 
 	JSONLine lines[STANDARD_VECTOR_SIZE];
 	yyjson_val *objects[STANDARD_VECTOR_SIZE];
 
 	idx_t batch_index;
 
+	//! Options when transforming the JSON to columnar data
+	DateFormatMap date_format_map;
+	JSONTransformOptions transform_options;
+
 private:
 	yyjson_val *ParseLine(char *line_start, idx_t line_size, idx_t remaining, JSONLine &line);
 

package/src/duckdb/extension/json/include/json_transform.hpp

@@ -14,19 +14,32 @@
 namespace duckdb {
 
 struct DateFormatMap;
+class BufferedJSONReader;
 
+//! Options for error handling while transforming JSON
 struct JSONTransformOptions {
+public:
+	JSONTransformOptions();
+	JSONTransformOptions(bool strict_cast, bool error_duplicate_key, bool error_missing_key, bool error_unkown_key);
+
 public:
 	//! Throws an error if the cast doesn't work (instead of NULL-ing it)
-	bool strict_cast;
+	bool strict_cast = false;
 	//! Throws an error if there is a duplicate key (instead of ignoring it)
-	bool error_duplicate_key;
+	bool error_duplicate_key = false;
 	//! Throws an error if a key is missing (instead of NULL-ing it)
-	bool error_missing_key;
+	bool error_missing_key = false;
 	//! Throws an error if an object has a key we didn't know about
-	bool error_unknown_key;
-	//! Date format used for parsing
-	DateFormatMap *date_format_map;
+	bool error_unknown_key = false;
+
+	//! JSON reader for adding line number information to transform errors (can be NULL)
+	bool from_file = false;
+	//! Date format used for parsing (can be NULL)
+	DateFormatMap *date_format_map = nullptr;
+	//! String to store errors in
+	string error_message;
+	//! Index of the object where the error occurred
+	idx_t object_index = DConstants::INVALID_INDEX;
 
 public:
 	void Serialize(FieldWriter &writer);
@@ -48,10 +61,12 @@ struct TryParseTimeStamp {
 };
 
 struct JSONTransform {
-	static void TransformObject(yyjson_val *objects[], yyjson_alc *alc, const idx_t count, const vector<string> &names,
-	                            const vector<Vector *> &result_vectors, const JSONTransformOptions &options);
-	static void GetStringVector(yyjson_val *vals[], const idx_t count, const LogicalType &target, Vector &string_vector,
-	                            const bool strict);
+	static bool Transform(yyjson_val *vals[], yyjson_alc *alc, Vector &result, const idx_t count,
+	                      JSONTransformOptions &options);
+	static bool TransformObject(yyjson_val *objects[], yyjson_alc *alc, const idx_t count, const vector<string> &names,
+	                            const vector<Vector *> &result_vectors, JSONTransformOptions &options);
+	static bool GetStringVector(yyjson_val *vals[], const idx_t count, const LogicalType &target, Vector &string_vector,
+	                            JSONTransformOptions &options);
 };
 
 } // namespace duckdb

package/src/duckdb/extension/json/json_common.cpp

@@ -2,11 +2,15 @@
 
 namespace duckdb {
 
-void JSONCommon::ThrowValFormatError(string error_string, yyjson_val *val) {
+string JSONCommon::ValToString(yyjson_val *val) {
 	JSONAllocator json_allocator(Allocator::DefaultAllocator());
 	idx_t len;
 	auto data = JSONCommon::WriteVal<yyjson_val>(val, json_allocator.GetYYJSONAllocator(), len);
-	error_string = StringUtil::Format(error_string, string(data, len));
+	return string(data, len);
+}
+
+void JSONCommon::ThrowValFormatError(string error_string, yyjson_val *val) {
+	error_string = StringUtil::Format(error_string, JSONCommon::ValToString(val));
 	throw InvalidInputException(error_string);
 }
 

package/src/duckdb/extension/json/json_functions/json_structure.cpp

@@ -28,10 +28,27 @@ JSONStructureNode::JSONStructureNode(yyjson_val *key_p, yyjson_val *val_p)
 }
 
 JSONStructureDescription &JSONStructureNode::GetOrCreateDescription(LogicalTypeId type) {
+	if (descriptions.empty()) {
+		// Empty, just put this type in there
+		descriptions.emplace_back(type);
+		return descriptions.back();
+	}
+
+	if (descriptions.size() == 1 && descriptions[0].type == LogicalTypeId::SQLNULL) {
+		// Only a NULL in there, override
+		descriptions[0].type = type;
+		return descriptions[0];
+	}
+
+	if (type == LogicalTypeId::SQLNULL) {
+		// 'descriptions' is non-empty, so let's not add NULL
+		return descriptions.back();
+	}
+
 	// Check if type is already in there or if we can merge numerics
 	const auto is_numeric = IsNumeric(type);
 	for (auto &description : descriptions) {
-		if (type == LogicalTypeId::SQLNULL || type == description.type) {
+		if (type == description.type) {
 			return description;
 		} else if (is_numeric && IsNumeric(description.type)) {
 			description.type = MaxNumericType(type, description.type);
@@ -111,19 +128,19 @@ void JSONStructureNode::RefineCandidateTypesArray(yyjson_val *vals[], idx_t coun
 
 	idx_t total_list_size = 0;
 	for (idx_t i = 0; i < count; i++) {
-		if (vals[i] && !yyjson_is_null(vals[i])) {
+		if (vals[i] && !unsafe_yyjson_is_null(vals[i])) {
 			D_ASSERT(yyjson_is_arr(vals[i]));
 			total_list_size += unsafe_yyjson_get_len(vals[i]);
 		}
 	}
 
 	idx_t offset = 0;
-	auto child_vals = (yyjson_val **)allocator.Allocate(total_list_size * sizeof(yyjson_val *));
+	auto child_vals = (yyjson_val **)allocator.AllocateAligned(total_list_size * sizeof(yyjson_val *));
 
 	size_t idx, max;
 	yyjson_val *child_val;
 	for (idx_t i = 0; i < count; i++) {
-		if (vals[i] && !yyjson_is_null(vals[i])) {
+		if (vals[i] && !unsafe_yyjson_is_null(vals[i])) {
 			yyjson_arr_foreach(vals[i], idx, max, child_val) {
 				child_vals[offset++] = child_val;
 			}
@@ -141,14 +158,20 @@ void JSONStructureNode::RefineCandidateTypesObject(yyjson_val *vals[], idx_t cou
 	vector<yyjson_val **> child_vals;
 	child_vals.reserve(child_count);
 	for (idx_t child_idx = 0; child_idx < child_count; child_idx++) {
-		child_vals.emplace_back((yyjson_val **)allocator.Allocate(count * sizeof(yyjson_val *)));
+		child_vals.emplace_back((yyjson_val **)allocator.AllocateAligned(count * sizeof(yyjson_val *)));
 	}
 
+	idx_t found_key_count;
+	auto found_keys = (bool *)allocator.AllocateAligned(sizeof(bool) * child_count);
+
 	const auto &key_map = desc.key_map;
 	size_t idx, max;
 	yyjson_val *child_key, *child_val;
 	for (idx_t i = 0; i < count; i++) {
-		if (vals[i] && !yyjson_is_null(vals[i])) {
+		if (vals[i] && !unsafe_yyjson_is_null(vals[i])) {
+			found_key_count = 0;
+			memset(found_keys, false, child_count);
+
 			D_ASSERT(yyjson_is_obj(vals[i]));
 			yyjson_obj_foreach(vals[i], idx, max, child_key, child_val) {
 				D_ASSERT(yyjson_is_str(child_key));
@@ -156,7 +179,19 @@ void JSONStructureNode::RefineCandidateTypesObject(yyjson_val *vals[], idx_t cou
 				auto key_len = unsafe_yyjson_get_len(child_key);
 				auto it = key_map.find({key_ptr, key_len});
 				D_ASSERT(it != key_map.end());
-				child_vals[it->second][i] = child_val;
+				const auto child_idx = it->second;
+				child_vals[child_idx][i] = child_val;
+				found_keys[child_idx] = true;
+				found_key_count++;
+			}
+
+			if (found_key_count != child_count) {
+				// Set child val to nullptr so recursion doesn't break
+				for (idx_t child_idx = 0; child_idx < child_count; child_idx++) {
+					if (!found_keys[child_idx]) {
+						child_vals[child_idx][i] = nullptr;
+					}
+				}
 			}
 		} else {
 			for (idx_t child_idx = 0; child_idx < child_count; child_idx++) {
@@ -180,7 +215,8 @@ void JSONStructureNode::RefineCandidateTypesString(yyjson_val *vals[], idx_t cou
 	if (descriptions[0].candidate_types.empty()) {
 		return;
 	}
-	JSONTransform::GetStringVector(vals, count, LogicalType::SQLNULL, string_vector, false);
+	static JSONTransformOptions OPTIONS;
+	JSONTransform::GetStringVector(vals, count, LogicalType::SQLNULL, string_vector, OPTIONS);
 	EliminateCandidateTypes(count, string_vector, date_format_map);
 }
 
@@ -203,7 +239,7 @@ void JSONStructureNode::EliminateCandidateTypes(idx_t count, Vector &string_vect
 			}
 		} else {
 			string error_message;
-			if (!VectorOperations::DefaultTryCast(string_vector, result_vector, count, &error_message)) {
+			if (!VectorOperations::DefaultTryCast(string_vector, result_vector, count, &error_message, true)) {
 				candidate_types.pop_back();
 			} else {
 				return;
@@ -467,6 +503,8 @@ LogicalType JSONStructure::StructureToType(ClientContext &context, const JSONStr
 		return StructureToTypeObject(context, node, max_depth, depth);
 	case LogicalTypeId::VARCHAR:
 		return StructureToTypeString(node);
+	case LogicalTypeId::SQLNULL:
+		return LogicalTypeId::INTEGER;
 	default:
 		return desc.type;
 	}