duckdb 0.8.2-dev33.0 → 0.8.2-dev3334.0

package/src/duckdb/extension/json/json_common.cpp

@@ -2,6 +2,8 @@
 
 namespace duckdb {
 
+using JSONPathType = JSONCommon::JSONPathType;
+
 string JSONCommon::ValToString(yyjson_val *val, idx_t max_len) {
 	JSONAllocator json_allocator(Allocator::DefaultAllocator());
 	idx_t len;
@@ -18,61 +20,291 @@ void JSONCommon::ThrowValFormatError(string error_string, yyjson_val *val) {
 	throw InvalidInputException(error_string);
 }
 
-string ThrowPathError(const char *ptr, const char *end) {
+string ThrowPathError(const char *ptr, const char *end, const bool binder) {
 	ptr--;
-	throw InvalidInputException("JSON path error near '%s'", string(ptr, end - ptr));
+	auto msg = StringUtil::Format("JSON path error near '%s'", string(ptr, end - ptr));
+	if (binder) {
+		throw BinderException(msg);
+	} else {
+		throw InvalidInputException(msg);
+	}
+}
+
+static inline idx_t ReadString(const char *ptr, const char *const end, const bool escaped) {
+	const char *const before = ptr;
+	if (escaped) {
+		while (ptr != end) {
+			if (*ptr == '"') {
+				break;
+			}
+			ptr++;
+		}
+		return ptr == end ? 0 : ptr - before;
+	} else {
+		while (ptr != end) {
+			if (*ptr == '.' || *ptr == '[') {
+				break;
+			}
+			ptr++;
+		}
+		return ptr - before;
+	}
+}
+
+static inline idx_t ReadInteger(const char *ptr, const char *const end, idx_t &idx) {
+	static constexpr auto IDX_T_SAFE_DIG = 19;
+	static constexpr auto IDX_T_MAX = ((idx_t)(~(idx_t)0));
+
+	const char *const before = ptr;
+	idx = 0;
+	for (idx_t i = 0; i < IDX_T_SAFE_DIG; i++) {
+		if (ptr == end) {
+			// No closing ']'
+			return 0;
+		}
+		if (*ptr == ']') {
+			break;
+		}
+		uint8_t add = (uint8_t)(*ptr - '0');
+		if (add <= 9) {
+			idx = add + idx * 10;
+		} else {
+			// Not a digit
+			return 0;
+		}
+		ptr++;
+	}
+	// Invalid if overflow
+	return idx >= (idx_t)IDX_T_MAX ? 0 : ptr - before;
+}
+
+static inline bool ReadKey(const char *&ptr, const char *const end, const char *&key_ptr, idx_t &key_len) {
+	D_ASSERT(ptr != end);
+	if (*ptr == '*') { // Wildcard
+		ptr++;
+		key_len = DConstants::INVALID_INDEX;
+		return true;
+	}
+	bool escaped = false;
+	if (*ptr == '"') {
+		ptr++; // Skip past opening '"'
+		escaped = true;
+	}
+	key_ptr = ptr;
+	key_len = ReadString(ptr, end, escaped);
+	if (key_len == 0) {
+		return false;
+	}
+	ptr += key_len;
+	if (escaped) {
+		ptr++; // Skip past closing '"'
+	}
+	return true;
 }
 
-void JSONCommon::ValidatePathDollar(const char *ptr, const idx_t &len) {
+static inline bool ReadArrayIndex(const char *&ptr, const char *const end, idx_t &array_index, bool &from_back) {
+	D_ASSERT(ptr != end);
+	from_back = false;
+	if (*ptr == '*') { // Wildcard
+		ptr++;
+		if (ptr == end || *ptr != ']') {
+			return false;
+		}
+		array_index = DConstants::INVALID_INDEX;
+	} else {
+		if (*ptr == '#') { // SQLite syntax to index from back of array
+			ptr++;         // Skip over '#'
+			if (ptr == end) {
+				return false;
+			}
+			if (*ptr == ']') {
+				// [#] always returns NULL in SQLite, so we return an array index that will do the same
+				array_index = NumericLimits<uint32_t>::Maximum();
+				ptr++;
+				return true;
+			}
+			if (*ptr != '-') {
+				return false;
+			}
+			from_back = true;
+		}
+		if (*ptr == '-') {
+			ptr++; // Skip over '-'
+			from_back = true;
+		}
+		auto idx_len = ReadInteger(ptr, end, array_index);
+		if (idx_len == 0) {
+			return false;
+		}
+		ptr += idx_len;
+	}
+	ptr++; // Skip past closing ']'
+	return true;
+}
+
+JSONPathType JSONCommon::ValidatePath(const char *ptr, const idx_t &len, const bool binder) {
+	D_ASSERT(len >= 1 && *ptr == '$');
+	JSONPathType path_type = JSONPathType::REGULAR;
 	const char *const end = ptr + len;
-	// Skip past '$'
-	ptr++;
+	ptr++; // Skip past '$'
 	while (ptr != end) {
 		const auto &c = *ptr++;
-		if (c == '.') {
-			// Object
-			bool escaped = false;
-			if (*ptr == '"') {
-				// Skip past opening '"'
-				ptr++;
-				escaped = true;
+		if (ptr == end) {
+			ThrowPathError(ptr, end, binder);
+		}
+		switch (c) {
+		case '.': { // Object field
+			const char *key_ptr;
+			idx_t key_len;
+			if (!ReadKey(ptr, end, key_ptr, key_len)) {
+				ThrowPathError(ptr, end, binder);
 			}
-			auto key_len = ReadString(ptr, end, escaped);
-			if (key_len == 0) {
-				ThrowPathError(ptr, end);
+			if (key_len == DConstants::INVALID_INDEX) {
+				path_type = JSONPathType::WILDCARD;
 			}
-			ptr += key_len;
-			if (escaped) {
-				// Skip past closing '"'
-				ptr++;
+			break;
+		}
+		case '[': { // Array index
+			idx_t array_index;
+			bool from_back;
+			if (!ReadArrayIndex(ptr, end, array_index, from_back)) {
+				ThrowPathError(ptr, end, binder);
 			}
-		} else if (c == '[') {
-			// Array
-			if (*ptr == '#') {
-				// Index from back of array
-				ptr++;
-				if (*ptr == ']') {
-					ptr++;
-					continue;
+			if (array_index == DConstants::INVALID_INDEX) {
+				path_type = JSONPathType::WILDCARD;
+			}
+			break;
+		}
+		default:
+			ThrowPathError(ptr, end, binder);
+		}
+	}
+	return path_type;
+}
+
+yyjson_val *JSONCommon::GetPath(yyjson_val *val, const char *ptr, const idx_t &len) {
+	// Path has been validated at this point
+	const char *const end = ptr + len;
+	ptr++; // Skip past '$'
+	while (val != nullptr && ptr != end) {
+		const auto &c = *ptr++;
+		D_ASSERT(ptr != end);
+		switch (c) {
+		case '.': { // Object field
+			if (!unsafe_yyjson_is_obj(val)) {
+				return nullptr;
+			}
+			const char *key_ptr;
+			idx_t key_len;
+#ifdef DEBUG
+			bool success =
+#endif
+			    ReadKey(ptr, end, key_ptr, key_len);
+#ifdef DEBUG
+			D_ASSERT(success);
+#endif
+			val = yyjson_obj_getn(val, key_ptr, key_len);
+			break;
+		}
+		case '[': { // Array index
+			if (!unsafe_yyjson_is_arr(val)) {
+				return nullptr;
+			}
+			idx_t array_index;
+			bool from_back;
+#ifdef DEBUG
+			bool success =
+#endif
+			    ReadArrayIndex(ptr, end, array_index, from_back);
+#ifdef DEBUG
+			D_ASSERT(success);
+#endif
+			if (from_back && array_index != 0) {
+				array_index = unsafe_yyjson_get_len(val) - array_index;
+			}
+			val = yyjson_arr_get(val, array_index);
+			break;
+		}
+		default: // LCOV_EXCL_START
+			throw InternalException(
+			    "Invalid JSON Path encountered in JSONCommon::GetPath, call JSONCommon::ValidatePath first!");
+		} // LCOV_EXCL_STOP
+	}
+	return val;
+}
+
+void GetWildcardPathInternal(yyjson_val *val, const char *ptr, const char *const end, vector<yyjson_val *> &vals) {
+	while (val != nullptr && ptr != end) {
+		const auto &c = *ptr++;
+		D_ASSERT(ptr != end);
+		switch (c) {
+		case '.': { // Object field
+			if (!unsafe_yyjson_is_obj(val)) {
+				return;
+			}
+			const char *key_ptr;
+			idx_t key_len;
+#ifdef DEBUG
+			bool success =
+#endif
+			    ReadKey(ptr, end, key_ptr, key_len);
+#ifdef DEBUG
+			D_ASSERT(success);
+#endif
+			if (key_len == DConstants::INVALID_INDEX) { // Wildcard
+				size_t idx, max;
+				yyjson_val *key, *obj_val;
+				yyjson_obj_foreach(val, idx, max, key, obj_val) {
+					GetWildcardPathInternal(obj_val, ptr, end, vals);
 				}
-				if (*ptr != '-') {
-					ThrowPathError(ptr, end);
+				return;
+			}
+			val = yyjson_obj_getn(val, key_ptr, key_len);
+			break;
+		}
+		case '[': { // Array index
+			if (!unsafe_yyjson_is_arr(val)) {
+				return;
+			}
+			idx_t array_index;
+			bool from_back;
+#ifdef DEBUG
+			bool success =
+#endif
+			    ReadArrayIndex(ptr, end, array_index, from_back);
+#ifdef DEBUG
+			D_ASSERT(success);
+#endif
+
+			if (array_index == DConstants::INVALID_INDEX) { // Wildcard
+				size_t idx, max;
+				yyjson_val *arr_val;
+				yyjson_arr_foreach(val, idx, max, arr_val) {
+					GetWildcardPathInternal(arr_val, ptr, end, vals);
 				}
-				// Skip past '-'
-				ptr++;
+				return;
 			}
-			idx_t idx;
-			auto idx_len = ReadIndex(ptr, end, idx);
-			if (idx_len == 0) {
-				ThrowPathError(ptr, end);
+			if (from_back && array_index != 0) {
+				array_index = unsafe_yyjson_get_len(val) - array_index;
 			}
-			ptr += idx_len;
-			// Skip past closing ']'
-			ptr++;
-		} else {
-			ThrowPathError(ptr, end);
+			val = yyjson_arr_get(val, array_index);
+			break;
 		}
+		default: // LCOV_EXCL_START
+			throw InternalException(
+			    "Invalid JSON Path encountered in GetWildcardPathInternal, call JSONCommon::ValidatePath first!");
+		} // LCOV_EXCL_STOP
 	}
+	if (val != nullptr) {
+		vals.emplace_back(val);
+	}
+}
+
+void JSONCommon::GetWildcardPath(yyjson_val *val, const char *ptr, const idx_t &len, vector<yyjson_val *> &vals) {
+	// Path has been validated at this point
+	const char *const end = ptr + len;
+	ptr++; // Skip past '$'
+	GetWildcardPathInternal(val, ptr, end, vals);
 }
 
 } // namespace duckdb

package/src/duckdb/extension/json/json_deserializer.cpp

@@ -3,7 +3,7 @@
 
 namespace duckdb {
 
-void JsonDeserializer::SetTag(const char *tag) {
+void JsonDeserializer::SetTag(const field_id_t, const char *tag) {
 	current_tag = tag;
 }
 
@@ -43,8 +43,10 @@ void JsonDeserializer::ThrowTypeError(yyjson_val *val, const char *expected) {
 	if (yyjson_is_obj(parent.val)) {
 		auto msg =
 		    StringUtil::Format("property '%s' expected type '%s', but got type: '%s'", current_tag, expected, actual);
+		throw ParserException(msg);
 	} else if (yyjson_is_arr(parent.val)) {
 		auto msg = StringUtil::Format("Sequence expect child of type '%s', but got type: %s", expected, actual);
+		throw ParserException(msg);
 	} else {
 		// unreachable?
 		throw InternalException("cannot get nested value from non object or array-type");
@@ -122,11 +124,11 @@ void JsonDeserializer::OnMapEntryBegin() {
 }
 
 void JsonDeserializer::OnMapKeyBegin() {
-	SetTag("key");
+	SetTag(100, "key");
 }
 
 void JsonDeserializer::OnMapValueBegin() {
-	SetTag("value");
+	SetTag(101, "value");
 }
 
 void JsonDeserializer::OnMapEntryEnd() {
@@ -146,11 +148,11 @@ void JsonDeserializer::OnPairBegin() {
 }
 
 void JsonDeserializer::OnPairKeyBegin() {
-	SetTag("key");
+	SetTag(100, "key");
 }
 
 void JsonDeserializer::OnPairValueBegin() {
-	SetTag("value");
+	SetTag(101, "value");
 }
 
 void JsonDeserializer::OnPairEnd() {
@@ -178,7 +180,7 @@ bool JsonDeserializer::ReadBool() {
 
 int8_t JsonDeserializer::ReadSignedInt8() {
 	auto val = GetNextValue();
-	if (!yyjson_is_sint(val)) {
+	if (!yyjson_is_int(val)) {
 		ThrowTypeError(val, "int8_t");
 	}
 	return yyjson_get_sint(val);
@@ -194,7 +196,7 @@ uint8_t JsonDeserializer::ReadUnsignedInt8() {
 
 int16_t JsonDeserializer::ReadSignedInt16() {
 	auto val = GetNextValue();
-	if (!yyjson_is_sint(val)) {
+	if (!yyjson_is_int(val)) {
 		ThrowTypeError(val, "int16_t");
 	}
 	return yyjson_get_sint(val);
@@ -210,7 +212,7 @@ uint16_t JsonDeserializer::ReadUnsignedInt16() {
 
 int32_t JsonDeserializer::ReadSignedInt32() {
 	auto val = GetNextValue();
-	if (!yyjson_is_sint(val)) {
+	if (!yyjson_is_int(val)) {
 		ThrowTypeError(val, "int32_t");
 	}
 	return yyjson_get_sint(val);
@@ -226,7 +228,7 @@ uint32_t JsonDeserializer::ReadUnsignedInt32() {
 
 int64_t JsonDeserializer::ReadSignedInt64() {
 	auto val = GetNextValue();
-	if (!yyjson_is_sint(val)) {
+	if (!yyjson_is_int(val)) {
 		ThrowTypeError(val, "int64_t");
 	}
 	return yyjson_get_sint(val);
@@ -271,9 +273,9 @@ interval_t JsonDeserializer::ReadInterval() {
 	}
 	Push(val);
 	interval_t result;
-	ReadProperty("months", result.months);
-	ReadProperty("days", result.days);
-	ReadProperty("micros", result.micros);
+	ReadProperty(100, "months", result.months);
+	ReadProperty(101, "days", result.days);
+	ReadProperty(102, "micros", result.micros);
 	Pop();
 	return result;
 }
@@ -285,8 +287,8 @@ hugeint_t JsonDeserializer::ReadHugeInt() {
 	}
 	Push(val);
 	hugeint_t result;
-	ReadProperty("upper", result.upper);
-	ReadProperty("lower", result.lower);
+	ReadProperty(100, "upper", result.upper);
+	ReadProperty(101, "lower", result.lower);
 	Pop();
 	return result;
 }

package/src/duckdb/extension/json/json_enums.cpp

@@ -0,0 +1,105 @@
+//===----------------------------------------------------------------------===//
+// This file is automatically generated by scripts/generate_enums.py
+// Do not edit this file manually, your changes will be overwritten
+//===----------------------------------------------------------------------===//
+
+#include "json_enums.hpp"
+#include "duckdb/common/string_util.hpp"
+
+namespace duckdb {
+
+template<>
+const char* EnumUtil::ToChars<JSONScanType>(JSONScanType value) {
+	switch(value) {
+	case JSONScanType::INVALID:
+		return "INVALID";
+	case JSONScanType::READ_JSON:
+		return "READ_JSON";
+	case JSONScanType::READ_JSON_OBJECTS:
+		return "READ_JSON_OBJECTS";
+	case JSONScanType::SAMPLE:
+		return "SAMPLE";
+	default:
+		throw NotImplementedException(StringUtil::Format("Enum value of type JSONScanType: '%d' not implemented", value));
+	}
+}
+
+template<>
+JSONScanType EnumUtil::FromString<JSONScanType>(const char *value) {
+    if (StringUtil::Equals(value, "INVALID")) {
+		return JSONScanType::INVALID;
+	}
+    if (StringUtil::Equals(value, "READ_JSON")) {
+		return JSONScanType::READ_JSON;
+	}
+    if (StringUtil::Equals(value, "READ_JSON_OBJECTS")) {
+		return JSONScanType::READ_JSON_OBJECTS;
+	}
+    if (StringUtil::Equals(value, "SAMPLE")) {
+		return JSONScanType::SAMPLE;
+	}
+   throw NotImplementedException(StringUtil::Format("Enum value of type JSONScanType: '%s' not implemented", value));
+}
+
+template<>
+const char* EnumUtil::ToChars<JSONRecordType>(JSONRecordType value) {
+	switch(value) {
+	case JSONRecordType::AUTO_DETECT:
+		return "AUTO_DETECT";
+	case JSONRecordType::RECORDS:
+		return "RECORDS";
+	case JSONRecordType::VALUES:
+		return "VALUES";
+	default:
+		throw NotImplementedException(StringUtil::Format("Enum value of type JSONRecordType: '%d' not implemented", value));
+	}
+}
+
+template<>
+JSONRecordType EnumUtil::FromString<JSONRecordType>(const char *value) {
+    if (StringUtil::Equals(value, "AUTO_DETECT")) {
+		return JSONRecordType::AUTO_DETECT;
+	}
+    if (StringUtil::Equals(value, "RECORDS")) {
+		return JSONRecordType::RECORDS;
+	}
+    if (StringUtil::Equals(value, "VALUES")) {
+		return JSONRecordType::VALUES;
+	}
+   throw NotImplementedException(StringUtil::Format("Enum value of type JSONRecordType: '%s' not implemented", value));
+}
+
+template<>
+const char* EnumUtil::ToChars<JSONFormat>(JSONFormat value) {
+	switch(value) {
+	case JSONFormat::AUTO_DETECT:
+		return "AUTO_DETECT";
+	case JSONFormat::UNSTRUCTURED:
+		return "UNSTRUCTURED";
+	case JSONFormat::NEWLINE_DELIMITED:
+		return "NEWLINE_DELIMITED";
+	case JSONFormat::ARRAY:
+		return "ARRAY";
+	default:
+		throw NotImplementedException(StringUtil::Format("Enum value of type JSONFormat: '%d' not implemented", value));
+	}
+}
+
+template<>
+JSONFormat EnumUtil::FromString<JSONFormat>(const char *value) {
+    if (StringUtil::Equals(value, "AUTO_DETECT")) {
+		return JSONFormat::AUTO_DETECT;
+	}
+    if (StringUtil::Equals(value, "UNSTRUCTURED")) {
+		return JSONFormat::UNSTRUCTURED;
+	}
+    if (StringUtil::Equals(value, "NEWLINE_DELIMITED")) {
+		return JSONFormat::NEWLINE_DELIMITED;
+	}
+    if (StringUtil::Equals(value, "ARRAY")) {
+		return JSONFormat::ARRAY;
+	}
+   throw NotImplementedException(StringUtil::Format("Enum value of type JSONFormat: '%s' not implemented", value));
+}
+
+} // namespace duckdb

package/src/duckdb/extension/json/json_functions/json_create.cpp

@@ -333,8 +333,22 @@ static void CreateValuesMap(const StructNames &names, yyjson_mut_doc *doc, yyjso
 static void CreateValuesUnion(const StructNames &names, yyjson_mut_doc *doc, yyjson_mut_val *vals[], Vector &value_v,
                               idx_t count) {
 	// Structs become values, therefore we initialize vals to JSON values
-	for (idx_t i = 0; i < count; i++) {
-		vals[i] = yyjson_mut_obj(doc);
+	UnifiedVectorFormat value_data;
+	value_v.ToUnifiedFormat(count, value_data);
+	if (value_data.validity.AllValid()) {
+		for (idx_t i = 0; i < count; i++) {
+			vals[i] = yyjson_mut_obj(doc);
+		}
+	} else {
+		for (idx_t i = 0; i < count; i++) {
+			auto index = value_data.sel->get_index(i);
+			if (!value_data.validity.RowIsValid(index)) {
+				// Make the entry NULL if the Union value is NULL
+				vals[i] = yyjson_mut_null(doc);
+			} else {
+				vals[i] = yyjson_mut_obj(doc);
+			}
+		}
 	}
 
 	// Initialize re-usable array for the nested values
@@ -361,6 +375,11 @@ static void CreateValuesUnion(const StructNames &names, yyjson_mut_doc *doc, yyj
 		auto keys = UnifiedVectorFormat::GetData<string_t>(key_data);
 
 		for (idx_t i = 0; i < count; i++) {
+			auto value_index = value_data.sel->get_index(i);
+			if (!value_data.validity.RowIsValid(value_index)) {
+				// This entry is just NULL in it's entirety
+				continue;
+			}
 			auto tag_idx = tag_data.sel->get_index(i);
 			if (!tag_data.validity.RowIsValid(tag_idx)) {
 				continue;

package/src/duckdb/extension/json/json_functions/json_structure.cpp

@@ -395,7 +395,7 @@ static inline void ExtractStructureObject(yyjson_val *obj, JSONStructureNode &no
 
 static inline void ExtractStructureVal(yyjson_val *val, JSONStructureNode &node) {
 	D_ASSERT(!yyjson_is_arr(val) && !yyjson_is_obj(val));
-	node.GetOrCreateDescription(JSONCommon::ValTypeToLogicalTypeId<yyjson_val>(val));
+	node.GetOrCreateDescription(JSONCommon::ValTypeToLogicalTypeId(val));
 }
 
 void JSONStructure::ExtractStructure(yyjson_val *val, JSONStructureNode &node) {