duckdb 0.8.2-dev77.0 → 0.9.0

package/src/duckdb/extension/json/json_deserializer.cpp

@@ -3,10 +3,25 @@
 
 namespace duckdb {
 
-void JsonDeserializer::SetTag(const char *tag) {
+void JsonDeserializer::OnPropertyBegin(const field_id_t, const char *tag) {
 	current_tag = tag;
 }
 
+void JsonDeserializer::OnPropertyEnd() {
+}
+
+bool JsonDeserializer::OnOptionalPropertyBegin(const field_id_t, const char *tag) {
+	auto parent = Current();
+	auto present = yyjson_obj_get(parent.val, tag) != nullptr;
+	if (present) {
+		current_tag = tag;
+	}
+	return present;
+}
+
+void JsonDeserializer::OnOptionalPropertyEnd(bool) {
+}
+
 // If inside an object, return the value associated by the current tag (property name)
 // If inside an array, return the next element in the sequence
 yyjson_val *JsonDeserializer::GetNextValue() {
@@ -43,8 +58,10 @@ void JsonDeserializer::ThrowTypeError(yyjson_val *val, const char *expected) {
 	if (yyjson_is_obj(parent.val)) {
 		auto msg =
 		    StringUtil::Format("property '%s' expected type '%s', but got type: '%s'", current_tag, expected, actual);
+		throw ParserException(msg);
 	} else if (yyjson_is_arr(parent.val)) {
 		auto msg = StringUtil::Format("Sequence expect child of type '%s', but got type: %s", expected, actual);
+		throw ParserException(msg);
 	} else {
 		// unreachable?
 		throw InternalException("cannot get nested value from non object or array-type");
@@ -103,68 +120,29 @@ void JsonDeserializer::OnListEnd() {
 	Pop();
 }
 
-// Deserialize maps as [ { key: ..., value: ... } ]
-idx_t JsonDeserializer::OnMapBegin() {
-	auto val = GetNextValue();
-	if (!yyjson_is_arr(val)) {
-		ThrowTypeError(val, "array");
+bool JsonDeserializer::OnNullableBegin() {
+	auto &parent_val = Current();
+	yyjson_arr_iter iter;
+	if (yyjson_is_arr(parent_val.val)) {
+		iter = parent_val.arr_iter;
 	}
-	Push(val);
-	return yyjson_arr_size(val);
-}
-
-void JsonDeserializer::OnMapEntryBegin() {
 	auto val = GetNextValue();
-	if (!yyjson_is_obj(val)) {
-		ThrowTypeError(val, "object");
-	}
-	Push(val);
-}
-
-void JsonDeserializer::OnMapKeyBegin() {
-	SetTag("key");
-}
-
-void JsonDeserializer::OnMapValueBegin() {
-	SetTag("value");
-}
 
-void JsonDeserializer::OnMapEntryEnd() {
-	stack.pop_back();
-}
-
-void JsonDeserializer::OnMapEnd() {
-	stack.pop_back();
-}
-
-void JsonDeserializer::OnPairBegin() {
-	auto val = GetNextValue();
-	if (!yyjson_is_obj(val)) {
-		ThrowTypeError(val, "object");
+	// Recover the iterator if we are inside an array
+	if (yyjson_is_arr(parent_val.val)) {
+		parent_val.arr_iter = iter;
 	}
-	Push(val);
-}
-
-void JsonDeserializer::OnPairKeyBegin() {
-	SetTag("key");
-}
-
-void JsonDeserializer::OnPairValueBegin() {
-	SetTag("value");
-}
-
-void JsonDeserializer::OnPairEnd() {
-	stack.pop_back();
-}
 
-bool JsonDeserializer::OnOptionalBegin() {
-	auto val = GetNextValue();
 	if (yyjson_is_null(val)) {
 		return false;
 	}
+
 	return true;
 }
 
+void JsonDeserializer::OnNullableEnd() {
+}
+
 //===--------------------------------------------------------------------===//
 // Primitive Types
 //===--------------------------------------------------------------------===//
@@ -178,7 +156,7 @@ bool JsonDeserializer::ReadBool() {
 
 int8_t JsonDeserializer::ReadSignedInt8() {
 	auto val = GetNextValue();
-	if (!yyjson_is_sint(val)) {
+	if (!yyjson_is_int(val)) {
 		ThrowTypeError(val, "int8_t");
 	}
 	return yyjson_get_sint(val);
@@ -194,7 +172,7 @@ uint8_t JsonDeserializer::ReadUnsignedInt8() {
 
 int16_t JsonDeserializer::ReadSignedInt16() {
 	auto val = GetNextValue();
-	if (!yyjson_is_sint(val)) {
+	if (!yyjson_is_int(val)) {
 		ThrowTypeError(val, "int16_t");
 	}
 	return yyjson_get_sint(val);
@@ -210,7 +188,7 @@ uint16_t JsonDeserializer::ReadUnsignedInt16() {
 
 int32_t JsonDeserializer::ReadSignedInt32() {
 	auto val = GetNextValue();
-	if (!yyjson_is_sint(val)) {
+	if (!yyjson_is_int(val)) {
 		ThrowTypeError(val, "int32_t");
 	}
 	return yyjson_get_sint(val);
@@ -226,7 +204,7 @@ uint32_t JsonDeserializer::ReadUnsignedInt32() {
 
 int64_t JsonDeserializer::ReadSignedInt64() {
 	auto val = GetNextValue();
-	if (!yyjson_is_sint(val)) {
+	if (!yyjson_is_int(val)) {
 		ThrowTypeError(val, "int64_t");
 	}
 	return yyjson_get_sint(val);
@@ -264,20 +242,6 @@ string JsonDeserializer::ReadString() {
 	return yyjson_get_str(val);
 }
 
-interval_t JsonDeserializer::ReadInterval() {
-	auto val = GetNextValue();
-	if (!yyjson_is_obj(val)) {
-		ThrowTypeError(val, "object");
-	}
-	Push(val);
-	interval_t result;
-	ReadProperty("months", result.months);
-	ReadProperty("days", result.days);
-	ReadProperty("micros", result.micros);
-	Pop();
-	return result;
-}
-
 hugeint_t JsonDeserializer::ReadHugeInt() {
 	auto val = GetNextValue();
 	if (!yyjson_is_obj(val)) {
@@ -285,8 +249,8 @@ hugeint_t JsonDeserializer::ReadHugeInt() {
 	}
 	Push(val);
 	hugeint_t result;
-	ReadProperty("upper", result.upper);
-	ReadProperty("lower", result.lower);
+	ReadProperty(100, "upper", result.upper);
+	ReadProperty(101, "lower", result.lower);
 	Pop();
 	return result;
 }
@@ -300,7 +264,7 @@ void JsonDeserializer::ReadDataPtr(data_ptr_t &ptr, idx_t count) {
 	auto len = yyjson_get_len(val);
 	D_ASSERT(len == count);
 	auto blob = string_t(str, len);
-	Blob::ToString(blob, (char *&)ptr);
+	Blob::ToString(blob, char_ptr_cast(ptr));
 }
 
 } // namespace duckdb

package/src/duckdb/extension/json/json_enums.cpp

@@ -0,0 +1,105 @@
+//===----------------------------------------------------------------------===//
+// This file is automatically generated by scripts/generate_enums.py
+// Do not edit this file manually, your changes will be overwritten
+//===----------------------------------------------------------------------===//
+
+#include "json_enums.hpp"
+#include "duckdb/common/string_util.hpp"
+
+namespace duckdb {
+
+template<>
+const char* EnumUtil::ToChars<JSONScanType>(JSONScanType value) {
+	switch(value) {
+	case JSONScanType::INVALID:
+		return "INVALID";
+	case JSONScanType::READ_JSON:
+		return "READ_JSON";
+	case JSONScanType::READ_JSON_OBJECTS:
+		return "READ_JSON_OBJECTS";
+	case JSONScanType::SAMPLE:
+		return "SAMPLE";
+	default:
+		throw NotImplementedException(StringUtil::Format("Enum value of type JSONScanType: '%d' not implemented", value));
+	}
+}
+
+template<>
+JSONScanType EnumUtil::FromString<JSONScanType>(const char *value) {
+    if (StringUtil::Equals(value, "INVALID")) {
+		return JSONScanType::INVALID;
+	}
+    if (StringUtil::Equals(value, "READ_JSON")) {
+		return JSONScanType::READ_JSON;
+	}
+    if (StringUtil::Equals(value, "READ_JSON_OBJECTS")) {
+		return JSONScanType::READ_JSON_OBJECTS;
+	}
+    if (StringUtil::Equals(value, "SAMPLE")) {
+		return JSONScanType::SAMPLE;
+	}
+   throw NotImplementedException(StringUtil::Format("Enum value of type JSONScanType: '%s' not implemented", value));
+}
+
+template<>
+const char* EnumUtil::ToChars<JSONRecordType>(JSONRecordType value) {
+	switch(value) {
+	case JSONRecordType::AUTO_DETECT:
+		return "AUTO_DETECT";
+	case JSONRecordType::RECORDS:
+		return "RECORDS";
+	case JSONRecordType::VALUES:
+		return "VALUES";
+	default:
+		throw NotImplementedException(StringUtil::Format("Enum value of type JSONRecordType: '%d' not implemented", value));
+	}
+}
+
+template<>
+JSONRecordType EnumUtil::FromString<JSONRecordType>(const char *value) {
+    if (StringUtil::Equals(value, "AUTO_DETECT")) {
+		return JSONRecordType::AUTO_DETECT;
+	}
+    if (StringUtil::Equals(value, "RECORDS")) {
+		return JSONRecordType::RECORDS;
+	}
+    if (StringUtil::Equals(value, "VALUES")) {
+		return JSONRecordType::VALUES;
+	}
+   throw NotImplementedException(StringUtil::Format("Enum value of type JSONRecordType: '%s' not implemented", value));
+}
+
+template<>
+const char* EnumUtil::ToChars<JSONFormat>(JSONFormat value) {
+	switch(value) {
+	case JSONFormat::AUTO_DETECT:
+		return "AUTO_DETECT";
+	case JSONFormat::UNSTRUCTURED:
+		return "UNSTRUCTURED";
+	case JSONFormat::NEWLINE_DELIMITED:
+		return "NEWLINE_DELIMITED";
+	case JSONFormat::ARRAY:
+		return "ARRAY";
+	default:
+		throw NotImplementedException(StringUtil::Format("Enum value of type JSONFormat: '%d' not implemented", value));
+	}
+}
+
+template<>
+JSONFormat EnumUtil::FromString<JSONFormat>(const char *value) {
+    if (StringUtil::Equals(value, "AUTO_DETECT")) {
+		return JSONFormat::AUTO_DETECT;
+	}
+    if (StringUtil::Equals(value, "UNSTRUCTURED")) {
+		return JSONFormat::UNSTRUCTURED;
+	}
+    if (StringUtil::Equals(value, "NEWLINE_DELIMITED")) {
+		return JSONFormat::NEWLINE_DELIMITED;
+	}
+    if (StringUtil::Equals(value, "ARRAY")) {
+		return JSONFormat::ARRAY;
+	}
+   throw NotImplementedException(StringUtil::Format("Enum value of type JSONFormat: '%s' not implemented", value));
+}
+
+} // namespace duckdb

package/src/duckdb/extension/json/json_functions/copy_json.cpp

@@ -101,7 +101,7 @@ static BoundStatement CopyToJSONPlan(Binder &binder, CopyStatement &stmt) {
 	info.options["quote"] = {""};
 	info.options["escape"] = {""};
 	info.options["delimiter"] = {"\n"};
-	info.options["header"] = {0};
+	info.options["header"] = {{0}};
 
 	return binder.Bind(*stmt_copy);
 }

package/src/duckdb/extension/json/json_functions/json_create.cpp

@@ -333,8 +333,22 @@ static void CreateValuesMap(const StructNames &names, yyjson_mut_doc *doc, yyjso
 static void CreateValuesUnion(const StructNames &names, yyjson_mut_doc *doc, yyjson_mut_val *vals[], Vector &value_v,
                               idx_t count) {
 	// Structs become values, therefore we initialize vals to JSON values
-	for (idx_t i = 0; i < count; i++) {
-		vals[i] = yyjson_mut_obj(doc);
+	UnifiedVectorFormat value_data;
+	value_v.ToUnifiedFormat(count, value_data);
+	if (value_data.validity.AllValid()) {
+		for (idx_t i = 0; i < count; i++) {
+			vals[i] = yyjson_mut_obj(doc);
+		}
+	} else {
+		for (idx_t i = 0; i < count; i++) {
+			auto index = value_data.sel->get_index(i);
+			if (!value_data.validity.RowIsValid(index)) {
+				// Make the entry NULL if the Union value is NULL
+				vals[i] = yyjson_mut_null(doc);
+			} else {
+				vals[i] = yyjson_mut_obj(doc);
+			}
+		}
 	}
 
 	// Initialize re-usable array for the nested values
@@ -361,6 +375,11 @@ static void CreateValuesUnion(const StructNames &names, yyjson_mut_doc *doc, yyj
 		auto keys = UnifiedVectorFormat::GetData<string_t>(key_data);
 
 		for (idx_t i = 0; i < count; i++) {
+			auto value_index = value_data.sel->get_index(i);
+			if (!value_data.validity.RowIsValid(value_index)) {
+				// This entry is just NULL in it's entirety
+				continue;
+			}
 			auto tag_idx = tag_data.sel->get_index(i);
 			if (!tag_data.validity.RowIsValid(tag_idx)) {
 				continue;

package/src/duckdb/extension/json/json_functions/json_serialize_sql.cpp

@@ -184,7 +184,7 @@ static unique_ptr<SelectStatement> DeserializeSelectStatement(string_t input, yy
 	}
 	auto stmt_json = yyjson_arr_get(statements, 0);
 	JsonDeserializer deserializer(stmt_json, doc);
-	return SelectStatement::FormatDeserialize(deserializer);
+	return SelectStatement::Deserialize(deserializer);
 }
 
 //----------------------------------------------------------------------
@@ -244,6 +244,9 @@ struct ExecuteSqlTableFunction {
 		auto result = make_uniq<BindData>();
 
 		result->con = make_uniq<Connection>(*context.db);
+		if (input.inputs[0].IsNull()) {
+			throw BinderException("json_execute_serialized_sql cannot execute NULL plan");
+		}
 		auto serialized = input.inputs[0].GetValueUnsafe<string>();
 		auto stmt = DeserializeSelectStatement(serialized, alc);
 		result->plan = result->con->RelationFromQuery(std::move(stmt));

package/src/duckdb/extension/json/json_functions/json_structure.cpp

@@ -395,7 +395,7 @@ static inline void ExtractStructureObject(yyjson_val *obj, JSONStructureNode &no
 
 static inline void ExtractStructureVal(yyjson_val *val, JSONStructureNode &node) {
 	D_ASSERT(!yyjson_is_arr(val) && !yyjson_is_obj(val));
-	node.GetOrCreateDescription(JSONCommon::ValTypeToLogicalTypeId<yyjson_val>(val));
+	node.GetOrCreateDescription(JSONCommon::ValTypeToLogicalTypeId(val));
 }
 
 void JSONStructure::ExtractStructure(yyjson_val *val, JSONStructureNode &node) {

package/src/duckdb/extension/json/json_functions/json_transform.cpp

@@ -1,6 +1,8 @@
 #include "json_transform.hpp"
 
 #include "duckdb/common/enum_util.hpp"
+#include "duckdb/common/serializer/deserializer.hpp"
+#include "duckdb/common/serializer/serializer.hpp"
 #include "duckdb/common/types.hpp"
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/function/cast/cast_function_set.hpp"
@@ -20,28 +22,12 @@ JSONTransformOptions::JSONTransformOptions(bool strict_cast_p, bool error_duplic
       error_unknown_key(error_unkown_key_p) {
 }
 
-void JSONTransformOptions::Serialize(FieldWriter &writer) const {
-	writer.WriteField(strict_cast);
-	writer.WriteField(error_duplicate_key);
-	writer.WriteField(error_missing_key);
-	writer.WriteField(error_unknown_key);
-	writer.WriteField(delay_error);
-}
-
-void JSONTransformOptions::Deserialize(FieldReader &reader) {
-	strict_cast = reader.ReadRequired<bool>();
-	error_duplicate_key = reader.ReadRequired<bool>();
-	error_missing_key = reader.ReadRequired<bool>();
-	error_unknown_key = reader.ReadRequired<bool>();
-	delay_error = reader.ReadRequired<bool>();
-}
-
 //! Forward declaration for recursion
 static LogicalType StructureStringToType(yyjson_val *val, ClientContext &context);
 
 static LogicalType StructureStringToTypeArray(yyjson_val *arr, ClientContext &context) {
 	if (yyjson_arr_size(arr) != 1) {
-		throw InvalidInputException("Too many values in array of JSON structure");
+		throw BinderException("Too many values in array of JSON structure");
 	}
 	return LogicalType::LIST(StructureStringToType(yyjson_arr_get_first(arr), context));
 }
@@ -62,7 +48,7 @@ static LogicalType StructureToTypeObject(yyjson_val *obj, ClientContext &context
 	}
 	D_ASSERT(yyjson_obj_size(obj) == names.size());
 	if (child_types.empty()) {
-		throw InvalidInputException("Empty object in JSON structure");
+		throw BinderException("Empty object in JSON structure");
 	}
 	return LogicalType::STRUCT(child_types);
 }
@@ -76,7 +62,7 @@ static LogicalType StructureStringToType(yyjson_val *val, ClientContext &context
 	case YYJSON_TYPE_STR | YYJSON_SUBTYPE_NONE:
 		return TransformStringToLogicalType(unsafe_yyjson_get_str(val), context);
 	default:
-		throw InvalidInputException("invalid JSON structure");
+		throw BinderException("invalid JSON structure");
 	}
 }
 
@@ -86,23 +72,19 @@ static unique_ptr<FunctionData> JSONTransformBind(ClientContext &context, Scalar
 	if (arguments[1]->HasParameter()) {
 		throw ParameterNotResolvedException();
 	}
-	if (arguments[1]->return_type == LogicalTypeId::SQLNULL) {
+	if (!arguments[1]->IsFoldable()) {
+		throw BinderException("JSON structure must be a constant!");
+	}
+	auto structure_val = ExpressionExecutor::EvaluateScalar(context, *arguments[1]);
+	if (structure_val.IsNull() || arguments[1]->return_type == LogicalTypeId::SQLNULL) {
 		bound_function.return_type = LogicalTypeId::SQLNULL;
-	} else if (!arguments[1]->IsFoldable()) {
-		throw InvalidInputException("JSON structure must be a constant!");
 	} else {
-		auto structure_val = ExpressionExecutor::EvaluateScalar(context, *arguments[1]);
 		if (!structure_val.DefaultTryCastAs(JSONCommon::JSONType())) {
-			throw InvalidInputException("Cannot cast JSON structure to string");
+			throw BinderException("Cannot cast JSON structure to string");
 		}
 		auto structure_string = structure_val.GetValueUnsafe<string_t>();
 		JSONAllocator json_allocator(Allocator::DefaultAllocator());
-		yyjson_read_err err;
-		auto doc =
-		    JSONCommon::ReadDocumentUnsafe(structure_string, JSONCommon::READ_FLAG, json_allocator.GetYYAlc(), &err);
-		if (err.code != YYJSON_READ_SUCCESS) {
-			JSONCommon::ThrowParseError(structure_string.GetData(), structure_string.GetSize(), err);
-		}
+		auto doc = JSONCommon::ReadDocument(structure_string, JSONCommon::READ_FLAG, json_allocator.GetYYAlc());
 		bound_function.return_type = StructureStringToType(doc->root, context);
 	}
 	return make_uniq<VariableReturnBindData>(bound_function.return_type);
@@ -114,10 +96,9 @@ static inline string_t GetString(yyjson_val *val) {
 
 template <class T, class OP = TryCast>
 static inline bool GetValueNumerical(yyjson_val *val, T &result, JSONTransformOptions &options) {
+	D_ASSERT(unsafe_yyjson_get_tag(val) != (YYJSON_TYPE_NULL | YYJSON_SUBTYPE_NONE));
 	bool success;
 	switch (unsafe_yyjson_get_tag(val)) {
-	case YYJSON_TYPE_NULL | YYJSON_SUBTYPE_NONE:
-		return false;
 	case YYJSON_TYPE_STR | YYJSON_SUBTYPE_NONE:
 		success = OP::template Operation<string_t, T>(GetString(val), result, options.strict_cast);
 		break;
@@ -150,10 +131,9 @@ static inline bool GetValueNumerical(yyjson_val *val, T &result, JSONTransformOp
 
 template <class T, class OP = TryCastToDecimal>
 static inline bool GetValueDecimal(yyjson_val *val, T &result, uint8_t w, uint8_t s, JSONTransformOptions &options) {
+	D_ASSERT(unsafe_yyjson_get_tag(val) != (YYJSON_TYPE_NULL | YYJSON_SUBTYPE_NONE));
 	bool success;
 	switch (unsafe_yyjson_get_tag(val)) {
-	case YYJSON_TYPE_NULL | YYJSON_SUBTYPE_NONE:
-		return false;
 	case YYJSON_TYPE_STR | YYJSON_SUBTYPE_NONE:
 		success = OP::template Operation<string_t, T>(GetString(val), result, &options.error_message, w, s);
 		break;
@@ -186,9 +166,8 @@ static inline bool GetValueDecimal(yyjson_val *val, T &result, uint8_t w, uint8_
 }
 
 static inline bool GetValueString(yyjson_val *val, yyjson_alc *alc, string_t &result, Vector &vector) {
+	D_ASSERT(unsafe_yyjson_get_tag(val) != (YYJSON_TYPE_NULL | YYJSON_SUBTYPE_NONE));
 	switch (unsafe_yyjson_get_tag(val)) {
-	case YYJSON_TYPE_NULL | YYJSON_SUBTYPE_NONE:
-		return true;
 	case YYJSON_TYPE_STR | YYJSON_SUBTYPE_NONE:
 		result = string_t(unsafe_yyjson_get_str(val), unsafe_yyjson_get_len(val));
 		return true;
@@ -318,26 +297,14 @@ static bool TransformStringWithFormat(Vector &string_vector, StrpTimeFormat &for
 	auto &target_validity = FlatVector::Validity(result);
 
 	bool success = true;
-	if (source_validity.AllValid()) {
-		for (idx_t i = 0; i < count; i++) {
-			if (!OP::template Operation<T>(format, source_strings[i], target_vals[i], options.error_message)) {
-				target_validity.SetInvalid(i);
-				if (success && options.strict_cast) {
-					options.object_index = i;
-					success = false;
-				}
-			}
-		}
-	} else {
-		for (idx_t i = 0; i < count; i++) {
-			if (!source_validity.RowIsValid(i)) {
-				target_validity.SetInvalid(i);
-			} else if (!OP::template Operation<T>(format, source_strings[i], target_vals[i], options.error_message)) {
-				target_validity.SetInvalid(i);
-				if (success && options.strict_cast) {
-					options.object_index = i;
-					success = false;
-				}
+	for (idx_t i = 0; i < count; i++) {
+		if (!source_validity.RowIsValid(i)) {
+			target_validity.SetInvalid(i);
+		} else if (!OP::template Operation<T>(format, source_strings[i], target_vals[i], options.error_message)) {
+			target_validity.SetInvalid(i);
+			if (success && options.strict_cast) {
+				options.object_index = i;
+				success = false;
 			}
 		}
 	}
@@ -689,6 +656,77 @@ bool TransformToJSON(yyjson_val *vals[], yyjson_alc *alc, Vector &result, const
 	return true;
 }
 
+bool TransformValueIntoUnion(yyjson_val **vals, yyjson_alc *alc, Vector &result, const idx_t count,
+                             JSONTransformOptions &options) {
+	auto type = result.GetType();
+
+	auto fields = UnionType::CopyMemberTypes(type);
+	vector<string> names;
+	for (const auto &field : fields) {
+		names.push_back(field.first);
+	}
+
+	bool success = true;
+
+	auto &validity = FlatVector::Validity(result);
+
+	auto set_error = [&](idx_t i, const string &message) {
+		validity.SetInvalid(i);
+		result.SetValue(i, Value(nullptr));
+		if (success && options.strict_cast) {
+			options.error_message = message;
+			options.object_index = i;
+			success = false;
+		}
+	};
+
+	for (idx_t i = 0; i < count; i++) {
+		const auto &obj = vals[i];
+
+		if (!obj || unsafe_yyjson_is_null(vals[i])) {
+			validity.SetInvalid(i);
+			result.SetValue(i, Value(nullptr));
+			continue;
+		}
+
+		if (!unsafe_yyjson_is_obj(obj)) {
+			set_error(i,
+			          StringUtil::Format("Expected an object representing a union, got %s", yyjson_get_type_desc(obj)));
+			continue;
+		}
+
+		auto len = unsafe_yyjson_get_len(obj);
+		if (len > 1) {
+			set_error(i, "Found object containing more than one key, instead of union");
+			continue;
+		} else if (len == 0) {
+			set_error(i, "Found empty object, instead of union");
+			continue;
+		}
+
+		auto key = unsafe_yyjson_get_first(obj);
+		auto val = yyjson_obj_iter_get_val(key);
+
+		auto tag = std::find(names.begin(), names.end(), unsafe_yyjson_get_str(key));
+		if (tag == names.end()) {
+			set_error(i, StringUtil::Format("Found object containing unknown key, instead of union: %s",
+			                                unsafe_yyjson_get_str(key)));
+			continue;
+		}
+
+		idx_t actual_tag = tag - names.begin();
+
+		Vector single(UnionType::GetMemberType(type, actual_tag), 1);
+		if (!JSONTransform::Transform(&val, alc, single, 1, options)) {
+			success = false;
+		}
+
+		result.SetValue(i, Value::UNION(fields, actual_tag, single.GetValue(0)));
+	}
+
+	return success;
+}
+
 bool JSONTransform::Transform(yyjson_val *vals[], yyjson_alc *alc, Vector &result, const idx_t count,
                               JSONTransformOptions &options) {
 	auto result_type = result.GetType();
@@ -704,6 +742,7 @@ bool JSONTransform::Transform(yyjson_val *vals[], yyjson_alc *alc, Vector &resul
 
 	switch (result_type.id()) {
 	case LogicalTypeId::SQLNULL:
+		FlatVector::Validity(result).SetAllInvalid(count);
 		return true;
 	case LogicalTypeId::BOOLEAN:
 		return TransformNumerical<bool>(vals, result, count, options);
@@ -767,8 +806,10 @@ bool JSONTransform::Transform(yyjson_val *vals[], yyjson_alc *alc, Vector &resul
 		return TransformArray(vals, alc, result, count, options);
 	case LogicalTypeId::MAP:
 		return TransformObjectToMap(vals, alc, result, count, options);
+	case LogicalTypeId::UNION:
+		return TransformValueIntoUnion(vals, alc, result, count, options);
 	default:
-		throw InternalException("Unexpected type at JSON Transform %s", result_type.ToString());
+		throw NotImplementedException("Cannot read a value of type %s from a json file", result_type.ToString());
 	}
 }
 

package/src/duckdb/extension/json/json_functions/json_type.cpp

@@ -3,7 +3,7 @@
 namespace duckdb {
 
 static inline string_t GetType(yyjson_val *val, yyjson_alc *alc, Vector &result) {
-	return JSONCommon::ValTypeToStringT<yyjson_val>(val);
+	return JSONCommon::ValTypeToStringT(val);
 }
 
 static void UnaryTypeFunction(DataChunk &args, ExpressionState &state, Vector &result) {

package/src/duckdb/extension/json/json_functions/read_json.cpp

@@ -17,6 +17,7 @@ void JSONScan::AutoDetect(ClientContext &context, JSONScanData &bind_data, vecto
 	Vector string_vector(LogicalType::VARCHAR);
 
 	// Loop through the files (if union_by_name, else just sample the first file)
+	idx_t remaining = bind_data.sample_size;
 	for (idx_t file_idx = 0; file_idx < bind_data.files.size(); file_idx++) {
 		// Create global/local state and place the reader in the right field
 		JSONScanGlobalState gstate(context, bind_data);
@@ -28,7 +29,6 @@ void JSONScan::AutoDetect(ClientContext &context, JSONScanData &bind_data, vecto
 		}
 
 		// Read and detect schema
-		idx_t remaining = bind_data.sample_size;
 		while (remaining != 0) {
 			allocator.Reset();
 			auto read_count = lstate.ReadNext(gstate);
@@ -56,7 +56,11 @@ void JSONScan::AutoDetect(ClientContext &context, JSONScanData &bind_data, vecto
 		}
 
 		// Close the file and stop detection if not union_by_name
-		if (!bind_data.options.file_options.union_by_name) {
+		if (bind_data.options.file_options.union_by_name) {
+			// When union_by_name=true we sample sample_size per file
+			remaining = bind_data.sample_size;
+		} else if (remaining == 0) {
+			// When union_by_name=false, we sample sample_size in total (across the first files)
 			break;
 		}
 	}
@@ -225,6 +229,21 @@ unique_ptr<FunctionData> ReadJSONBind(ClientContext &context, TableFunctionBindI
 	transform_options.error_unknown_key = bind_data->auto_detect && !bind_data->ignore_errors;
 	transform_options.delay_error = true;
 
+	if (bind_data->auto_detect) {
+		// JSON may contain columns such as "id" and "Id", which are duplicates for us due to case-insensitivity
+		// We rename them so we can parse the file anyway. Note that we can't change bind_data->names,
+		// because the JSON reader gets columns by exact name, not position
+		case_insensitive_map_t<idx_t> name_count_map;
+		for (auto &name : names) {
+			auto it = name_count_map.find(name);
+			if (it == name_count_map.end()) {
+				name_count_map[name] = 1;
+			} else {
+				name = StringUtil::Format("%s_%llu", name, it->second++);
+			}
+		}
+	}
+
 	return std::move(bind_data);
 }