duckdb 1.0.1-dev22.0 → 1.0.1-dev27.0

package/src/duckdb/extension/json/json_functions/json_structure.cpp

@@ -5,13 +5,15 @@
 #include "json_scan.hpp"
 #include "json_transform.hpp"
 
+#include <duckdb/common/extra_type_info.hpp>
+
 namespace duckdb {
 
-static inline bool IsNumeric(LogicalTypeId type) {
+static bool IsNumeric(LogicalTypeId type) {
 	return type == LogicalTypeId::DOUBLE || type == LogicalTypeId::UBIGINT || type == LogicalTypeId::BIGINT;
 }
 
-static inline LogicalTypeId MaxNumericType(LogicalTypeId &a, LogicalTypeId &b) {
+static LogicalTypeId MaxNumericType(const LogicalTypeId &a, const LogicalTypeId &b) {
 	D_ASSERT(a != b);
 	if (a == LogicalTypeId::DOUBLE || b == LogicalTypeId::DOUBLE) {
 		return LogicalTypeId::DOUBLE;
@@ -19,31 +21,36 @@ static inline LogicalTypeId MaxNumericType(LogicalTypeId &a, LogicalTypeId &b) {
 	return LogicalTypeId::BIGINT;
 }
 
-JSONStructureNode::JSONStructureNode() : initialized(false), count(0) {
+JSONStructureNode::JSONStructureNode() : count(0), null_count(0) {
+}
+
+JSONStructureNode::JSONStructureNode(const char *key_ptr, const size_t key_len) : JSONStructureNode() {
+	key = make_uniq<string>(key_ptr, key_len);
 }
 
 JSONStructureNode::JSONStructureNode(yyjson_val *key_p, yyjson_val *val_p, const bool ignore_errors)
-    : key(make_uniq<string>(unsafe_yyjson_get_str(key_p), unsafe_yyjson_get_len(key_p))), initialized(false), count(0) {
-	D_ASSERT(yyjson_is_str(key_p));
+    : JSONStructureNode(unsafe_yyjson_get_str(key_p), unsafe_yyjson_get_len(key_p)) {
 	JSONStructure::ExtractStructure(val_p, *this, ignore_errors);
 }
 
+static void SwapJSONStructureNode(JSONStructureNode &a, JSONStructureNode &b) noexcept {
+	std::swap(a.key, b.key);
+	std::swap(a.initialized, b.initialized);
+	std::swap(a.descriptions, b.descriptions);
+	std::swap(a.count, b.count);
+	std::swap(a.null_count, b.null_count);
+}
+
 JSONStructureNode::JSONStructureNode(JSONStructureNode &&other) noexcept {
-	std::swap(key, other.key);
-	std::swap(initialized, other.initialized);
-	std::swap(descriptions, other.descriptions);
-	std::swap(count, other.count);
+	SwapJSONStructureNode(*this, other);
 }
 
 JSONStructureNode &JSONStructureNode::operator=(JSONStructureNode &&other) noexcept {
-	std::swap(key, other.key);
-	std::swap(initialized, other.initialized);
-	std::swap(descriptions, other.descriptions);
-	std::swap(count, other.count);
+	SwapJSONStructureNode(*this, other);
 	return *this;
 }
 
-JSONStructureDescription &JSONStructureNode::GetOrCreateDescription(LogicalTypeId type) {
+JSONStructureDescription &JSONStructureNode::GetOrCreateDescription(const LogicalTypeId type) {
 	if (descriptions.empty()) {
 		// Empty, just put this type in there
 		descriptions.emplace_back(type);
@@ -66,7 +73,8 @@ JSONStructureDescription &JSONStructureNode::GetOrCreateDescription(LogicalTypeI
 	for (auto &description : descriptions) {
 		if (type == description.type) {
 			return description;
-		} else if (is_numeric && IsNumeric(description.type)) {
+		}
+		if (is_numeric && IsNumeric(description.type)) {
 			description.type = MaxNumericType(type, description.type);
 			return description;
 		}
@@ -95,7 +103,7 @@ bool JSONStructureNode::ContainsVarchar() const {
 }
 
 void JSONStructureNode::InitializeCandidateTypes(const idx_t max_depth, const bool convert_strings_to_integers,
-                                                 idx_t depth) {
+                                                 const idx_t depth) {
 	if (depth >= max_depth) {
 		return;
 	}
@@ -113,14 +121,15 @@ void JSONStructureNode::InitializeCandidateTypes(const idx_t max_depth, const bo
 			description.candidate_types = {LogicalTypeId::UUID, LogicalTypeId::TIMESTAMP, LogicalTypeId::DATE,
 			                               LogicalTypeId::TIME};
 		}
-	}
-	initialized = true;
-	for (auto &child : description.children) {
-		child.InitializeCandidateTypes(max_depth, convert_strings_to_integers, depth + 1);
+		initialized = true;
+	} else {
+		for (auto &child : description.children) {
+			child.InitializeCandidateTypes(max_depth, convert_strings_to_integers, depth + 1);
+		}
 	}
 }
 
-void JSONStructureNode::RefineCandidateTypes(yyjson_val *vals[], idx_t val_count, Vector &string_vector,
+void JSONStructureNode::RefineCandidateTypes(yyjson_val *vals[], const idx_t val_count, Vector &string_vector,
                                              ArenaAllocator &allocator, DateFormatMap &date_format_map) {
 	if (descriptions.size() != 1) {
 		// We can't refine types if we have more than 1 description (yet), defaults to JSON type for now
@@ -142,7 +151,7 @@ void JSONStructureNode::RefineCandidateTypes(yyjson_val *vals[], idx_t val_count
 	}
 }
 
-void JSONStructureNode::RefineCandidateTypesArray(yyjson_val *vals[], idx_t val_count, Vector &string_vector,
+void JSONStructureNode::RefineCandidateTypesArray(yyjson_val *vals[], const idx_t val_count, Vector &string_vector,
                                                   ArenaAllocator &allocator, DateFormatMap &date_format_map) {
 	D_ASSERT(descriptions.size() == 1 && descriptions[0].type == LogicalTypeId::LIST);
 	auto &desc = descriptions[0];
@@ -173,7 +182,7 @@ void JSONStructureNode::RefineCandidateTypesArray(yyjson_val *vals[], idx_t val_
 	child.RefineCandidateTypes(child_vals, total_list_size, string_vector, allocator, date_format_map);
 }
 
-void JSONStructureNode::RefineCandidateTypesObject(yyjson_val *vals[], idx_t val_count, Vector &string_vector,
+void JSONStructureNode::RefineCandidateTypesObject(yyjson_val *vals[], const idx_t val_count, Vector &string_vector,
                                                    ArenaAllocator &allocator, DateFormatMap &date_format_map) {
 	D_ASSERT(descriptions.size() == 1 && descriptions[0].type == LogicalTypeId::STRUCT);
 	auto &desc = descriptions[0];
@@ -186,22 +195,21 @@ void JSONStructureNode::RefineCandidateTypesObject(yyjson_val *vals[], idx_t val
 		    reinterpret_cast<yyjson_val **>(allocator.AllocateAligned(val_count * sizeof(yyjson_val *))));
 	}
 
-	idx_t found_key_count;
-	auto found_keys = reinterpret_cast<bool *>(allocator.AllocateAligned(sizeof(bool) * child_count));
+	const auto found_keys = reinterpret_cast<bool *>(allocator.AllocateAligned(sizeof(bool) * child_count));
 
 	const auto &key_map = desc.key_map;
 	size_t idx, max;
 	yyjson_val *child_key, *child_val;
 	for (idx_t i = 0; i < val_count; i++) {
 		if (vals[i] && !unsafe_yyjson_is_null(vals[i])) {
-			found_key_count = 0;
+			idx_t found_key_count = 0;
 			memset(found_keys, false, child_count);
 
 			D_ASSERT(yyjson_is_obj(vals[i]));
 			yyjson_obj_foreach(vals[i], idx, max, child_key, child_val) {
 				D_ASSERT(yyjson_is_str(child_key));
-				auto key_ptr = unsafe_yyjson_get_str(child_key);
-				auto key_len = unsafe_yyjson_get_len(child_key);
+				const auto key_ptr = unsafe_yyjson_get_str(child_key);
+				const auto key_len = unsafe_yyjson_get_len(child_key);
 				auto it = key_map.find({key_ptr, key_len});
 				D_ASSERT(it != key_map.end());
 				const auto child_idx = it->second;
@@ -231,7 +239,7 @@ void JSONStructureNode::RefineCandidateTypesObject(yyjson_val *vals[], idx_t val
 	}
 }
 
-void JSONStructureNode::RefineCandidateTypesString(yyjson_val *vals[], idx_t val_count, Vector &string_vector,
+void JSONStructureNode::RefineCandidateTypesString(yyjson_val *vals[], const idx_t val_count, Vector &string_vector,
                                                    DateFormatMap &date_format_map) {
 	D_ASSERT(descriptions.size() == 1 && descriptions[0].type == LogicalTypeId::VARCHAR);
 	if (descriptions[0].candidate_types.empty()) {
@@ -242,7 +250,7 @@ void JSONStructureNode::RefineCandidateTypesString(yyjson_val *vals[], idx_t val
 	EliminateCandidateTypes(val_count, string_vector, date_format_map);
 }
 
-void JSONStructureNode::EliminateCandidateTypes(idx_t vec_count, Vector &string_vector,
+void JSONStructureNode::EliminateCandidateTypes(const idx_t vec_count, Vector &string_vector,
                                                 DateFormatMap &date_format_map) {
 	D_ASSERT(descriptions.size() == 1 && descriptions[0].type == LogicalTypeId::VARCHAR);
 	auto &description = descriptions[0];
@@ -296,12 +304,12 @@ bool TryParse(Vector &string_vector, StrpTimeFormat &format, const idx_t count)
 	return true;
 }
 
-bool JSONStructureNode::EliminateCandidateFormats(idx_t vec_count, Vector &string_vector, Vector &result_vector,
-                                                  vector<StrpTimeFormat> &formats) {
+bool JSONStructureNode::EliminateCandidateFormats(const idx_t vec_count, Vector &string_vector,
+                                                  const Vector &result_vector, vector<StrpTimeFormat> &formats) {
 	D_ASSERT(descriptions.size() == 1 && descriptions[0].type == LogicalTypeId::VARCHAR);
 	const auto type = result_vector.GetType().id();
 	for (idx_t i = formats.size(); i != 0; i--) {
-		idx_t actual_index = i - 1;
+		const idx_t actual_index = i - 1;
 		auto &format = formats[actual_index];
 		bool success;
 		switch (type) {
@@ -324,21 +332,22 @@ bool JSONStructureNode::EliminateCandidateFormats(idx_t vec_count, Vector &strin
 	return false;
 }
 
-JSONStructureDescription::JSONStructureDescription(LogicalTypeId type_p) : type(type_p) {
+JSONStructureDescription::JSONStructureDescription(const LogicalTypeId type_p) : type(type_p) {
+}
+
+static void SwapJSONStructureDescription(JSONStructureDescription &a, JSONStructureDescription &b) noexcept {
+	std::swap(a.type, b.type);
+	std::swap(a.key_map, b.key_map);
+	std::swap(a.children, b.children);
+	std::swap(a.candidate_types, b.candidate_types);
 }
 
 JSONStructureDescription::JSONStructureDescription(JSONStructureDescription &&other) noexcept {
-	std::swap(type, other.type);
-	std::swap(key_map, other.key_map);
-	std::swap(children, other.children);
-	std::swap(candidate_types, other.candidate_types);
+	SwapJSONStructureDescription(*this, other);
 }
 
 JSONStructureDescription &JSONStructureDescription::operator=(JSONStructureDescription &&other) noexcept {
-	std::swap(type, other.type);
-	std::swap(key_map, other.key_map);
-	std::swap(children, other.children);
-	std::swap(candidate_types, other.candidate_types);
+	SwapJSONStructureDescription(*this, other);
 	return *this;
 }
 
@@ -351,27 +360,31 @@ JSONStructureNode &JSONStructureDescription::GetOrCreateChild() {
 	return children.back();
 }
 
+JSONStructureNode &JSONStructureDescription::GetOrCreateChild(const char *key_ptr, const size_t key_size) {
+	// Check if there is already a child with the same key
+	const JSONKey temp_key {key_ptr, key_size};
+	const auto it = key_map.find(temp_key);
+	if (it != key_map.end()) {
+		return children[it->second]; // Found it
+	}
+
+	// Didn't find, create a new child
+	children.emplace_back(key_ptr, key_size);
+	const auto &persistent_key_string = *children.back().key;
+	JSONKey new_key {persistent_key_string.c_str(), persistent_key_string.length()};
+	key_map.emplace(new_key, children.size() - 1);
+	return children.back();
+}
+
 JSONStructureNode &JSONStructureDescription::GetOrCreateChild(yyjson_val *key, yyjson_val *val,
                                                               const bool ignore_errors) {
 	D_ASSERT(yyjson_is_str(key));
-	// Check if there is already a child with the same key
-	idx_t child_idx;
-	JSONKey temp_key {unsafe_yyjson_get_str(key), unsafe_yyjson_get_len(key)};
-	auto it = key_map.find(temp_key);
-	if (it == key_map.end()) { // Didn't find, create a new child
-		child_idx = children.size();
-		children.emplace_back(key, val, ignore_errors);
-		const auto &persistent_key_string = children.back().key;
-		JSONKey new_key {persistent_key_string->c_str(), persistent_key_string->length()};
-		key_map.emplace(new_key, child_idx);
-	} else { // Found it
-		child_idx = it->second;
-		JSONStructure::ExtractStructure(val, children[child_idx], ignore_errors);
-	}
-	return children[child_idx];
-}
-
-static inline void ExtractStructureArray(yyjson_val *arr, JSONStructureNode &node, const bool ignore_errors) {
+	auto &child = GetOrCreateChild(unsafe_yyjson_get_str(key), unsafe_yyjson_get_len(key));
+	JSONStructure::ExtractStructure(val, child, ignore_errors);
+	return child;
+}
+
+static void ExtractStructureArray(yyjson_val *arr, JSONStructureNode &node, const bool ignore_errors) {
 	D_ASSERT(yyjson_is_arr(arr));
 	auto &description = node.GetOrCreateDescription(LogicalTypeId::LIST);
 	auto &child = description.GetOrCreateChild();
@@ -383,7 +396,7 @@ static inline void ExtractStructureArray(yyjson_val *arr, JSONStructureNode &nod
 	}
 }
 
-static inline void ExtractStructureObject(yyjson_val *obj, JSONStructureNode &node, const bool ignore_errors) {
+static void ExtractStructureObject(yyjson_val *obj, JSONStructureNode &node, const bool ignore_errors) {
 	D_ASSERT(yyjson_is_obj(obj));
 	auto &description = node.GetOrCreateDescription(LogicalTypeId::STRUCT);
 
@@ -409,14 +422,19 @@ static inline void ExtractStructureObject(yyjson_val *obj, JSONStructureNode &no
 	}
 }
 
-static inline void ExtractStructureVal(yyjson_val *val, JSONStructureNode &node) {
+static void ExtractStructureVal(yyjson_val *val, JSONStructureNode &node) {
 	D_ASSERT(!yyjson_is_arr(val) && !yyjson_is_obj(val));
 	node.GetOrCreateDescription(JSONCommon::ValTypeToLogicalTypeId(val));
 }
 
 void JSONStructure::ExtractStructure(yyjson_val *val, JSONStructureNode &node, const bool ignore_errors) {
 	node.count++;
-	switch (yyjson_get_tag(val)) {
+	const auto tag = yyjson_get_tag(val);
+	if (tag == (YYJSON_TYPE_NULL | YYJSON_SUBTYPE_NONE)) {
+		node.null_count++;
+	}
+
+	switch (tag) {
 	case YYJSON_TYPE_ARR | YYJSON_SUBTYPE_NONE:
 		return ExtractStructureArray(val, node, ignore_errors);
 	case YYJSON_TYPE_OBJ | YYJSON_SUBTYPE_NONE:
@@ -433,19 +451,19 @@ JSONStructureNode ExtractStructureInternal(yyjson_val *val, const bool ignore_er
 }
 
 //! Forward declaration for recursion
-static inline yyjson_mut_val *ConvertStructure(const JSONStructureNode &node, yyjson_mut_doc *doc);
+static yyjson_mut_val *ConvertStructure(const JSONStructureNode &node, yyjson_mut_doc *doc);
 
-static inline yyjson_mut_val *ConvertStructureArray(const JSONStructureNode &node, yyjson_mut_doc *doc) {
+static yyjson_mut_val *ConvertStructureArray(const JSONStructureNode &node, yyjson_mut_doc *doc) {
 	D_ASSERT(node.descriptions.size() == 1 && node.descriptions[0].type == LogicalTypeId::LIST);
 	const auto &desc = node.descriptions[0];
 	D_ASSERT(desc.children.size() == 1);
 
-	auto arr = yyjson_mut_arr(doc);
+	const auto arr = yyjson_mut_arr(doc);
 	yyjson_mut_arr_append(arr, ConvertStructure(desc.children[0], doc));
 	return arr;
 }
 
-static inline yyjson_mut_val *ConvertStructureObject(const JSONStructureNode &node, yyjson_mut_doc *doc) {
+static yyjson_mut_val *ConvertStructureObject(const JSONStructureNode &node, yyjson_mut_doc *doc) {
 	D_ASSERT(node.descriptions.size() == 1 && node.descriptions[0].type == LogicalTypeId::STRUCT);
 	auto &desc = node.descriptions[0];
 	if (desc.children.empty()) {
@@ -453,7 +471,7 @@ static inline yyjson_mut_val *ConvertStructureObject(const JSONStructureNode &no
 		return yyjson_mut_str(doc, LogicalType::JSON_TYPE_NAME);
 	}
 
-	auto obj = yyjson_mut_obj(doc);
+	const auto obj = yyjson_mut_obj(doc);
 	for (auto &child : desc.children) {
 		D_ASSERT(child.key);
 		yyjson_mut_obj_add(obj, yyjson_mut_strn(doc, child.key->c_str(), child.key->length()),
@@ -462,7 +480,7 @@ static inline yyjson_mut_val *ConvertStructureObject(const JSONStructureNode &no
 	return obj;
 }
 
-static inline yyjson_mut_val *ConvertStructure(const JSONStructureNode &node, yyjson_mut_doc *doc) {
+static yyjson_mut_val *ConvertStructure(const JSONStructureNode &node, yyjson_mut_doc *doc) {
 	if (node.descriptions.empty()) {
 		return yyjson_mut_str(doc, JSONCommon::TYPE_STRING_NULL);
 	}
@@ -481,7 +499,7 @@ static inline yyjson_mut_val *ConvertStructure(const JSONStructureNode &node, yy
 	}
 }
 
-static inline string_t JSONStructureFunction(yyjson_val *val, yyjson_alc *alc, Vector &) {
+static string_t JSONStructureFunction(yyjson_val *val, yyjson_alc *alc, Vector &, ValidityMask &, idx_t) {
 	return JSONCommon::WriteVal<yyjson_mut_val>(
 	    ConvertStructure(ExtractStructureInternal(val, true), yyjson_mut_doc_new(alc)), alc);
 }
@@ -503,45 +521,236 @@ ScalarFunctionSet JSONFunctions::GetStructureFunction() {
 }
 
 static LogicalType StructureToTypeArray(ClientContext &context, const JSONStructureNode &node, const idx_t max_depth,
-                                        const double field_appearance_threshold, idx_t depth) {
+                                        const double field_appearance_threshold, const idx_t map_inference_threshold,
+                                        const idx_t depth, const LogicalType &null_type) {
 	D_ASSERT(node.descriptions.size() == 1 && node.descriptions[0].type == LogicalTypeId::LIST);
 	const auto &desc = node.descriptions[0];
 	D_ASSERT(desc.children.size() == 1);
 
-	return LogicalType::LIST(JSONStructure::StructureToType(
-	    context, desc.children[0], max_depth, field_appearance_threshold, depth + 1, desc.children[0].count));
+	return LogicalType::LIST(JSONStructure::StructureToType(context, desc.children[0], max_depth,
+	                                                        field_appearance_threshold, map_inference_threshold,
+	                                                        depth + 1, null_type));
+}
+
+static void MergeNodes(JSONStructureNode &merged, const JSONStructureNode &node);
+
+static void MergeNodeArray(JSONStructureNode &merged, const JSONStructureDescription &child_desc) {
+	D_ASSERT(child_desc.type == LogicalTypeId::LIST);
+	auto &merged_desc = merged.GetOrCreateDescription(LogicalTypeId::LIST);
+	auto &merged_child = merged_desc.GetOrCreateChild();
+	for (auto &list_child : child_desc.children) {
+		MergeNodes(merged_child, list_child);
+	}
+}
+
+static void MergeNodeObject(JSONStructureNode &merged, const JSONStructureDescription &child_desc) {
+	D_ASSERT(child_desc.type == LogicalTypeId::STRUCT);
+	auto &merged_desc = merged.GetOrCreateDescription(LogicalTypeId::STRUCT);
+	for (auto &struct_child : child_desc.children) {
+		const auto &struct_child_key = *struct_child.key;
+		auto &merged_child = merged_desc.GetOrCreateChild(struct_child_key.c_str(), struct_child_key.length());
+		MergeNodes(merged_child, struct_child);
+	}
+}
+
+static void MergeNodeVal(JSONStructureNode &merged, const JSONStructureDescription &child_desc,
+                         const bool node_initialized) {
+	D_ASSERT(child_desc.type != LogicalTypeId::LIST && child_desc.type != LogicalTypeId::STRUCT);
+	auto &merged_desc = merged.GetOrCreateDescription(child_desc.type);
+	if (merged_desc.type != LogicalTypeId::VARCHAR || !node_initialized || merged.descriptions.size() != 1) {
+		return;
+	}
+	if (!merged.initialized) {
+		merged_desc.candidate_types = child_desc.candidate_types;
+	} else if (!merged_desc.candidate_types.empty() && !child_desc.candidate_types.empty() &&
+	           merged_desc.candidate_types.back() != child_desc.candidate_types.back()) {
+		merged_desc.candidate_types.clear(); // Not the same, default to VARCHAR
+	}
+	merged.initialized = true;
+}
+
+static void MergeNodes(JSONStructureNode &merged, const JSONStructureNode &node) {
+	merged.count += node.count;
+	merged.null_count += node.null_count;
+	for (const auto &child_desc : node.descriptions) {
+		switch (child_desc.type) {
+		case LogicalTypeId::LIST:
+			MergeNodeArray(merged, child_desc);
+			break;
+		case LogicalTypeId::STRUCT:
+			MergeNodeObject(merged, child_desc);
+			break;
+		default:
+			MergeNodeVal(merged, child_desc, node.initialized);
+			break;
+		}
+	}
+}
+
+static double CalculateTypeSimilarity(const LogicalType &merged, const LogicalType &type, idx_t max_depth, idx_t depth);
+
+static double CalculateMapAndStructSimilarity(const LogicalType &map_type, const LogicalType &struct_type,
+                                              const bool swapped, const idx_t max_depth, const idx_t depth) {
+	const auto &map_value_type = MapType::ValueType(map_type);
+	const auto &struct_child_types = StructType::GetChildTypes(struct_type);
+	double total_similarity = 0;
+	for (const auto &struct_child_type : struct_child_types) {
+		const auto similarity =
+		    swapped ? CalculateTypeSimilarity(struct_child_type.second, map_value_type, max_depth, depth + 1)
+		            : CalculateTypeSimilarity(map_value_type, struct_child_type.second, max_depth, depth + 1);
+		if (similarity < 0) {
+			return similarity;
+		}
+		total_similarity += similarity;
+	}
+	return total_similarity / static_cast<double>(struct_child_types.size());
+}
+
+static double CalculateTypeSimilarity(const LogicalType &merged, const LogicalType &type, const idx_t max_depth,
+                                      const idx_t depth) {
+	if (depth >= max_depth || merged.id() == LogicalTypeId::SQLNULL || type.id() == LogicalTypeId::SQLNULL) {
+		return 1;
+	}
+	if (merged.IsJSONType()) {
+		// Incompatible types
+		return -1;
+	}
+	if (type.IsJSONType() || merged == type) {
+		return 1;
+	}
+
+	switch (merged.id()) {
+	case LogicalTypeId::STRUCT: {
+		if (type.id() == LogicalTypeId::MAP) {
+			// This can happen for empty structs/maps ("{}"), or in rare cases where an inconsistent struct becomes
+			// consistent when merged, but does not have enough children to be considered a map.
+			return CalculateMapAndStructSimilarity(type, merged, true, max_depth, depth);
+		}
+
+		// Only structs can be merged into a struct
+		D_ASSERT(type.id() == LogicalTypeId::STRUCT);
+		const auto &merged_child_types = StructType::GetChildTypes(merged);
+		const auto &type_child_types = StructType::GetChildTypes(type);
+
+		unordered_map<string, const LogicalType &> merged_child_types_map;
+		for (const auto &merged_child : merged_child_types) {
+			merged_child_types_map.emplace(merged_child.first, merged_child.second);
+		}
+
+		double total_similarity = 0;
+		for (const auto &type_child_type : type_child_types) {
+			const auto it = merged_child_types_map.find(type_child_type.first);
+			if (it == merged_child_types_map.end()) {
+				return -1;
+			}
+			const auto similarity = CalculateTypeSimilarity(it->second, type_child_type.second, max_depth, depth + 1);
+			if (similarity < 0) {
+				return similarity;
+			}
+			total_similarity += similarity;
+		}
+		return total_similarity / static_cast<double>(merged_child_types.size());
+	}
+	case LogicalTypeId::MAP: {
+		if (type.id() == LogicalTypeId::MAP) {
+			return CalculateTypeSimilarity(MapType::ValueType(merged), MapType::ValueType(type), max_depth, depth + 1);
+		}
+
+		// Only maps and structs can be merged into a map
+		D_ASSERT(type.id() == LogicalTypeId::STRUCT);
+		return CalculateMapAndStructSimilarity(merged, type, false, max_depth, depth);
+	}
+	case LogicalTypeId::LIST: {
+		// Only lists can be merged into a list
+		D_ASSERT(type.id() == LogicalTypeId::LIST);
+		const auto &merged_child_type = ListType::GetChildType(merged);
+		const auto &type_child_type = ListType::GetChildType(type);
+		return CalculateTypeSimilarity(merged_child_type, type_child_type, max_depth, depth + 1);
+	}
+	default:
+		// This is only reachable if type has been inferred using candidate_types, but candidate_types were not
+		// consistent among all map values
+		return 1;
+	}
+}
+
+static bool IsStructureInconsistent(const JSONStructureDescription &desc, const idx_t sample_count,
+                                    const idx_t null_count, const double field_appearance_threshold) {
+	D_ASSERT(sample_count > null_count);
+	double total_child_counts = 0;
+	for (const auto &child : desc.children) {
+		total_child_counts += static_cast<double>(child.count) / static_cast<double>(sample_count - null_count);
+	}
+	const auto avg_occurrence = total_child_counts / static_cast<double>(desc.children.size());
+	return avg_occurrence < field_appearance_threshold;
+}
+
+static LogicalType GetMergedType(ClientContext &context, const JSONStructureNode &node, const idx_t max_depth,
+                                 const double field_appearance_threshold, const idx_t map_inference_threshold,
+                                 const idx_t depth, const LogicalType &null_type) {
+	D_ASSERT(node.descriptions.size() == 1);
+	auto &desc = node.descriptions[0];
+	JSONStructureNode merged;
+	for (const auto &child : desc.children) {
+		MergeNodes(merged, child);
+	}
+	return JSONStructure::StructureToType(context, merged, max_depth, field_appearance_threshold,
+	                                      map_inference_threshold, depth + 1, null_type);
 }
 
 static LogicalType StructureToTypeObject(ClientContext &context, const JSONStructureNode &node, const idx_t max_depth,
-                                         const double field_appearance_threshold, idx_t depth,
-                                         const idx_t sample_count) {
+                                         const double field_appearance_threshold, const idx_t map_inference_threshold,
+                                         const idx_t depth, const LogicalType &null_type) {
 	D_ASSERT(node.descriptions.size() == 1 && node.descriptions[0].type == LogicalTypeId::STRUCT);
 	auto &desc = node.descriptions[0];
 
-	// If it's an empty struct we do JSON instead
+	// If it's an empty struct we do MAP of JSON instead
 	if (desc.children.empty()) {
-		// Empty struct - let's do JSON instead
-		return LogicalType::JSON();
+		// Empty struct - let's do MAP of JSON instead
+		return LogicalType::MAP(LogicalType::VARCHAR, null_type);
 	}
 
-	// If it's an inconsistent object we also just do JSON
-	double total_child_counts = 0;
-	for (const auto &child : desc.children) {
-		total_child_counts += double(child.count) / sample_count;
-	}
-	const auto avg_occurrence = total_child_counts / desc.children.size();
-	if (avg_occurrence < field_appearance_threshold) {
-		return LogicalType::JSON();
+	// If it's an inconsistent object we also just do MAP with the best-possible, recursively-merged value type
+	if (IsStructureInconsistent(desc, node.count, node.null_count, field_appearance_threshold)) {
+		return LogicalType::MAP(LogicalType::VARCHAR,
+		                        GetMergedType(context, node, max_depth, field_appearance_threshold,
+		                                      map_inference_threshold, depth + 1, null_type));
 	}
 
+	// We have a consistent object
 	child_list_t<LogicalType> child_types;
 	child_types.reserve(desc.children.size());
 	for (auto &child : desc.children) {
 		D_ASSERT(child.key);
 		child_types.emplace_back(*child.key,
 		                         JSONStructure::StructureToType(context, child, max_depth, field_appearance_threshold,
-		                                                        depth + 1, sample_count));
+		                                                        map_inference_threshold, depth + 1, null_type));
 	}
+
+	// If we have many children and all children have similar-enough types we infer map
+	if (desc.children.size() >= map_inference_threshold) {
+		LogicalType map_value_type = GetMergedType(context, node, max_depth, field_appearance_threshold,
+		                                           map_inference_threshold, depth + 1, LogicalTypeId::SQLNULL);
+
+		double total_similarity = 0;
+		for (const auto &child_type : child_types) {
+			const auto similarity = CalculateTypeSimilarity(map_value_type, child_type.second, max_depth, depth + 1);
+			if (similarity < 0) {
+				total_similarity = similarity;
+				break;
+			}
+			total_similarity += similarity;
+		}
+		const auto avg_similarity = total_similarity / static_cast<double>(child_types.size());
+		if (avg_similarity >= 0.8) {
+			if (null_type != LogicalTypeId::SQLNULL) {
+				map_value_type = GetMergedType(context, node, max_depth, field_appearance_threshold,
+				                               map_inference_threshold, depth + 1, null_type);
+			}
+			return LogicalType::MAP(LogicalType::VARCHAR, map_value_type);
+		}
+	}
+
 	return LogicalType::STRUCT(child_types);
 }
 
@@ -555,30 +764,32 @@ static LogicalType StructureToTypeString(const JSONStructureNode &node) {
 }
 
 LogicalType JSONStructure::StructureToType(ClientContext &context, const JSONStructureNode &node, const idx_t max_depth,
-                                           const double field_appearance_threshold, idx_t depth, idx_t sample_count) {
+                                           const double field_appearance_threshold, const idx_t map_inference_threshold,
+                                           const idx_t depth, const LogicalType &null_type) {
 	if (depth >= max_depth) {
 		return LogicalType::JSON();
 	}
 	if (node.descriptions.empty()) {
-		return LogicalType::JSON();
+		return null_type;
 	}
 	if (node.descriptions.size() != 1) { // Inconsistent types, so we resort to JSON
 		return LogicalType::JSON();
 	}
-	sample_count = sample_count == DConstants::INVALID_INDEX ? node.count : sample_count;
 	auto &desc = node.descriptions[0];
 	D_ASSERT(desc.type != LogicalTypeId::INVALID);
 	switch (desc.type) {
 	case LogicalTypeId::LIST:
-		return StructureToTypeArray(context, node, max_depth, field_appearance_threshold, depth);
+		return StructureToTypeArray(context, node, max_depth, field_appearance_threshold, map_inference_threshold,
+		                            depth, null_type);
 	case LogicalTypeId::STRUCT:
-		return StructureToTypeObject(context, node, max_depth, field_appearance_threshold, depth, sample_count);
+		return StructureToTypeObject(context, node, max_depth, field_appearance_threshold, map_inference_threshold,
+		                             depth, null_type);
 	case LogicalTypeId::VARCHAR:
 		return StructureToTypeString(node);
-	case LogicalTypeId::SQLNULL:
-		return LogicalType::JSON();
 	case LogicalTypeId::UBIGINT:
 		return LogicalTypeId::BIGINT; // We prefer not to return UBIGINT in our type auto-detection
+	case LogicalTypeId::SQLNULL:
+		return null_type;
 	default:
 		return desc.type;
 	}

package/src/duckdb/extension/json/json_functions/json_transform.cpp

@@ -1012,7 +1012,7 @@ void JSONFunctions::RegisterJSONTransformCastFunctions(CastFunctionSet &casts) {
 			target_type = LogicalType::UNION({{"any", LogicalType::ANY}});
 			break;
 		case LogicalTypeId::ARRAY:
-			target_type = LogicalType::ARRAY(LogicalType::ANY);
+			target_type = LogicalType::ARRAY(LogicalType::ANY, optional_idx());
 			break;
 		case LogicalTypeId::VARCHAR:
 			// We skip this one here as it's handled in json_functions.cpp

package/src/duckdb/extension/json/json_functions/json_type.cpp

@@ -2,7 +2,7 @@
 
 namespace duckdb {
 
-static inline string_t GetType(yyjson_val *val, yyjson_alc *alc, Vector &result) {
+static inline string_t GetType(yyjson_val *val, yyjson_alc *, Vector &, ValidityMask &mask, idx_t idx) {
 	return JSONCommon::ValTypeToStringT(val);
 }
 
@@ -11,11 +11,11 @@ static void UnaryTypeFunction(DataChunk &args, ExpressionState &state, Vector &r
 }
 
 static void BinaryTypeFunction(DataChunk &args, ExpressionState &state, Vector &result) {
-	JSONExecutors::BinaryExecute<string_t, false>(args, state, result, GetType);
+	JSONExecutors::BinaryExecute<string_t>(args, state, result, GetType);
 }
 
 static void ManyTypeFunction(DataChunk &args, ExpressionState &state, Vector &result) {
-	JSONExecutors::ExecuteMany<string_t, false>(args, state, result, GetType);
+	JSONExecutors::ExecuteMany<string_t>(args, state, result, GetType);
 }
 
 static void GetTypeFunctionsInternal(ScalarFunctionSet &set, const LogicalType &input_type) {