@hestia-earth/data-validation 0.37.7

package/hestia_earth/validation/validators/product.py

@@ -0,0 +1,263 @@
+import traceback
+from hestia_earth.schema import TermTermType, CycleFunctionalUnit
+from hestia_earth.utils.lookup import extract_grouped_data
+from hestia_earth.utils.tools import list_sum, flatten, non_empty_list
+from hestia_earth.utils.model import filter_list_term_type
+
+from hestia_earth.validation.log import logger
+from hestia_earth.validation.utils import (
+    _list_sum,
+    _filter_list_errors,
+    get_lookup_value,
+)
+from hestia_earth.validation.distribution import is_enabled as distribution_is_enabled
+from .shared import CROP_SITE_TYPE
+
+
+def validate_economicValueShare(products: list):
+    sum = _list_sum(products, "economicValueShare")
+    return sum <= 100.5 or {
+        "level": "error",
+        "dataPath": ".products",
+        "message": "economicValueShare should sum to 100 or less across all products",
+        "params": {"sum": sum},
+    }
+
+
+def validate_value_empty(products: list):
+    def validate(values: tuple):
+        index, product = values
+        return len(product.get("value", [])) > 0 or {
+            "level": "warning",
+            "dataPath": f".products[{index}]",
+            "message": "may not be 0",
+        }
+
+    return _filter_list_errors(map(validate, enumerate(products)))
+
+
+def validate_value_0(products: list):
+    def validate(values: tuple):
+        index, product = values
+        value = list_sum(product.get("value", [-1]), -1)
+        eva = product.get("economicValueShare", 0)
+        revenue = product.get("revenue", 0)
+        return value != 0 or _filter_list_errors(
+            [
+                eva == 0
+                or {
+                    "level": "error",
+                    "dataPath": f".products[{index}].value",
+                    "message": "economicValueShare must be 0 for product value 0",
+                    "params": {"value": eva, "term": product.get("term")},
+                },
+                revenue == 0
+                or {
+                    "level": "error",
+                    "dataPath": f".products[{index}].value",
+                    "message": "revenue must be 0 for product value 0",
+                    "params": {"value": revenue, "term": product.get("term")},
+                },
+            ]
+        )
+
+    return _filter_list_errors(flatten(map(validate, enumerate(products))))
+
+
+MAX_PRIMARY_PRODUCTS = 1
+
+
+def validate_primary(products: list):
+    primary = list(filter(lambda p: p.get("primary", False), products))
+    return len(primary) <= MAX_PRIMARY_PRODUCTS or {
+        "level": "error",
+        "dataPath": ".products",
+        "message": "only 1 primary product allowed",
+    }
+
+
+def validate_product_ha_functional_unit_ha(cycle: dict, list_key: str = "products"):
+    functional_unit = cycle.get("functionalUnit", CycleFunctionalUnit.RELATIVE.value)
+
+    def validate(values: tuple):
+        index, product = values
+        term_units = product.get("term", {}).get("units")
+        value = list_sum(product.get("value", [0]))
+        return (
+            term_units != "ha"
+            or value <= 1
+            or {
+                "level": "error",
+                "dataPath": f".{list_key}[{index}].value",
+                "message": "must be below or equal to 1 for unit in ha",
+                "params": {"term": product.get("term", {})},
+            }
+        )
+
+    return functional_unit != CycleFunctionalUnit._1_HA.value or _filter_list_errors(
+        map(validate, enumerate(cycle.get(list_key, [])))
+    )
+
+
+def _validate_product_yield(country: dict, list_key: str, threshold: float):
+    from .distribution import (
+        YIELD_COLUMN,
+        get_stats_by_group_key,
+        validate as validate_distribution,
+    )
+
+    country_id = country.get("@id")
+
+    def validate(values: tuple):
+        index, product = values
+
+        product_id = product.get("term", {}).get("@id")
+        product_value = product.get("value", [])
+
+        def _get_mu_sd():
+            return get_stats_by_group_key(YIELD_COLUMN, country_id, product_id)
+
+        valid, outliers, min, max = validate_distribution(
+            product_value, threshold, get_mu_sd=_get_mu_sd
+        )
+        return valid or {
+            "level": "warning",
+            "dataPath": f".{list_key}[{index}].value",
+            "message": "is outside confidence interval",
+            "params": {
+                "term": product.get("term", {}),
+                "country": country,
+                "outliers": outliers,
+                "threshold": threshold,
+                "min": min,
+                "max": max,
+            },
+        }
+
+    return validate
+
+
+def validate_product_yield(
+    cycle: dict, site: dict, list_key: str = "products", threshold: float = 0.95
+):
+    country = site.get("country", {})
+    products = cycle.get(list_key, [])
+
+    try:
+        return (
+            site.get("siteType") not in CROP_SITE_TYPE
+            or (
+                _filter_list_errors(
+                    map(
+                        _validate_product_yield(country, list_key, threshold),
+                        enumerate(products),
+                    )
+                )
+            )
+            if distribution_is_enabled()
+            else True
+        )
+    except Exception:
+        stack = traceback.format_exc()
+        logger.error(f"Error validating using distribution: '{stack}'")
+        return True
+
+
+def _excreta_term_ids(term: dict, column: str, group_key: str):
+    value = get_lookup_value(term, column)
+    # handle using `|` to allow multiple values
+    return non_empty_list((extract_grouped_data(value, group_key) or "").split("|"))
+
+
+def _grouped_excreta_term_ids(term: dict, group_keys: list):
+    grouped_columns = {
+        "kg": "excretaKgMassTermIds",
+        "kg N": "excretaKgNTermIds",
+        "kg VS": "excretaKgVsTermIds",
+    }
+    grouped_values = {
+        units: flatten(
+            [_excreta_term_ids(term, column, group_key) for group_key in group_keys]
+        )
+        for units, column in grouped_columns.items()
+    }
+    # include `default` key if no values found
+    return {
+        units: grouped_values.get(units, [])
+        + (
+            _excreta_term_ids(term, column, "default")
+            if not grouped_values.get(units)
+            else []
+        )
+        for units, column in grouped_columns.items()
+    }
+
+
+def validate_excreta_product(cycle: dict, list_key: str = "products"):
+    animal_products = [
+        (index, product)
+        for index, product in enumerate(cycle.get(list_key, []))
+        if product.get("term", {}).get("termType")
+        in [
+            TermTermType.ANIMALPRODUCT.value,
+            TermTermType.LIVEANIMAL.value,
+            TermTermType.LIVEAQUATICSPECIES.value,
+        ]
+    ]
+    excreta_products = [
+        (index, product)
+        for index, product in enumerate(cycle.get(list_key, []))
+        if product.get("term", {}).get("termType") in [TermTermType.EXCRETA.value]
+    ]
+    systems = filter_list_term_type(cycle.get("practices", []), TermTermType.SYSTEM)
+    group_keys = [s.get("term", {}).get("@id") for s in systems]
+    allowed_excreta_per_product = {
+        product.get("term", {}).get("@id"): _grouped_excreta_term_ids(
+            product.get("term", {}), group_keys
+        )
+        for _i, product in animal_products
+    }
+
+    def validate_animal(values: tuple):
+        """
+        Validate the the animal product has an excreta term
+        """
+        index, blank_node = values
+        term = blank_node.get("term", {})
+        term_id = term.get("@id")
+        allowed_ids = flatten(allowed_excreta_per_product.get(term_id, {}).values())
+        return {
+            "level": "warning",
+            "dataPath": f".{list_key}[{index}]",
+            "message": "should add an excreta product",
+            "params": {"expected": allowed_ids},
+        }
+
+    def validate_excreta(values: tuple):
+        """
+        Validate that the excreta is allowed.
+        """
+        index, blank_node = values
+        term = blank_node.get("term", {})
+        term_id = term.get("@id")
+        allowed_ids = non_empty_list(
+            flatten(
+                [v.get(term.get("units")) for v in allowed_excreta_per_product.values()]
+            )
+        )
+        return term_id in allowed_ids or {
+            "level": "error",
+            "dataPath": f".{list_key}[{index}].term.@id",
+            "message": "is not an allowed excreta product",
+            "params": {"current": term_id, "expected": allowed_ids},
+        }
+
+    return (
+        (
+            _filter_list_errors(map(validate_excreta, excreta_products))
+            if excreta_products
+            else _filter_list_errors(map(validate_animal, animal_products))
+        )
+        if animal_products
+        else True
+    )

package/hestia_earth/validation/validators/property.py

@@ -0,0 +1,266 @@
+from hestia_earth.utils.api import download_hestia
+from hestia_earth.utils.model import find_term_match
+from hestia_earth.utils.tools import flatten, non_empty_list, safe_parse_float
+
+from hestia_earth.validation.utils import (
+    match_value_type,
+    term_valueType,
+    get_lookup_value,
+    update_error_path,
+    _filter_list_errors,
+    _flatten_errors,
+)
+from .shared import (
+    value_difference,
+    is_value_below,
+    validate_list_value_between_min_max,
+    validate_list_min_max_lookup,
+)
+
+
+PROPERTIES_KEY = "properties"
+
+
+def validate_property_valueType(node: dict, list_key: str):
+    def is_valid(values: tuple):
+        index, property = values
+        term = property.get("term", {})
+        expected_value_type = term_valueType(term)
+        value = property.get("value")
+        return (
+            value is None
+            or match_value_type(expected_value_type, value)
+            or {
+                "level": "error",
+                "dataPath": f".{PROPERTIES_KEY}[{index}].value",
+                "message": "the property value type is incorrect",
+                "params": {"expected": expected_value_type},
+            }
+        )
+
+    def validate(values: tuple):
+        index, blank_node = values
+        errors = list(map(is_valid, enumerate(blank_node.get(PROPERTIES_KEY, []))))
+        return _filter_list_errors(
+            [
+                update_error_path(error, list_key, index)
+                for error in errors
+                if error is not True
+            ]
+        )
+
+    return _filter_list_errors(
+        flatten(map(validate, enumerate(node.get(list_key, []))))
+    )
+
+
+def validate_term_type(node: dict, list_key: str):
+    def is_valid(blank_node: dict):
+        def check(values: tuple):
+            index, property = values
+            term = property.get("term", {})
+            term_types = get_lookup_value(term, "termTypesAllowed")
+            expected_term_types = (term_types or "all").split(";")
+            term_type = blank_node.get("term", {}).get("termType")
+            return any(
+                ["all" in expected_term_types, term_type in expected_term_types]
+            ) or {
+                "level": "error",
+                "dataPath": f".{PROPERTIES_KEY}[{index}].term.termType",
+                "message": "can not be used on this termType",
+                "params": {"current": term_type, "expected": expected_term_types},
+            }
+
+        return check
+
+    def validate(values: tuple):
+        index, blank_node = values
+        errors = list(
+            map(is_valid(blank_node), enumerate(blank_node.get(PROPERTIES_KEY, [])))
+        )
+        return _filter_list_errors(
+            [
+                update_error_path(error, list_key, index)
+                for error in errors
+                if error is not True
+            ]
+        )
+
+    return _filter_list_errors(
+        flatten(map(validate, enumerate(node.get(list_key, []))))
+    )
+
+
+def _property_default_value(term_id: str, property_term_id: str):
+    # load the term defaultProperties and find the matching property
+    term = download_hestia(term_id)
+    if not term:
+        raise Exception(f"Term not found: {term_id}")
+    return safe_parse_float(
+        find_term_match(term.get("defaultProperties", []), property_term_id).get(
+            "value"
+        )
+    )
+
+
+def _property_default_allowed_values(term: dict):
+    allowed = get_lookup_value(term, "validationAllowedExceptions")
+    try:
+        allowed_values = non_empty_list(allowed.split(";")) if allowed else []
+        return [safe_parse_float(v) for v in allowed_values]
+    # failure to split by `;` as single value allowed
+    except AttributeError:
+        return [safe_parse_float(allowed)]
+
+
+def validate_default_value(node: dict, list_key: str):
+    threshold = 0.25
+
+    def is_valid(term_id: str):
+        def validate(values: tuple):
+            index, prop = values
+            value = safe_parse_float(prop.get("value"))
+            prop_term_id = prop.get("term", {}).get("@id")
+            default_value = _property_default_value(term_id, prop_term_id)
+            delta = value_difference(value, default_value)
+            values_allowed = (
+                _property_default_allowed_values(prop.get("term", {}))
+                if prop_term_id
+                else []
+            )
+            return (
+                prop.get("value") is None
+                or delta < threshold
+                or value in values_allowed
+                or {
+                    "level": "warning",
+                    "dataPath": f".{PROPERTIES_KEY}[{index}].value",
+                    "message": "should be within percentage of default value",
+                    "params": {
+                        "current": value,
+                        "default": default_value,
+                        "percentage": delta * 100,
+                        "threshold": threshold,
+                    },
+                }
+            )
+
+        return validate
+
+    def validate(values: tuple):
+        index, blank_node = values
+        term_id = blank_node.get("term", {}).get("@id")
+        errors = list(
+            map(is_valid(term_id), enumerate(blank_node.get(PROPERTIES_KEY, [])))
+        )
+        return _filter_list_errors(
+            [
+                update_error_path(error, list_key, index)
+                for error in errors
+                if error is not True
+            ]
+        )
+
+    return _filter_list_errors(
+        flatten(map(validate, enumerate(node.get(list_key, []))))
+    )
+
+
+VSC_ID = "volatileSolidsContent"
+VSC_MIN = {"kg": 0, "kg Vs": 100, "kg N": 0}
+VSC_MAX = {"kg": 100, "kg Vs": 100, "kg N": None}
+
+
+def _volatileSolidsContent_error(min: float = None, max: float = None):
+    return (
+        f"must be {max}"
+        if min == max
+        else (
+            f"must be above {min}"
+            if max is None
+            else (
+                f"must be below {max}"
+                if min is None
+                else f"must be between {min} and {max}"
+            )
+        )
+    )
+
+
+def validate_volatileSolidsContent(node: dict, list_key: str):
+    def is_valid(blank_node: dict):
+        units = blank_node.get("term", {}).get("units")
+
+        def validate(values: tuple):
+            index, property = values
+            term_id = property.get("term", {}).get("@id")
+            value = property.get("value", 0)
+            min = VSC_MIN.get(units)
+            max = VSC_MAX.get(units)
+            return (
+                term_id != VSC_ID
+                or all([is_value_below(value, max), is_value_below(min, value)])
+                or {
+                    "level": "error",
+                    "dataPath": f".{PROPERTIES_KEY}[{index}].value",
+                    "message": _volatileSolidsContent_error(min, max),
+                }
+            )
+
+        return validate
+
+    def validate(values: tuple):
+        index, blank_node = values
+        errors = flatten(
+            map(is_valid(blank_node), enumerate(blank_node.get(PROPERTIES_KEY, [])))
+        )
+        return _filter_list_errors(
+            [
+                update_error_path(error, list_key, index)
+                for error in errors
+                if error is not True
+            ]
+        )
+
+    return _filter_list_errors(
+        flatten(map(validate, enumerate(node.get(list_key, []))))
+    )
+
+
+def validate_value_min_max(node: dict, list_key: str):
+    def validate(values: tuple):
+        index, blank_node = values
+        # handle skip maximum validation for some properties
+        skip_maximum_properties = (
+            get_lookup_value(blank_node.get("term"), "skipValidateTermIdsMaximum") or ""
+        ).split(";")
+        errors = _flatten_errors(
+            [
+                validate_list_value_between_min_max(blank_node, PROPERTIES_KEY),
+                validate_list_min_max_lookup(
+                    blank_node, PROPERTIES_KEY, skip_max_ids=skip_maximum_properties
+                ),
+            ]
+        )
+        return _filter_list_errors(
+            [
+                update_error_path(error, list_key, index)
+                for error in errors
+                if error is not True
+            ]
+        )
+
+    return _filter_list_errors(
+        flatten(map(validate, enumerate(node.get(list_key, []))))
+    )
+
+
+def validate_all(node: dict, list_key: str):
+    return _filter_list_errors(
+        [
+            validate_default_value(node, list_key),
+            validate_term_type(node, list_key),
+            validate_property_valueType(node, list_key),
+            validate_value_min_max(node, list_key),
+        ]
+    )