@hestia-earth/data-validation 0.37.7

package/hestia_earth/validation/validators/input.py

@@ -0,0 +1,268 @@
+import traceback
+from hestia_earth.schema import (
+    NodeType,
+    TermTermType,
+    SiteSiteType,
+    CycleFunctionalUnit,
+)
+from hestia_earth.utils.model import find_term_match, find_primary_product
+from hestia_earth.utils.tools import non_empty_list, flatten
+
+from hestia_earth.validation.log import logger
+from hestia_earth.validation.utils import (
+    _filter_list_errors,
+    update_error_path,
+    is_live_animal_cycle,
+    get_lookup_value,
+)
+from hestia_earth.validation.distribution import is_enabled as distribution_is_enabled
+from .shared import validate_country, CROP_SITE_TYPE
+
+MUST_INCLUDE_ID_TERM_TYPES = [TermTermType.INORGANICFERTILISER.value]
+FEED_TERM_TYPES = [
+    TermTermType.ANIMALPRODUCT.value,
+    TermTermType.CROP.value,
+    TermTermType.FEEDFOODADDITIVE.value,
+    TermTermType.FORAGE.value,
+    TermTermType.LIVEAQUATICSPECIES.value,
+    TermTermType.PROCESSEDFOOD.value,
+    TermTermType.WASTE.value,
+]
+FEED_SITE_TYPE = [
+    SiteSiteType.CROPLAND.value,
+    SiteSiteType.PERMANENT_PASTURE.value,
+    SiteSiteType.ANIMAL_HOUSING.value,
+]
+
+
+def validate_must_include_id(inputs: list):
+    def missingRequiredIds(term: dict):
+        other_term_ids = (get_lookup_value(term, "complementaryTermIds") or "").split(
+            ";"
+        )
+        return non_empty_list(
+            [
+                term_id
+                for term_id in other_term_ids
+                if find_term_match(inputs, term_id, None) is None
+            ]
+        )
+
+    def validate(values: tuple):
+        index, input = values
+        term = input.get("term", {})
+        should_validate = term.get("termType") in MUST_INCLUDE_ID_TERM_TYPES
+        missing_ids = missingRequiredIds(term) if should_validate else []
+        # added gap-filling which makes it non-required anymore
+        return len(missing_ids) == 0 or {
+            "level": "warning",
+            "dataPath": f".inputs[{index}]",
+            "message": "should add missing inputs",
+            "params": {"expected": missing_ids},
+        }
+
+    return _filter_list_errors(map(validate, enumerate(inputs)))
+
+
+def validate_input_country(node: dict, list_key: list = "inputs"):
+    def validate(values: tuple):
+        index, input = values
+        country = input.get("country")
+        error = country is None or validate_country(input)
+        return error is True or update_error_path(error, list_key, index)
+
+    return _filter_list_errors(map(validate, enumerate(node.get(list_key, []))))
+
+
+def validate_related_impacts(node: dict, list_key: list, node_map: dict = {}):
+    related_impacts = node_map.get(NodeType.IMPACTASSESSMENT.value)
+
+    def validate(values: tuple):
+        index, input = values
+        impact_id = input.get("impactAssessment", {}).get("id")
+        impact = related_impacts.get(impact_id) if impact_id else None
+        related_node_id = (
+            impact.get(node.get("type").lower(), {}).get("id") if impact else None
+        )
+        return (
+            related_node_id is None
+            or related_node_id != node.get("id")
+            or {
+                "level": "error",
+                "dataPath": f".inputs[{index}].impactAssessment",
+                "message": "can not be linked to the same type",
+                "params": {"current": node.get("type")},
+            }
+        )
+
+    return (
+        _filter_list_errors(map(validate, enumerate(node.get(list_key, []))))
+        if related_impacts
+        else True
+    )
+
+
+def _validate_input_value(cycle: dict, country: dict, list_key: str, threshold: float):
+    from hestia_earth.distribution.utils.cycle import (
+        group_cycle_inputs,
+        get_input_group,
+    )
+    from .distribution import (
+        get_stats_by_group_key,
+        cycle_completeness_key,
+        validate as validate_distribution,
+    )
+
+    groups = group_cycle_inputs(cycle)
+    completeness = cycle.get("completeness", {})
+    country_id = country.get("@id")
+    product = find_primary_product(cycle) or {}
+    product_id = product.get("term", {}).get("@id")
+
+    def validate(values: tuple):
+        index, input = values
+
+        input_id = input.get("term", {}).get("@id")
+        group_key = get_input_group(input)
+        input_value = groups.get(group_key)
+
+        def _get_mu_sd():
+            return get_stats_by_group_key(group_key, country_id, product_id, input_id)
+
+        complete = completeness.get(cycle_completeness_key(group_key), False)
+        valid, outliers, min, max = (
+            validate_distribution([input_value], threshold, get_mu_sd=_get_mu_sd)
+            if complete
+            else (True, None, None, None)
+        )
+
+        return valid or {
+            "level": "warning",
+            "dataPath": f".{list_key}[{index}].value",
+            "message": "is outside confidence interval",
+            "params": {
+                "term": input.get("term", {}),
+                "group": group_key,
+                "country": country,
+                "outliers": outliers,
+                "threshold": threshold,
+                "min": min,
+                "max": max,
+            },
+        }
+
+    return validate
+
+
+def validate_input_distribution_value(
+    cycle: dict, site: dict, list_key: str = "inputs", threshold: float = 0.95
+):
+    try:
+        country = site.get("country", {})
+        inputs = cycle.get(list_key, [])
+        validate_input = (
+            _validate_input_value(cycle, country, list_key, threshold)
+            if distribution_is_enabled()
+            else None
+        )
+        return (
+            (
+                site.get("siteType") not in CROP_SITE_TYPE
+                or _filter_list_errors(map(validate_input, enumerate(inputs)))
+            )
+            if distribution_is_enabled()
+            else True
+        )
+    except Exception:
+        stack = traceback.format_exc()
+        logger.error(f"Error validating using distribution: '{stack}'")
+        return True
+
+
+def validate_animalFeed_requires_isAnimalFeed(cycle: dict, site: dict):
+    site_type = site.get("siteType")
+    is_liveAnimal = is_live_animal_cycle(cycle)
+
+    def validate(values: tuple):
+        index, input = values
+        term_type = input.get("term", {}).get("termType")
+        return (
+            term_type not in FEED_TERM_TYPES
+            or input.get("isAnimalFeed") is not None
+            or {
+                "level": "error",
+                "dataPath": f".inputs[{index}]",
+                "message": "must specify is it an animal feed",
+            }
+        )
+
+    def validate_animal(values: tuple):
+        index, blank_node = values
+        errors = list(map(validate, enumerate(blank_node.get("inputs", []))))
+        return _filter_list_errors(
+            [
+                update_error_path(error, "animals", index)
+                for error in errors
+                if error is not True
+            ]
+        )
+
+    return (
+        site_type not in FEED_SITE_TYPE
+        or not is_liveAnimal
+        or _filter_list_errors(
+            flatten(
+                list(map(validate, enumerate(cycle.get("inputs", []))))
+                + [list(map(validate_animal, enumerate(cycle.get("animals", []))))]
+            )
+        )
+    )
+
+
+def validate_saplings(cycle: dict, list_key: str = "inputs"):
+    functional_unit = cycle.get("functionalUnit")
+    product = (find_primary_product(cycle) or {}).get("term", {})
+    product_term_type = product.get("termType")
+    product_is_plantation = not not get_lookup_value(product, "isPlantation")
+
+    term_id = "saplings"
+    inputs = cycle.get(list_key, [])
+    saplings_indexes = [
+        i for i in range(len(inputs)) if inputs[i].get("term", {}).get("@id") == term_id
+    ]
+
+    return (
+        functional_unit != CycleFunctionalUnit._1_HA.value
+        or product_term_type != TermTermType.CROP.value
+        or not product_is_plantation
+        or len(saplings_indexes) == 0
+        or _filter_list_errors(
+            [
+                {
+                    "level": "error",
+                    "dataPath": f".{list_key}[{index}].term",
+                    "message": "saplings cannot be used as an input here",
+                    "params": {
+                        "current": term_id,
+                        "expected": "saplingsDepreciatedAmountPerCycle",
+                    },
+                }
+            ]
+            for index in saplings_indexes
+        )
+    )
+
+
+def validate_input_is_product(node: dict, list_key: str = "inputs"):
+    def validate(values: tuple):
+        index, blank_node = values
+        term = blank_node.get("term", {})
+        must_be_product = get_lookup_value(term, "mustBeProduct")
+        return not must_be_product or {
+            "level": "error",
+            "dataPath": f".{list_key}[{index}].term",
+            "message": "must be a product",
+            "params": {"term": term},
+        }
+
+    return _filter_list_errors(map(validate, enumerate(node.get(list_key, []))))

package/hestia_earth/validation/validators/management.py

@@ -0,0 +1,131 @@
+from hestia_earth.schema import TermTermType, SiteSiteType
+from hestia_earth.utils.model import filter_list_term_type
+from hestia_earth.utils.tools import flatten, safe_parse_date
+from hestia_earth.utils.date import diff_in_days
+
+from hestia_earth.validation.utils import _filter_list_errors, cycle_start_date
+
+_SITE_TYPE_TO_TERM_TYPES = {
+    SiteSiteType.CROPLAND.value: [
+        TermTermType.CROPRESIDUEMANAGEMENT,
+        TermTermType.LANDUSEMANAGEMENT,
+        TermTermType.TILLAGE,
+        TermTermType.WATERREGIME,
+    ],
+    SiteSiteType.PERMANENT_PASTURE.value: [
+        TermTermType.LANDUSEMANAGEMENT,
+        TermTermType.WATERREGIME,
+    ],
+}
+
+
+def validate_has_termType(site: dict, term_type: TermTermType):
+    blank_nodes = filter_list_term_type(site.get("management", []), term_type)
+    return len(blank_nodes) > 0 or {
+        "level": "warning",
+        "dataPath": ".management",
+        "message": "should contain at least one management node",
+        "params": {"termType": term_type.value},
+    }
+
+
+def validate_has_termTypes(site: dict):
+    blank_nodes = site.get("management", [])
+    term_types = _SITE_TYPE_TO_TERM_TYPES.get(site.get("siteType"), [])
+    return (
+        len(term_types) == 0
+        or len(blank_nodes) == 0
+        or _filter_list_errors(
+            [validate_has_termType(site, term_type) for term_type in term_types]
+        )
+    )
+
+
+def validate_exists(site: dict):
+    blank_nodes = site.get("management", [])
+    term_types = _SITE_TYPE_TO_TERM_TYPES.get(site.get("siteType"), [])
+    return (
+        len(term_types) == 0
+        or len(blank_nodes) > 0
+        or {
+            "level": "warning",
+            "dataPath": ".management",
+            "message": "should contain at least one management node",
+        }
+    )
+
+
+_FALLOW_DATE_VALID_FUNC = {
+    "less-than-1-year": lambda days: days <= 365,
+    "between-1-and-5-years": lambda days: 365 < days <= 365.25 * 5,
+    "less-than-20-years": lambda days: days <= 365.25 * 20,
+    "": lambda *args: True,
+}
+
+
+_FALLOW_DATE_VALID_TERM_TO_KEY = {
+    "shortFallow": "less-than-1-year",
+    "shortBareFallow": "less-than-1-year",
+    "longFallow": "between-1-and-5-years",
+    "longBareFallow": "between-1-and-5-years",
+    "setAside": "less-than-20-years",
+}
+
+
+def validate_fallow_dates(data: dict, list_key: str = "management"):
+    def validate(values: tuple):
+        index, blank_node = values
+        term = blank_node.get("term", {})
+        term_id = term.get("@id")
+        start_date = blank_node.get("startDate")
+        end_date = blank_node.get("endDate")
+        days = (
+            diff_in_days(start_date, end_date) if all([start_date, end_date]) else None
+        )
+        validation_key = _FALLOW_DATE_VALID_TERM_TO_KEY.get(term_id, "")
+        return (
+            days is None
+            or _FALLOW_DATE_VALID_FUNC.get(validation_key)(days)
+            or {
+                "level": "error",
+                "dataPath": f".{list_key}[{index}]",
+                "message": "duration must be in specified interval",
+                "params": {"term": term, "current": days, "expected": validation_key},
+            }
+        )
+
+    return _filter_list_errors(
+        flatten(map(validate, enumerate(data.get(list_key, []))))
+    )
+
+
+def validate_cycles_overlap(data: dict, cycles: list, list_key: str = "management"):
+    """
+    Prevent overlap between management dates and cycle dates.
+    """
+    first_cycle = sorted(cycles, key=lambda v: v.get("endDate"))[0] if cycles else None
+    earliest_start_date = cycle_start_date(first_cycle) if first_cycle else None
+    earliest_start_date_str = (
+        earliest_start_date.strftime("%Y-%m-%d") if earliest_start_date else None
+    )
+
+    def validate(values: tuple):
+        index, blank_node = values
+        end_date = blank_node.get("endDate")
+        return (
+            not end_date
+            or safe_parse_date(end_date) < earliest_start_date
+            or {
+                "level": "error",
+                "dataPath": f".{list_key}[{index}].endDate",
+                "message": "management date must be before cycle start date",
+                "params": {
+                    "current": end_date,
+                    "expected": earliest_start_date_str,
+                },
+            }
+        )
+
+    return not earliest_start_date or _filter_list_errors(
+        flatten(map(validate, enumerate(data.get(list_key, []))))
+    )