@hestia-earth/data-validation 0.37.7

package/hestia_earth/validation/validators/distribution.py

@@ -0,0 +1,86 @@
+import math
+from hestia_earth.distribution.utils.cycle import (
+    FERTILISER_COLUMNS,
+    PESTICIDE_COLUMN,
+    IRRIGATION_COLUMN,
+    YIELD_COLUMN,
+)
+from hestia_earth.distribution.posterior_fert import get_post as get_post_fert
+from hestia_earth.distribution.prior_fert import get_prior as get_prior_fert
+from hestia_earth.distribution.posterior_pest import get_post as get_post_pest
+from hestia_earth.distribution.prior_pest import get_prior as get_prior_pest
+from hestia_earth.distribution.posterior_irrigation import get_post as get_post_irri
+from hestia_earth.distribution.prior_irrigation import get_prior as get_prior_irri
+from hestia_earth.distribution.posterior_yield import get_post as get_post_yield
+from hestia_earth.distribution.prior_yield import get_prior as get_prior_yield
+
+UNIVARIATE_DEFAULT_THRESHOLD = 0.95
+UNIVARIATE_DEFAULT_ZSCORE = 1.96
+UNIVARIATE_CI_TO_ZSCORE = {
+    0.9: 1.65,
+    UNIVARIATE_DEFAULT_THRESHOLD: UNIVARIATE_DEFAULT_ZSCORE,
+    0.99: 2.58,
+}
+
+
+def _process_fertiliser(country_id: str, product_id: str, input_id: str):
+    mu, sd = get_post_fert(country_id, product_id, input_id)
+    return (mu, sd) if mu is not None else get_prior_fert(country_id, input_id)
+
+
+def _process_pesticide(country_id: str, product_id: str, *args):
+    mu, sd = get_post_pest(country_id, product_id)
+    return (mu, sd) if mu is not None else get_prior_pest(country_id)
+
+
+def _process_irrigation(country_id: str, product_id: str, *args):
+    mu, sd = get_post_irri(country_id, product_id)
+    return (mu, sd) if mu is not None else get_prior_irri(country_id)
+
+
+def _process_yield(country_id: str, product_id: str, *args):
+    mu, sd = get_post_yield(country_id, product_id)
+    return (mu, sd) if mu is not None else get_prior_yield(country_id, product_id)
+
+
+_PROCESS_BY_KEY = {
+    PESTICIDE_COLUMN: _process_pesticide,
+    IRRIGATION_COLUMN: _process_irrigation,
+    YIELD_COLUMN: _process_yield,
+} | {key: _process_fertiliser for key in FERTILISER_COLUMNS}
+
+
+def get_stats_by_group_key(
+    key: str, country_id: str, product_id: str, input_id: str = None
+):
+    return _PROCESS_BY_KEY[key](country_id, product_id, input_id)
+
+
+def cycle_completeness_key(key: str):
+    return (
+        "fertiliser"
+        if key in FERTILISER_COLUMNS
+        else (
+            "pesticideVeterinaryDrug"
+            if key == PESTICIDE_COLUMN
+            else "water" if key == IRRIGATION_COLUMN else ""
+        )
+    )
+
+
+def validate(values: list, threshold: float, get_mu_sd):
+    def exec():
+        z = UNIVARIATE_CI_TO_ZSCORE[threshold]
+        mu, sd = get_mu_sd()
+        _min = mu - (z * sd) if mu is not None else None
+        _max = mu + (z * sd) if mu is not None else None
+        passes = [
+            _min <= y <= _max if all([mu is not None, not math.isnan(y)]) else True
+            for y in values
+        ]
+        outliers = (
+            [y for y in values if not _min <= y <= _max] if mu is not None else []
+        )
+        return all(passes), outliers, max(_min or 0, 0), _max
+
+    return exec()

package/hestia_earth/validation/validators/emission.py

@@ -0,0 +1,109 @@
+from hestia_earth.schema import EmissionMethodTier
+from hestia_earth.utils.tools import flatten
+from hestia_earth.utils.model import find_term_match
+from hestia_earth.utils.blank_node import get_lookup_value
+
+from hestia_earth.validation.utils import _filter_list_errors
+
+
+def is_inputs_production(emission: dict):
+    return get_lookup_value(emission, "inputProductionGroupId") == emission.get(
+        "term", {}
+    ).get("@id")
+
+
+def validate_linked_terms(
+    cycle: dict, list_key: str, linked_key: str, linked_list_key: str, soft_check=False
+):
+    linked_nodes = cycle.get(linked_list_key, [])
+
+    def validate(values: tuple):
+        index, emission = values
+        linked_items = emission.get(linked_key, [])
+        return (
+            len(linked_items) == 0
+            or any(
+                [
+                    find_term_match(linked_nodes, item.get("@id"))
+                    for item in (
+                        [linked_items]
+                        if isinstance(linked_items, dict)
+                        else linked_items
+                    )
+                ]
+            )
+            or {
+                "level": "warning" if soft_check else "error",
+                "dataPath": f".{list_key}[{index}]",
+                "message": f"{'should' if soft_check else 'must'} add the linked {linked_list_key} to the cycle",
+                "params": {"term": emission.get("term", {}), "expected": linked_items},
+            }
+        )
+
+    return _filter_list_errors(
+        flatten(map(validate, enumerate(cycle.get(list_key, []))))
+    )
+
+
+_NOT_RELEVANT_ID = "emissionNotRelevant"
+
+
+def validate_method_not_relevant(cycle: dict, list_key: str):
+    def validate(values: tuple):
+        index, emission = values
+        term_id = emission.get("methodModel", {}).get("@id")
+        return term_id != _NOT_RELEVANT_ID or {
+            "level": "warning",
+            "dataPath": f".{list_key}[{index}].methodModel.@id",
+            "message": "should not use not relevant model",
+            "params": {
+                "term": emission.get("term", {}),
+                "model": emission.get("methodModel", {}),
+            },
+        }
+
+    return _filter_list_errors(
+        flatten(map(validate, enumerate(cycle.get(list_key, []))))
+    )
+
+
+def validate_methodTier_not_relevant(cycle: dict, list_key: str):
+    def validate(values: tuple):
+        index, emission = values
+        methodTier = emission.get("methodTier")
+        return methodTier != EmissionMethodTier.NOT_RELEVANT.value or {
+            "level": "warning",
+            "dataPath": f".{list_key}[{index}].methodTier",
+            "message": "should not use not relevant methodTier",
+            "params": {"term": emission.get("term", {})},
+        }
+
+    return _filter_list_errors(
+        flatten(map(validate, enumerate(cycle.get(list_key, []))))
+    )
+
+
+def validate_methodTier_background(node: dict, list_key: str):
+    allowed_values = [
+        e.value
+        for e in EmissionMethodTier
+        if e not in [EmissionMethodTier.BACKGROUND, EmissionMethodTier.NOT_RELEVANT]
+    ]
+
+    def validate(values: tuple):
+        index, emission = values
+        methodTier = emission.get("methodTier")
+        return (
+            methodTier != EmissionMethodTier.BACKGROUND.value
+            or is_inputs_production(emission)
+            or {
+                "level": "error",
+                "dataPath": f".{list_key}[{index}].methodTier",
+                "message": "should be equal to one of the allowed values",
+                "params": {"allowedValues": allowed_values},
+            }
+        )
+
+    return _filter_list_errors(
+        flatten(map(validate, enumerate(node.get(list_key, []))))
+    )

package/hestia_earth/validation/validators/impact_assessment.py

@@ -0,0 +1,138 @@
+"""
+Impact Assessment validation
+
+Here is the list of validations running on a [Impact Assessment](/schema/ImpactAssessment).
+"""
+
+from hestia_earth.utils.tools import flatten
+
+from hestia_earth.validation.utils import (
+    find_linked_node,
+    find_by_product,
+    find_by_unique_product,
+)
+from .shared import (
+    validate_list_min_below_max,
+    validate_country_region,
+    validate_list_country_region,
+    validate_list_term_percent,
+    validate_linked_source_privacy,
+    validate_date_lt_today,
+    validate_list_model,
+    validate_private_has_source,
+    validate_list_value_between_min_max,
+    is_date_equal,
+    validate_other_model,
+    validate_nested_existing_node,
+)
+from .emission import validate_methodTier_background
+from .indicator import (
+    validate_characterisedIndicator_model,
+    validate_landTransformation,
+    validate_inonising_compounds_waste,
+)
+
+
+def validate_empty(impact_assessment: dict):
+    return any(
+        [
+            impact_assessment.get("cycle"),
+            impact_assessment.get("emissionsResourceUse", []),
+            len(impact_assessment.get("impacts", [])) > 0,
+            len(impact_assessment.get("endpoints", [])) > 0,
+        ]
+    ) or {
+        "level": "error",
+        "dataPath": "",
+        "message": "should not be empty",
+        "params": {"type": "ImpactAssessment"},
+    }
+
+
+def validate_linked_cycle_product(impact_assessment: dict, cycle: dict):
+    product = impact_assessment.get("product", {})
+    cycle_product = find_by_product(cycle, product) or find_by_unique_product(
+        cycle, product
+    )
+    return cycle_product is not None or {
+        "level": "error",
+        "dataPath": ".product",
+        "message": "should be included in the cycle products",
+        "params": {
+            "product": product.get("term", {}),
+            "node": {"type": "Cycle", "id": cycle.get("id", cycle.get("@id"))},
+        },
+    }
+
+
+def validate_linked_cycle_endDate(impact_assessment: dict, cycle: dict):
+    key = "endDate"
+    return is_date_equal(impact_assessment.get(key), cycle.get(key), True) or {
+        "level": "error",
+        "dataPath": f".{key}",
+        "message": "must be equal to the Cycle endDate",
+    }
+
+
+def validate_impact_assessment(impact_assessment: dict, node_map: dict = {}):
+    cycle = find_linked_node(node_map, impact_assessment.get("cycle", {}))
+    return [
+        validate_empty(impact_assessment),
+        validate_date_lt_today(impact_assessment, "startDate"),
+        validate_date_lt_today(impact_assessment, "endDate"),
+        validate_linked_source_privacy(impact_assessment, "source", node_map),
+        validate_private_has_source(impact_assessment, "source"),
+        validate_country_region(impact_assessment),
+        validate_linked_cycle_product(impact_assessment, cycle) if cycle else True,
+        validate_linked_cycle_endDate(impact_assessment, cycle) if cycle else True,
+        validate_nested_existing_node(impact_assessment, "cycle"),
+        validate_nested_existing_node(impact_assessment, "site"),
+    ] + flatten(
+        (
+            [
+                validate_list_country_region(impact_assessment, "emissionsResourceUse"),
+                validate_list_min_below_max(impact_assessment, "emissionsResourceUse"),
+                validate_list_value_between_min_max(
+                    impact_assessment, "emissionsResourceUse"
+                ),
+                validate_list_term_percent(impact_assessment, "emissionsResourceUse"),
+                validate_characterisedIndicator_model(
+                    impact_assessment, "emissionsResourceUse"
+                ),
+                validate_landTransformation(impact_assessment, "emissionsResourceUse"),
+                validate_methodTier_background(
+                    impact_assessment, "emissionsResourceUse"
+                ),
+                validate_other_model(impact_assessment, "emissionsResourceUse"),
+                validate_inonising_compounds_waste(
+                    impact_assessment, "emissionsResourceUse"
+                ),
+            ]
+            if len(impact_assessment.get("emissionsResourceUse", [])) > 0
+            else []
+        )
+        + (
+            [
+                validate_list_country_region(impact_assessment, "emissionsResourceUse"),
+                validate_list_min_below_max(impact_assessment, "impacts"),
+                validate_list_value_between_min_max(impact_assessment, "impacts"),
+                validate_list_term_percent(impact_assessment, "impacts"),
+                validate_list_model(impact_assessment, "impacts"),
+                validate_characterisedIndicator_model(impact_assessment, "impacts"),
+                validate_other_model(impact_assessment, "emissionsResourceUse"),
+            ]
+            if len(impact_assessment.get("impacts", [])) > 0
+            else []
+        )
+        + (
+            [
+                validate_list_country_region(impact_assessment, "emissionsResourceUse"),
+                validate_list_min_below_max(impact_assessment, "endpoints"),
+                validate_list_value_between_min_max(impact_assessment, "endpoints"),
+                validate_list_term_percent(impact_assessment, "endpoints"),
+                validate_other_model(impact_assessment, "emissionsResourceUse"),
+            ]
+            if len(impact_assessment.get("endpoints", [])) > 0
+            else []
+        )
+    )

package/hestia_earth/validation/validators/indicator.py

@@ -0,0 +1,154 @@
+from functools import reduce
+from hestia_earth.schema import TermTermType
+from hestia_earth.utils.lookup import download_lookup, get_table_value, lookup_term_ids
+from hestia_earth.utils.tools import flatten, non_empty_list
+
+from hestia_earth.validation.utils import _filter_list_errors
+from hestia_earth.validation.terms import TERMS_QUERY, get_terms
+
+IGNORE_MODELS = [
+    "aggregatedModels"  # used only for aggregations, should not be shown as possible values
+]
+
+
+def _indicator_group_key(indicator: dict, include_term: bool = True):
+    return "-".join(
+        non_empty_list(
+            flatten(
+                [
+                    indicator.get("term", {}).get("@id") if include_term else "",
+                    indicator.get("country", {}).get("@id"),
+                    indicator.get("landCover", {}).get("@id"),
+                    [i.get("@id") for i in indicator.get("inputs", [])],
+                ]
+            )
+        )
+    )
+
+
+def _group_indicators(indicators: list, include_term: bool = True):
+    def group_by(groups: dict, indicator: dict):
+        group_key = _indicator_group_key(indicator, include_term)
+        groups[group_key] = groups.get(group_key, []) + [indicator]
+        return groups
+
+    return reduce(group_by, indicators, {})
+
+
+def _allowed_characterisedIndicator_model(lookup, models: list, term_id: str):
+    return [
+        m
+        for m in models
+        if m != "term.id"
+        and get_table_value(lookup, "term.id", term_id, m)
+        and m not in IGNORE_MODELS
+    ]
+
+
+def _is_method_allowed(lookup, term_id: str, model: str):
+    value = get_table_value(lookup, "term.id", term_id, model)
+    # bug numpy bool not returning `True`
+    return True if value else False
+
+
+def validate_characterisedIndicator_model(node: dict, list_key: str):
+    lookup = download_lookup(
+        "characterisedIndicator-model-mapping.csv", keep_in_memory=False
+    )
+    models = get_terms(TERMS_QUERY.MODEL)
+
+    def validate(values: tuple):
+        index, value = values
+        term = value.get("term", {})
+        term_id = term.get("@id")
+        model = value.get("methodModel", {})
+        model_id = model.get("@id")
+        should_validate = term_id in lookup_term_ids(lookup) and model_id is not None
+        return (
+            not should_validate
+            or _is_method_allowed(lookup, term_id, model_id)
+            or {
+                "level": "error",
+                "dataPath": f".{list_key}[{index}].methodModel.@id",
+                "message": "is not allowed for this characterisedIndicator",
+                "params": {
+                    "term": term,
+                    "model": model,
+                    "allowedValues": _allowed_characterisedIndicator_model(
+                        lookup, models, term_id
+                    ),
+                },
+            }
+        )
+
+    return _filter_list_errors(list(map(validate, enumerate(node.get(list_key, [])))))
+
+
+def _below_occupation(index: int, value: float, blank_nodes: list, suffix: str):
+    other_values = list(
+        filter(lambda b: b.get("term", {}).get("@id").endswith(suffix), blank_nodes)
+    )
+    other_values = [other_value.get("value", 0) for other_value in other_values]
+    return (
+        len(other_values) == 0
+        or all([value <= (other_value * 1.05) for other_value in other_values])
+        or {
+            "level": "error",
+            "dataPath": f".emissionsResourceUse[{index}].value",
+            "message": "must be less than or equal to land occupation",
+            "params": {"current": value, "max": min(other_values)},
+        }
+    )
+
+
+def validate_landTransformation(node: dict, list_key="emissionsResourceUse"):
+    blank_nodes = node.get(list_key, [])
+    grouped_land_occupation = _group_indicators(
+        [
+            v
+            for v in blank_nodes
+            if v.get("term", {}).get("@id", "").startswith("landOccupation")
+        ],
+        include_term=False,
+    )
+
+    def validate(values: tuple):
+        index, blank_node = values
+        term_id = blank_node.get("term", {}).get("@id")
+        value = blank_node.get("value", 0)
+        group_key = _indicator_group_key(blank_node, include_term=False)
+        land_occupation_blank_nodes = grouped_land_occupation.get(group_key) or []
+        return not term_id.startswith("landTransformation") or [
+            not term_id.endswith("DuringCycle")
+            or _below_occupation(
+                index, value, land_occupation_blank_nodes, "DuringCycle"
+            ),
+            not term_id.endswith("InputsProduction")
+            or _below_occupation(
+                index, value, land_occupation_blank_nodes, "InputsProduction"
+            ),
+        ]
+
+    return _filter_list_errors(flatten(map(validate, enumerate(blank_nodes))))
+
+
+def validate_inonising_compounds_waste(node: dict, list_key="emissionsResourceUse"):
+    blank_nodes = node.get(list_key, [])
+    allowed_term_types = [None, TermTermType.WASTE.value]
+
+    def validate(values: tuple):
+        index, blank_node = values
+        term_id = blank_node.get("term", {}).get("@id")
+        key_term_type = blank_node.get("key", {}).get("termType")
+        return (
+            not term_id.startswith("ionisingCompounds")
+            or key_term_type in allowed_term_types
+            or {
+                "level": "error",
+                "dataPath": f".{list_key}[{index}].key.termType",
+                "message": "should be equal to one of the allowed values",
+                "params": {"allowedValues": non_empty_list(allowed_term_types)},
+            }
+        )
+
+    return _filter_list_errors(flatten(map(validate, enumerate(blank_nodes))))

package/hestia_earth/validation/validators/infrastructure.py

@@ -0,0 +1,25 @@
+from hestia_earth.utils.date import diff_in_years
+
+from hestia_earth.validation.utils import _filter_list_errors
+
+
+def validate_lifespan(infrastructure: list):
+    def validate(values: tuple):
+        index, value = values
+        start_date = value.get("startDate")
+        end_date = value.get("endDate")
+        lifespan = value.get("defaultLifespan", -1)
+        diff = diff_in_years(start_date, end_date) if start_date and end_date else -1
+        return (
+            lifespan == -1
+            or diff == -1
+            or diff == round(lifespan, 1)
+            or {
+                "level": "error",
+                "dataPath": f".infrastructure[{index}].defaultLifespan",
+                "message": "must equal to endDate - startDate in decimal years",
+                "params": {"expected": diff},
+            }
+        )
+
+    return _filter_list_errors(map(validate, enumerate(infrastructure)))