@hestia-earth/data-validation 0.37.7 → 0.37.9

package/hestia_earth/validation/validators/distribution.py

@@ -1,86 +0,0 @@
-import math
-from hestia_earth.distribution.utils.cycle import (
-    FERTILISER_COLUMNS,
-    PESTICIDE_COLUMN,
-    IRRIGATION_COLUMN,
-    YIELD_COLUMN,
-)
-from hestia_earth.distribution.posterior_fert import get_post as get_post_fert
-from hestia_earth.distribution.prior_fert import get_prior as get_prior_fert
-from hestia_earth.distribution.posterior_pest import get_post as get_post_pest
-from hestia_earth.distribution.prior_pest import get_prior as get_prior_pest
-from hestia_earth.distribution.posterior_irrigation import get_post as get_post_irri
-from hestia_earth.distribution.prior_irrigation import get_prior as get_prior_irri
-from hestia_earth.distribution.posterior_yield import get_post as get_post_yield
-from hestia_earth.distribution.prior_yield import get_prior as get_prior_yield
-
-UNIVARIATE_DEFAULT_THRESHOLD = 0.95
-UNIVARIATE_DEFAULT_ZSCORE = 1.96
-UNIVARIATE_CI_TO_ZSCORE = {
-    0.9: 1.65,
-    UNIVARIATE_DEFAULT_THRESHOLD: UNIVARIATE_DEFAULT_ZSCORE,
-    0.99: 2.58,
-}
-
-
-def _process_fertiliser(country_id: str, product_id: str, input_id: str):
-    mu, sd = get_post_fert(country_id, product_id, input_id)
-    return (mu, sd) if mu is not None else get_prior_fert(country_id, input_id)
-
-
-def _process_pesticide(country_id: str, product_id: str, *args):
-    mu, sd = get_post_pest(country_id, product_id)
-    return (mu, sd) if mu is not None else get_prior_pest(country_id)
-
-
-def _process_irrigation(country_id: str, product_id: str, *args):
-    mu, sd = get_post_irri(country_id, product_id)
-    return (mu, sd) if mu is not None else get_prior_irri(country_id)
-
-
-def _process_yield(country_id: str, product_id: str, *args):
-    mu, sd = get_post_yield(country_id, product_id)
-    return (mu, sd) if mu is not None else get_prior_yield(country_id, product_id)
-
-
-_PROCESS_BY_KEY = {
-    PESTICIDE_COLUMN: _process_pesticide,
-    IRRIGATION_COLUMN: _process_irrigation,
-    YIELD_COLUMN: _process_yield,
-} | {key: _process_fertiliser for key in FERTILISER_COLUMNS}
-
-
-def get_stats_by_group_key(
-    key: str, country_id: str, product_id: str, input_id: str = None
-):
-    return _PROCESS_BY_KEY[key](country_id, product_id, input_id)
-
-
-def cycle_completeness_key(key: str):
-    return (
-        "fertiliser"
-        if key in FERTILISER_COLUMNS
-        else (
-            "pesticideVeterinaryDrug"
-            if key == PESTICIDE_COLUMN
-            else "water" if key == IRRIGATION_COLUMN else ""
-        )
-    )
-
-
-def validate(values: list, threshold: float, get_mu_sd):
-    def exec():
-        z = UNIVARIATE_CI_TO_ZSCORE[threshold]
-        mu, sd = get_mu_sd()
-        _min = mu - (z * sd) if mu is not None else None
-        _max = mu + (z * sd) if mu is not None else None
-        passes = [
-            _min <= y <= _max if all([mu is not None, not math.isnan(y)]) else True
-            for y in values
-        ]
-        outliers = (
-            [y for y in values if not _min <= y <= _max] if mu is not None else []
-        )
-        return all(passes), outliers, max(_min or 0, 0), _max
-
-    return exec()

package/hestia_earth/validation/validators/emission.py

@@ -1,109 +0,0 @@
-from hestia_earth.schema import EmissionMethodTier
-from hestia_earth.utils.tools import flatten
-from hestia_earth.utils.model import find_term_match
-from hestia_earth.utils.blank_node import get_lookup_value
-
-from hestia_earth.validation.utils import _filter_list_errors
-
-
-def is_inputs_production(emission: dict):
-    return get_lookup_value(emission, "inputProductionGroupId") == emission.get(
-        "term", {}
-    ).get("@id")
-
-
-def validate_linked_terms(
-    cycle: dict, list_key: str, linked_key: str, linked_list_key: str, soft_check=False
-):
-    linked_nodes = cycle.get(linked_list_key, [])
-
-    def validate(values: tuple):
-        index, emission = values
-        linked_items = emission.get(linked_key, [])
-        return (
-            len(linked_items) == 0
-            or any(
-                [
-                    find_term_match(linked_nodes, item.get("@id"))
-                    for item in (
-                        [linked_items]
-                        if isinstance(linked_items, dict)
-                        else linked_items
-                    )
-                ]
-            )
-            or {
-                "level": "warning" if soft_check else "error",
-                "dataPath": f".{list_key}[{index}]",
-                "message": f"{'should' if soft_check else 'must'} add the linked {linked_list_key} to the cycle",
-                "params": {"term": emission.get("term", {}), "expected": linked_items},
-            }
-        )
-
-    return _filter_list_errors(
-        flatten(map(validate, enumerate(cycle.get(list_key, []))))
-    )
-
-
-_NOT_RELEVANT_ID = "emissionNotRelevant"
-
-
-def validate_method_not_relevant(cycle: dict, list_key: str):
-    def validate(values: tuple):
-        index, emission = values
-        term_id = emission.get("methodModel", {}).get("@id")
-        return term_id != _NOT_RELEVANT_ID or {
-            "level": "warning",
-            "dataPath": f".{list_key}[{index}].methodModel.@id",
-            "message": "should not use not relevant model",
-            "params": {
-                "term": emission.get("term", {}),
-                "model": emission.get("methodModel", {}),
-            },
-        }
-
-    return _filter_list_errors(
-        flatten(map(validate, enumerate(cycle.get(list_key, []))))
-    )
-
-
-def validate_methodTier_not_relevant(cycle: dict, list_key: str):
-    def validate(values: tuple):
-        index, emission = values
-        methodTier = emission.get("methodTier")
-        return methodTier != EmissionMethodTier.NOT_RELEVANT.value or {
-            "level": "warning",
-            "dataPath": f".{list_key}[{index}].methodTier",
-            "message": "should not use not relevant methodTier",
-            "params": {"term": emission.get("term", {})},
-        }
-
-    return _filter_list_errors(
-        flatten(map(validate, enumerate(cycle.get(list_key, []))))
-    )
-
-
-def validate_methodTier_background(node: dict, list_key: str):
-    allowed_values = [
-        e.value
-        for e in EmissionMethodTier
-        if e not in [EmissionMethodTier.BACKGROUND, EmissionMethodTier.NOT_RELEVANT]
-    ]
-
-    def validate(values: tuple):
-        index, emission = values
-        methodTier = emission.get("methodTier")
-        return (
-            methodTier != EmissionMethodTier.BACKGROUND.value
-            or is_inputs_production(emission)
-            or {
-                "level": "error",
-                "dataPath": f".{list_key}[{index}].methodTier",
-                "message": "should be equal to one of the allowed values",
-                "params": {"allowedValues": allowed_values},
-            }
-        )
-
-    return _filter_list_errors(
-        flatten(map(validate, enumerate(node.get(list_key, []))))
-    )

package/hestia_earth/validation/validators/impact_assessment.py

@@ -1,138 +0,0 @@
-"""
-Impact Assessment validation
-
-Here is the list of validations running on a [Impact Assessment](/schema/ImpactAssessment).
-"""
-
-from hestia_earth.utils.tools import flatten
-
-from hestia_earth.validation.utils import (
-    find_linked_node,
-    find_by_product,
-    find_by_unique_product,
-)
-from .shared import (
-    validate_list_min_below_max,
-    validate_country_region,
-    validate_list_country_region,
-    validate_list_term_percent,
-    validate_linked_source_privacy,
-    validate_date_lt_today,
-    validate_list_model,
-    validate_private_has_source,
-    validate_list_value_between_min_max,
-    is_date_equal,
-    validate_other_model,
-    validate_nested_existing_node,
-)
-from .emission import validate_methodTier_background
-from .indicator import (
-    validate_characterisedIndicator_model,
-    validate_landTransformation,
-    validate_inonising_compounds_waste,
-)
-
-
-def validate_empty(impact_assessment: dict):
-    return any(
-        [
-            impact_assessment.get("cycle"),
-            impact_assessment.get("emissionsResourceUse", []),
-            len(impact_assessment.get("impacts", [])) > 0,
-            len(impact_assessment.get("endpoints", [])) > 0,
-        ]
-    ) or {
-        "level": "error",
-        "dataPath": "",
-        "message": "should not be empty",
-        "params": {"type": "ImpactAssessment"},
-    }
-
-
-def validate_linked_cycle_product(impact_assessment: dict, cycle: dict):
-    product = impact_assessment.get("product", {})
-    cycle_product = find_by_product(cycle, product) or find_by_unique_product(
-        cycle, product
-    )
-    return cycle_product is not None or {
-        "level": "error",
-        "dataPath": ".product",
-        "message": "should be included in the cycle products",
-        "params": {
-            "product": product.get("term", {}),
-            "node": {"type": "Cycle", "id": cycle.get("id", cycle.get("@id"))},
-        },
-    }
-
-
-def validate_linked_cycle_endDate(impact_assessment: dict, cycle: dict):
-    key = "endDate"
-    return is_date_equal(impact_assessment.get(key), cycle.get(key), True) or {
-        "level": "error",
-        "dataPath": f".{key}",
-        "message": "must be equal to the Cycle endDate",
-    }
-
-
-def validate_impact_assessment(impact_assessment: dict, node_map: dict = {}):
-    cycle = find_linked_node(node_map, impact_assessment.get("cycle", {}))
-    return [
-        validate_empty(impact_assessment),
-        validate_date_lt_today(impact_assessment, "startDate"),
-        validate_date_lt_today(impact_assessment, "endDate"),
-        validate_linked_source_privacy(impact_assessment, "source", node_map),
-        validate_private_has_source(impact_assessment, "source"),
-        validate_country_region(impact_assessment),
-        validate_linked_cycle_product(impact_assessment, cycle) if cycle else True,
-        validate_linked_cycle_endDate(impact_assessment, cycle) if cycle else True,
-        validate_nested_existing_node(impact_assessment, "cycle"),
-        validate_nested_existing_node(impact_assessment, "site"),
-    ] + flatten(
-        (
-            [
-                validate_list_country_region(impact_assessment, "emissionsResourceUse"),
-                validate_list_min_below_max(impact_assessment, "emissionsResourceUse"),
-                validate_list_value_between_min_max(
-                    impact_assessment, "emissionsResourceUse"
-                ),
-                validate_list_term_percent(impact_assessment, "emissionsResourceUse"),
-                validate_characterisedIndicator_model(
-                    impact_assessment, "emissionsResourceUse"
-                ),
-                validate_landTransformation(impact_assessment, "emissionsResourceUse"),
-                validate_methodTier_background(
-                    impact_assessment, "emissionsResourceUse"
-                ),
-                validate_other_model(impact_assessment, "emissionsResourceUse"),
-                validate_inonising_compounds_waste(
-                    impact_assessment, "emissionsResourceUse"
-                ),
-            ]
-            if len(impact_assessment.get("emissionsResourceUse", [])) > 0
-            else []
-        )
-        + (
-            [
-                validate_list_country_region(impact_assessment, "emissionsResourceUse"),
-                validate_list_min_below_max(impact_assessment, "impacts"),
-                validate_list_value_between_min_max(impact_assessment, "impacts"),
-                validate_list_term_percent(impact_assessment, "impacts"),
-                validate_list_model(impact_assessment, "impacts"),
-                validate_characterisedIndicator_model(impact_assessment, "impacts"),
-                validate_other_model(impact_assessment, "emissionsResourceUse"),
-            ]
-            if len(impact_assessment.get("impacts", [])) > 0
-            else []
-        )
-        + (
-            [
-                validate_list_country_region(impact_assessment, "emissionsResourceUse"),
-                validate_list_min_below_max(impact_assessment, "endpoints"),
-                validate_list_value_between_min_max(impact_assessment, "endpoints"),
-                validate_list_term_percent(impact_assessment, "endpoints"),
-                validate_other_model(impact_assessment, "emissionsResourceUse"),
-            ]
-            if len(impact_assessment.get("endpoints", [])) > 0
-            else []
-        )
-    )

package/hestia_earth/validation/validators/indicator.py

@@ -1,154 +0,0 @@
-from functools import reduce
-from hestia_earth.schema import TermTermType
-from hestia_earth.utils.lookup import download_lookup, get_table_value, lookup_term_ids
-from hestia_earth.utils.tools import flatten, non_empty_list
-
-from hestia_earth.validation.utils import _filter_list_errors
-from hestia_earth.validation.terms import TERMS_QUERY, get_terms
-
-IGNORE_MODELS = [
-    "aggregatedModels"  # used only for aggregations, should not be shown as possible values
-]
-
-
-def _indicator_group_key(indicator: dict, include_term: bool = True):
-    return "-".join(
-        non_empty_list(
-            flatten(
-                [
-                    indicator.get("term", {}).get("@id") if include_term else "",
-                    indicator.get("country", {}).get("@id"),
-                    indicator.get("landCover", {}).get("@id"),
-                    [i.get("@id") for i in indicator.get("inputs", [])],
-                ]
-            )
-        )
-    )
-
-
-def _group_indicators(indicators: list, include_term: bool = True):
-    def group_by(groups: dict, indicator: dict):
-        group_key = _indicator_group_key(indicator, include_term)
-        groups[group_key] = groups.get(group_key, []) + [indicator]
-        return groups
-
-    return reduce(group_by, indicators, {})
-
-
-def _allowed_characterisedIndicator_model(lookup, models: list, term_id: str):
-    return [
-        m
-        for m in models
-        if m != "term.id"
-        and get_table_value(lookup, "term.id", term_id, m)
-        and m not in IGNORE_MODELS
-    ]
-
-
-def _is_method_allowed(lookup, term_id: str, model: str):
-    value = get_table_value(lookup, "term.id", term_id, model)
-    # bug numpy bool not returning `True`
-    return True if value else False
-
-
-def validate_characterisedIndicator_model(node: dict, list_key: str):
-    lookup = download_lookup(
-        "characterisedIndicator-model-mapping.csv", keep_in_memory=False
-    )
-    models = get_terms(TERMS_QUERY.MODEL)
-
-    def validate(values: tuple):
-        index, value = values
-        term = value.get("term", {})
-        term_id = term.get("@id")
-        model = value.get("methodModel", {})
-        model_id = model.get("@id")
-        should_validate = term_id in lookup_term_ids(lookup) and model_id is not None
-        return (
-            not should_validate
-            or _is_method_allowed(lookup, term_id, model_id)
-            or {
-                "level": "error",
-                "dataPath": f".{list_key}[{index}].methodModel.@id",
-                "message": "is not allowed for this characterisedIndicator",
-                "params": {
-                    "term": term,
-                    "model": model,
-                    "allowedValues": _allowed_characterisedIndicator_model(
-                        lookup, models, term_id
-                    ),
-                },
-            }
-        )
-
-    return _filter_list_errors(list(map(validate, enumerate(node.get(list_key, [])))))
-
-
-def _below_occupation(index: int, value: float, blank_nodes: list, suffix: str):
-    other_values = list(
-        filter(lambda b: b.get("term", {}).get("@id").endswith(suffix), blank_nodes)
-    )
-    other_values = [other_value.get("value", 0) for other_value in other_values]
-    return (
-        len(other_values) == 0
-        or all([value <= (other_value * 1.05) for other_value in other_values])
-        or {
-            "level": "error",
-            "dataPath": f".emissionsResourceUse[{index}].value",
-            "message": "must be less than or equal to land occupation",
-            "params": {"current": value, "max": min(other_values)},
-        }
-    )
-
-
-def validate_landTransformation(node: dict, list_key="emissionsResourceUse"):
-    blank_nodes = node.get(list_key, [])
-    grouped_land_occupation = _group_indicators(
-        [
-            v
-            for v in blank_nodes
-            if v.get("term", {}).get("@id", "").startswith("landOccupation")
-        ],
-        include_term=False,
-    )
-
-    def validate(values: tuple):
-        index, blank_node = values
-        term_id = blank_node.get("term", {}).get("@id")
-        value = blank_node.get("value", 0)
-        group_key = _indicator_group_key(blank_node, include_term=False)
-        land_occupation_blank_nodes = grouped_land_occupation.get(group_key) or []
-        return not term_id.startswith("landTransformation") or [
-            not term_id.endswith("DuringCycle")
-            or _below_occupation(
-                index, value, land_occupation_blank_nodes, "DuringCycle"
-            ),
-            not term_id.endswith("InputsProduction")
-            or _below_occupation(
-                index, value, land_occupation_blank_nodes, "InputsProduction"
-            ),
-        ]
-
-    return _filter_list_errors(flatten(map(validate, enumerate(blank_nodes))))
-
-
-def validate_inonising_compounds_waste(node: dict, list_key="emissionsResourceUse"):
-    blank_nodes = node.get(list_key, [])
-    allowed_term_types = [None, TermTermType.WASTE.value]
-
-    def validate(values: tuple):
-        index, blank_node = values
-        term_id = blank_node.get("term", {}).get("@id")
-        key_term_type = blank_node.get("key", {}).get("termType")
-        return (
-            not term_id.startswith("ionisingCompounds")
-            or key_term_type in allowed_term_types
-            or {
-                "level": "error",
-                "dataPath": f".{list_key}[{index}].key.termType",
-                "message": "should be equal to one of the allowed values",
-                "params": {"allowedValues": non_empty_list(allowed_term_types)},
-            }
-        )
-
-    return _filter_list_errors(flatten(map(validate, enumerate(blank_nodes))))

package/hestia_earth/validation/validators/infrastructure.py

@@ -1,25 +0,0 @@
-from hestia_earth.utils.date import diff_in_years
-
-from hestia_earth.validation.utils import _filter_list_errors
-
-
-def validate_lifespan(infrastructure: list):
-    def validate(values: tuple):
-        index, value = values
-        start_date = value.get("startDate")
-        end_date = value.get("endDate")
-        lifespan = value.get("defaultLifespan", -1)
-        diff = diff_in_years(start_date, end_date) if start_date and end_date else -1
-        return (
-            lifespan == -1
-            or diff == -1
-            or diff == round(lifespan, 1)
-            or {
-                "level": "error",
-                "dataPath": f".infrastructure[{index}].defaultLifespan",
-                "message": "must equal to endDate - startDate in decimal years",
-                "params": {"expected": diff},
-            }
-        )
-
-    return _filter_list_errors(map(validate, enumerate(infrastructure)))