@hestia-earth/data-validation 0.37.7 → 0.37.9

package/hestia_earth/validation/validators/product.py

@@ -1,263 +0,0 @@
-import traceback
-from hestia_earth.schema import TermTermType, CycleFunctionalUnit
-from hestia_earth.utils.lookup import extract_grouped_data
-from hestia_earth.utils.tools import list_sum, flatten, non_empty_list
-from hestia_earth.utils.model import filter_list_term_type
-
-from hestia_earth.validation.log import logger
-from hestia_earth.validation.utils import (
-    _list_sum,
-    _filter_list_errors,
-    get_lookup_value,
-)
-from hestia_earth.validation.distribution import is_enabled as distribution_is_enabled
-from .shared import CROP_SITE_TYPE
-
-
-def validate_economicValueShare(products: list):
-    sum = _list_sum(products, "economicValueShare")
-    return sum <= 100.5 or {
-        "level": "error",
-        "dataPath": ".products",
-        "message": "economicValueShare should sum to 100 or less across all products",
-        "params": {"sum": sum},
-    }
-
-
-def validate_value_empty(products: list):
-    def validate(values: tuple):
-        index, product = values
-        return len(product.get("value", [])) > 0 or {
-            "level": "warning",
-            "dataPath": f".products[{index}]",
-            "message": "may not be 0",
-        }
-
-    return _filter_list_errors(map(validate, enumerate(products)))
-
-
-def validate_value_0(products: list):
-    def validate(values: tuple):
-        index, product = values
-        value = list_sum(product.get("value", [-1]), -1)
-        eva = product.get("economicValueShare", 0)
-        revenue = product.get("revenue", 0)
-        return value != 0 or _filter_list_errors(
-            [
-                eva == 0
-                or {
-                    "level": "error",
-                    "dataPath": f".products[{index}].value",
-                    "message": "economicValueShare must be 0 for product value 0",
-                    "params": {"value": eva, "term": product.get("term")},
-                },
-                revenue == 0
-                or {
-                    "level": "error",
-                    "dataPath": f".products[{index}].value",
-                    "message": "revenue must be 0 for product value 0",
-                    "params": {"value": revenue, "term": product.get("term")},
-                },
-            ]
-        )
-
-    return _filter_list_errors(flatten(map(validate, enumerate(products))))
-
-
-MAX_PRIMARY_PRODUCTS = 1
-
-
-def validate_primary(products: list):
-    primary = list(filter(lambda p: p.get("primary", False), products))
-    return len(primary) <= MAX_PRIMARY_PRODUCTS or {
-        "level": "error",
-        "dataPath": ".products",
-        "message": "only 1 primary product allowed",
-    }
-
-
-def validate_product_ha_functional_unit_ha(cycle: dict, list_key: str = "products"):
-    functional_unit = cycle.get("functionalUnit", CycleFunctionalUnit.RELATIVE.value)
-
-    def validate(values: tuple):
-        index, product = values
-        term_units = product.get("term", {}).get("units")
-        value = list_sum(product.get("value", [0]))
-        return (
-            term_units != "ha"
-            or value <= 1
-            or {
-                "level": "error",
-                "dataPath": f".{list_key}[{index}].value",
-                "message": "must be below or equal to 1 for unit in ha",
-                "params": {"term": product.get("term", {})},
-            }
-        )
-
-    return functional_unit != CycleFunctionalUnit._1_HA.value or _filter_list_errors(
-        map(validate, enumerate(cycle.get(list_key, [])))
-    )
-
-
-def _validate_product_yield(country: dict, list_key: str, threshold: float):
-    from .distribution import (
-        YIELD_COLUMN,
-        get_stats_by_group_key,
-        validate as validate_distribution,
-    )
-
-    country_id = country.get("@id")
-
-    def validate(values: tuple):
-        index, product = values
-
-        product_id = product.get("term", {}).get("@id")
-        product_value = product.get("value", [])
-
-        def _get_mu_sd():
-            return get_stats_by_group_key(YIELD_COLUMN, country_id, product_id)
-
-        valid, outliers, min, max = validate_distribution(
-            product_value, threshold, get_mu_sd=_get_mu_sd
-        )
-        return valid or {
-            "level": "warning",
-            "dataPath": f".{list_key}[{index}].value",
-            "message": "is outside confidence interval",
-            "params": {
-                "term": product.get("term", {}),
-                "country": country,
-                "outliers": outliers,
-                "threshold": threshold,
-                "min": min,
-                "max": max,
-            },
-        }
-
-    return validate
-
-
-def validate_product_yield(
-    cycle: dict, site: dict, list_key: str = "products", threshold: float = 0.95
-):
-    country = site.get("country", {})
-    products = cycle.get(list_key, [])
-
-    try:
-        return (
-            site.get("siteType") not in CROP_SITE_TYPE
-            or (
-                _filter_list_errors(
-                    map(
-                        _validate_product_yield(country, list_key, threshold),
-                        enumerate(products),
-                    )
-                )
-            )
-            if distribution_is_enabled()
-            else True
-        )
-    except Exception:
-        stack = traceback.format_exc()
-        logger.error(f"Error validating using distribution: '{stack}'")
-        return True
-
-
-def _excreta_term_ids(term: dict, column: str, group_key: str):
-    value = get_lookup_value(term, column)
-    # handle using `|` to allow multiple values
-    return non_empty_list((extract_grouped_data(value, group_key) or "").split("|"))
-
-
-def _grouped_excreta_term_ids(term: dict, group_keys: list):
-    grouped_columns = {
-        "kg": "excretaKgMassTermIds",
-        "kg N": "excretaKgNTermIds",
-        "kg VS": "excretaKgVsTermIds",
-    }
-    grouped_values = {
-        units: flatten(
-            [_excreta_term_ids(term, column, group_key) for group_key in group_keys]
-        )
-        for units, column in grouped_columns.items()
-    }
-    # include `default` key if no values found
-    return {
-        units: grouped_values.get(units, [])
-        + (
-            _excreta_term_ids(term, column, "default")
-            if not grouped_values.get(units)
-            else []
-        )
-        for units, column in grouped_columns.items()
-    }
-
-
-def validate_excreta_product(cycle: dict, list_key: str = "products"):
-    animal_products = [
-        (index, product)
-        for index, product in enumerate(cycle.get(list_key, []))
-        if product.get("term", {}).get("termType")
-        in [
-            TermTermType.ANIMALPRODUCT.value,
-            TermTermType.LIVEANIMAL.value,
-            TermTermType.LIVEAQUATICSPECIES.value,
-        ]
-    ]
-    excreta_products = [
-        (index, product)
-        for index, product in enumerate(cycle.get(list_key, []))
-        if product.get("term", {}).get("termType") in [TermTermType.EXCRETA.value]
-    ]
-    systems = filter_list_term_type(cycle.get("practices", []), TermTermType.SYSTEM)
-    group_keys = [s.get("term", {}).get("@id") for s in systems]
-    allowed_excreta_per_product = {
-        product.get("term", {}).get("@id"): _grouped_excreta_term_ids(
-            product.get("term", {}), group_keys
-        )
-        for _i, product in animal_products
-    }
-
-    def validate_animal(values: tuple):
-        """
-        Validate the the animal product has an excreta term
-        """
-        index, blank_node = values
-        term = blank_node.get("term", {})
-        term_id = term.get("@id")
-        allowed_ids = flatten(allowed_excreta_per_product.get(term_id, {}).values())
-        return {
-            "level": "warning",
-            "dataPath": f".{list_key}[{index}]",
-            "message": "should add an excreta product",
-            "params": {"expected": allowed_ids},
-        }
-
-    def validate_excreta(values: tuple):
-        """
-        Validate that the excreta is allowed.
-        """
-        index, blank_node = values
-        term = blank_node.get("term", {})
-        term_id = term.get("@id")
-        allowed_ids = non_empty_list(
-            flatten(
-                [v.get(term.get("units")) for v in allowed_excreta_per_product.values()]
-            )
-        )
-        return term_id in allowed_ids or {
-            "level": "error",
-            "dataPath": f".{list_key}[{index}].term.@id",
-            "message": "is not an allowed excreta product",
-            "params": {"current": term_id, "expected": allowed_ids},
-        }
-
-    return (
-        (
-            _filter_list_errors(map(validate_excreta, excreta_products))
-            if excreta_products
-            else _filter_list_errors(map(validate_animal, animal_products))
-        )
-        if animal_products
-        else True
-    )

package/hestia_earth/validation/validators/property.py

@@ -1,266 +0,0 @@
-from hestia_earth.utils.api import download_hestia
-from hestia_earth.utils.model import find_term_match
-from hestia_earth.utils.tools import flatten, non_empty_list, safe_parse_float
-
-from hestia_earth.validation.utils import (
-    match_value_type,
-    term_valueType,
-    get_lookup_value,
-    update_error_path,
-    _filter_list_errors,
-    _flatten_errors,
-)
-from .shared import (
-    value_difference,
-    is_value_below,
-    validate_list_value_between_min_max,
-    validate_list_min_max_lookup,
-)
-
-
-PROPERTIES_KEY = "properties"
-
-
-def validate_property_valueType(node: dict, list_key: str):
-    def is_valid(values: tuple):
-        index, property = values
-        term = property.get("term", {})
-        expected_value_type = term_valueType(term)
-        value = property.get("value")
-        return (
-            value is None
-            or match_value_type(expected_value_type, value)
-            or {
-                "level": "error",
-                "dataPath": f".{PROPERTIES_KEY}[{index}].value",
-                "message": "the property value type is incorrect",
-                "params": {"expected": expected_value_type},
-            }
-        )
-
-    def validate(values: tuple):
-        index, blank_node = values
-        errors = list(map(is_valid, enumerate(blank_node.get(PROPERTIES_KEY, []))))
-        return _filter_list_errors(
-            [
-                update_error_path(error, list_key, index)
-                for error in errors
-                if error is not True
-            ]
-        )
-
-    return _filter_list_errors(
-        flatten(map(validate, enumerate(node.get(list_key, []))))
-    )
-
-
-def validate_term_type(node: dict, list_key: str):
-    def is_valid(blank_node: dict):
-        def check(values: tuple):
-            index, property = values
-            term = property.get("term", {})
-            term_types = get_lookup_value(term, "termTypesAllowed")
-            expected_term_types = (term_types or "all").split(";")
-            term_type = blank_node.get("term", {}).get("termType")
-            return any(
-                ["all" in expected_term_types, term_type in expected_term_types]
-            ) or {
-                "level": "error",
-                "dataPath": f".{PROPERTIES_KEY}[{index}].term.termType",
-                "message": "can not be used on this termType",
-                "params": {"current": term_type, "expected": expected_term_types},
-            }
-
-        return check
-
-    def validate(values: tuple):
-        index, blank_node = values
-        errors = list(
-            map(is_valid(blank_node), enumerate(blank_node.get(PROPERTIES_KEY, [])))
-        )
-        return _filter_list_errors(
-            [
-                update_error_path(error, list_key, index)
-                for error in errors
-                if error is not True
-            ]
-        )
-
-    return _filter_list_errors(
-        flatten(map(validate, enumerate(node.get(list_key, []))))
-    )
-
-
-def _property_default_value(term_id: str, property_term_id: str):
-    # load the term defaultProperties and find the matching property
-    term = download_hestia(term_id)
-    if not term:
-        raise Exception(f"Term not found: {term_id}")
-    return safe_parse_float(
-        find_term_match(term.get("defaultProperties", []), property_term_id).get(
-            "value"
-        )
-    )
-
-
-def _property_default_allowed_values(term: dict):
-    allowed = get_lookup_value(term, "validationAllowedExceptions")
-    try:
-        allowed_values = non_empty_list(allowed.split(";")) if allowed else []
-        return [safe_parse_float(v) for v in allowed_values]
-    # failure to split by `;` as single value allowed
-    except AttributeError:
-        return [safe_parse_float(allowed)]
-
-
-def validate_default_value(node: dict, list_key: str):
-    threshold = 0.25
-
-    def is_valid(term_id: str):
-        def validate(values: tuple):
-            index, prop = values
-            value = safe_parse_float(prop.get("value"))
-            prop_term_id = prop.get("term", {}).get("@id")
-            default_value = _property_default_value(term_id, prop_term_id)
-            delta = value_difference(value, default_value)
-            values_allowed = (
-                _property_default_allowed_values(prop.get("term", {}))
-                if prop_term_id
-                else []
-            )
-            return (
-                prop.get("value") is None
-                or delta < threshold
-                or value in values_allowed
-                or {
-                    "level": "warning",
-                    "dataPath": f".{PROPERTIES_KEY}[{index}].value",
-                    "message": "should be within percentage of default value",
-                    "params": {
-                        "current": value,
-                        "default": default_value,
-                        "percentage": delta * 100,
-                        "threshold": threshold,
-                    },
-                }
-            )
-
-        return validate
-
-    def validate(values: tuple):
-        index, blank_node = values
-        term_id = blank_node.get("term", {}).get("@id")
-        errors = list(
-            map(is_valid(term_id), enumerate(blank_node.get(PROPERTIES_KEY, [])))
-        )
-        return _filter_list_errors(
-            [
-                update_error_path(error, list_key, index)
-                for error in errors
-                if error is not True
-            ]
-        )
-
-    return _filter_list_errors(
-        flatten(map(validate, enumerate(node.get(list_key, []))))
-    )
-
-
-VSC_ID = "volatileSolidsContent"
-VSC_MIN = {"kg": 0, "kg Vs": 100, "kg N": 0}
-VSC_MAX = {"kg": 100, "kg Vs": 100, "kg N": None}
-
-
-def _volatileSolidsContent_error(min: float = None, max: float = None):
-    return (
-        f"must be {max}"
-        if min == max
-        else (
-            f"must be above {min}"
-            if max is None
-            else (
-                f"must be below {max}"
-                if min is None
-                else f"must be between {min} and {max}"
-            )
-        )
-    )
-
-
-def validate_volatileSolidsContent(node: dict, list_key: str):
-    def is_valid(blank_node: dict):
-        units = blank_node.get("term", {}).get("units")
-
-        def validate(values: tuple):
-            index, property = values
-            term_id = property.get("term", {}).get("@id")
-            value = property.get("value", 0)
-            min = VSC_MIN.get(units)
-            max = VSC_MAX.get(units)
-            return (
-                term_id != VSC_ID
-                or all([is_value_below(value, max), is_value_below(min, value)])
-                or {
-                    "level": "error",
-                    "dataPath": f".{PROPERTIES_KEY}[{index}].value",
-                    "message": _volatileSolidsContent_error(min, max),
-                }
-            )
-
-        return validate
-
-    def validate(values: tuple):
-        index, blank_node = values
-        errors = flatten(
-            map(is_valid(blank_node), enumerate(blank_node.get(PROPERTIES_KEY, [])))
-        )
-        return _filter_list_errors(
-            [
-                update_error_path(error, list_key, index)
-                for error in errors
-                if error is not True
-            ]
-        )
-
-    return _filter_list_errors(
-        flatten(map(validate, enumerate(node.get(list_key, []))))
-    )
-
-
-def validate_value_min_max(node: dict, list_key: str):
-    def validate(values: tuple):
-        index, blank_node = values
-        # handle skip maximum validation for some properties
-        skip_maximum_properties = (
-            get_lookup_value(blank_node.get("term"), "skipValidateTermIdsMaximum") or ""
-        ).split(";")
-        errors = _flatten_errors(
-            [
-                validate_list_value_between_min_max(blank_node, PROPERTIES_KEY),
-                validate_list_min_max_lookup(
-                    blank_node, PROPERTIES_KEY, skip_max_ids=skip_maximum_properties
-                ),
-            ]
-        )
-        return _filter_list_errors(
-            [
-                update_error_path(error, list_key, index)
-                for error in errors
-                if error is not True
-            ]
-        )
-
-    return _filter_list_errors(
-        flatten(map(validate, enumerate(node.get(list_key, []))))
-    )
-
-
-def validate_all(node: dict, list_key: str):
-    return _filter_list_errors(
-        [
-            validate_default_value(node, list_key),
-            validate_term_type(node, list_key),
-            validate_property_valueType(node, list_key),
-            validate_value_min_max(node, list_key),
-        ]
-    )