@hestia-earth/data-validation 0.37.7 → 0.37.9

package/hestia_earth/validation/utils.py

@@ -1,444 +0,0 @@
-import json
-from typing import List
-from functools import reduce, lru_cache
-from datetime import datetime, timedelta
-from hestia_earth.schema import NodeType, TermTermType, UNIQUENESS_FIELDS
-from hestia_earth.utils.api import download_hestia
-from hestia_earth.utils.lookup import download_lookup, get_table_value
-from hestia_earth.utils.tools import (
-    list_average,
-    safe_parse_date,
-    safe_parse_float,
-    non_empty_list,
-    is_number,
-    is_boolean,
-    get_dict_key,
-    flatten,
-)
-from hestia_earth.utils.model import filter_list_term_type, find_primary_product
-
-ANIMAL_TERM_TYPES = [TermTermType.LIVEANIMAL, TermTermType.LIVEAQUATICSPECIES]
-
-
-@lru_cache()
-def _get_term_lookup_value(term_id: str, term_type: str, column: str):
-    lookup = download_lookup(f"{term_type}.csv", keep_in_memory=True)
-    return get_table_value(lookup, "term.id", term_id, column)
-
-
-def get_lookup_value(lookup_term: dict, column: str):
-    value = (
-        _get_term_lookup_value(
-            lookup_term.get("@id"), lookup_term.get("termType"), column
-        )
-        if lookup_term
-        else None
-    )
-    return value
-
-
-def _next_error(values: list):
-    return next((x for x in values if x is not True), True)
-
-
-def _filter_list_errors(values: list, return_single=True):
-    values = list(filter(lambda x: x is not True, values))
-    return (
-        True
-        if return_single and len(values) == 0
-        else (values[0] if return_single and len(values) == 1 else values)
-    )
-
-
-def _flatten_errors(errors):
-    errors_list = [
-        [] if isinstance(v, bool) else v if isinstance(v, list) else [v] for v in errors
-    ]
-    return flatten(errors_list)
-
-
-def _list_except_item(values: list, item):
-    try:
-        idx = values.index(item)
-        return values[:idx] + values[idx + 1 :]
-    except ValueError:
-        return values
-
-
-def update_error_path(error: dict, key: str, index=None):
-    path = (
-        f".{key}[{index}]{error.get('dataPath')}"
-        if index is not None
-        else f".{key}{error.get('dataPath')}"
-    )
-    return error | {"dataPath": path}
-
-
-def _safe_cast(val, to_type, default=None):
-    try:
-        return to_type(val)
-    except (ValueError, TypeError):
-        return default
-
-
-def hash_dict(value: dict):
-    return json.dumps(value, sort_keys=True)
-
-
-def is_same_dict(a: dict, b: dict):
-    return hash_dict(a) == hash_dict(b)
-
-
-def _dict_without_key(a: dict, key: str):
-    no_key = a.copy()
-    if key in no_key:
-        no_key.pop(key)
-    return no_key
-
-
-_GROUP_NODE_TYPES = [
-    NodeType.CYCLE.value,
-    NodeType.IMPACTASSESSMENT.value,
-    NodeType.SITE.value,
-    NodeType.SOURCE.value,
-]
-
-
-def _is_node(value):
-    return (
-        isinstance(value, dict)
-        and (value.get("type") or value.get("@type")) in _GROUP_NODE_TYPES
-        and (value.get("id") or value.get("@id"))
-    )
-
-
-def _nodes_grouper(nodes: List[dict], grouping_func):
-    def group(groups: dict, node: dict):
-        if _is_node(node):
-            id = node.get("id") or node.get("@id")
-            type = node.get("type") or node.get("@type")
-            grouping_func(groups, node, type, id)
-        return groups
-
-    # check for nested nodes
-    nested_nodes = flatten(
-        [
-            v
-            for node in nodes
-            for v in node.values()
-            if _is_node(v) or (isinstance(v, list) and len(v) > 0 and _is_node(v[0]))
-        ]
-    )
-    data = reduce(group, nested_nodes, {})
-    return reduce(group, nodes, data)
-
-
-def _group_nodes(nodes: List[dict]):
-    def group_by(groups: dict, node: dict, type: str, id: str):
-        groups[type] = groups.get(type, {})
-        groups[type][id] = node
-
-    return _nodes_grouper(nodes, group_by)
-
-
-def _hash_nodes(nodes: List[dict]):
-    def group_by(groups: dict, node: dict, type: str, id: str):
-        # store the hash of the node without the `id` for uniqueness check
-        key = hash_dict(_dict_without_key(node, "id"))
-        groups[key] = groups.get(key, []) + [node]
-
-    return _nodes_grouper(nodes, group_by)
-
-
-def _list_sum(values: list, prop: str):
-    return sum(map(lambda v: _safe_cast(v.get(prop, 0), float, 0.0), values))
-
-
-def list_sum_terms(values: list, term_ids=[], default=None):
-    average_values = non_empty_list(
-        [
-            _value_average(node, default=default)
-            for node in values
-            if node.get("term", {}).get("@id") in term_ids
-        ]
-    )
-    return sum(average_values) if average_values else None
-
-
-def _compare_values(x, y):
-    return (
-        next((True for item in x if item in y), False)
-        if isinstance(x, list) and isinstance(y, list)
-        else x == y
-    )
-
-
-def _same_properties(value: dict, props: List[str]):
-    def identical(test: dict):
-        same_values = list(
-            filter(
-                lambda x: _compare_values(
-                    get_dict_key(value, x), get_dict_key(test, x)
-                ),
-                props,
-            )
-        )
-        return test if len(same_values) == len(props) else None
-
-    return identical
-
-
-def _value_average(node: dict, default=0, key="value"):
-    try:
-        value = node.get(key)
-        return (
-            list_average(value, default)
-            if isinstance(value, list)
-            else (value or default)
-        )
-    except Exception:
-        return default
-
-
-def term_id_prefix(term_id: str):
-    return term_id.split("Kg")[0]
-
-
-def _download_linked_node(node: dict):
-    data = (
-        download_hestia(node.get("@id"), node.get("@type"))
-        if node.get("@id") and node.get("@type")
-        else None
-    )
-    return data if (data or {}).get("@id") == node.get("@id") else None
-
-
-def find_linked_node(node_map: dict, node: dict):
-    """
-    Find the Node by type and id in the list of nodes.
-    """
-    return node_map.get(node.get("type"), {}).get(
-        node.get("id") or node.get("@id")
-    ) or _download_linked_node(node)
-
-
-def _value_as_array(data: dict, key: str):
-    value = data.get(key)
-    return value if isinstance(value, list) else non_empty_list([value])
-
-
-def find_related_nodes(
-    node_map: dict, node: dict, related_key: str, related_type: NodeType
-):
-    """
-    Find all nodes related to the same node via a key.
-    Example: find all Cycles related to a Site via the key "site".
-
-    Parameters
-    ----------
-    node_map : dict
-        The list of all nodes to do cross-validation, grouped by `type` and `id`.
-    node : dict
-        The node the other nodes should be related to.
-    related_key : str
-        How the other nodes are related to the `node`.
-    related_type : NodeType
-        The type of the related nodes.
-
-    Returns
-    -------
-    List[dict]
-        The list of nodes related to the `node`.
-    """
-    node_id = node.get("@id", node.get("id"))
-    nodes = node_map.get(related_type.value, {}).values()
-    return list(
-        {
-            n.get("@id", n.get("id")): n
-            for n in nodes
-            for related_node in _value_as_array(n, related_key)
-            if (related_node.get("@id", related_node.get("id")) == node_id)
-        }.values()
-    )
-
-
-def _is_before_today(date: str):
-    return safe_parse_date(date).date() <= datetime.now().date()
-
-
-def _node_year(node: dict):
-    date = node.get("endDate", node.get("startDate"))
-    date = safe_parse_date(date) if date else None
-    return date.year if date else None
-
-
-def is_live_animal_cycle(cycle: dict):
-    blank_nodes = cycle.get("animals", []) + cycle.get("products", [])
-    animals = filter_list_term_type(blank_nodes, ANIMAL_TERM_TYPES)
-    return len(animals) > 0
-
-
-def contains_grazing_animals(cycle: dict):
-    blank_nodes = cycle.get("animals", []) + cycle.get("products", [])
-    animals = filter_list_term_type(blank_nodes, ANIMAL_TERM_TYPES)
-    return any(
-        [v for v in animals if get_lookup_value(v.get("term", {}), "isGrazingAnimal")]
-    )
-
-
-def _match_list_el(source: list, dest: list, key: str):
-    src_values = non_empty_list([get_dict_key(x, key) for x in source])
-    dest_values = non_empty_list([get_dict_key(x, key) for x in dest])
-    return sorted(src_values) == sorted(dest_values)
-
-
-def _match_el(source: dict, dest: dict, fields: list):
-    def match(key: str):
-        keys = key.split(".")
-        is_list = len(keys) >= 2 and (
-            isinstance(get_dict_key(source, keys[0]), list)
-            or isinstance(get_dict_key(dest, keys[0]), list)
-        )
-        return (
-            _match_list_el(
-                get_dict_key(source, keys[0]) or [],
-                get_dict_key(dest, keys[0]) or [],
-                ".".join(keys[1:]),
-            )
-            if is_list
-            else get_dict_key(source, key) == get_dict_key(dest, key)
-        )
-
-    return all(map(match, fields))
-
-
-def find_by_unique_product(node: dict, product: dict, list_key: str = "products"):
-    """
-    Fallback to finding a product with unique keys if a single product has the same `term.@id`.
-    """
-    products = node.get(list_key, [])
-    products = [p for p in products if _match_el(p, product, ["term.@id"])]
-    return products[0] if len(products) == 1 else None
-
-
-def find_by_product(node: dict, product: dict, list_key: str = "products"):
-    keys = UNIQUENESS_FIELDS.get(node.get("type", node.get("@type")), {}).get(
-        list_key, ["term.@id"]
-    )
-    products = node.get(list_key, [])
-    return next((p for p in products if _match_el(p, product, keys)), None)
-
-
-def is_same_product(p1: dict, p2: dict):
-    return find_by_product({"type": NodeType.CYCLE.value, "products": [p1]}, p2)
-
-
-def _formatDepth(depth: str):
-    # handle float values
-    return str(int(depth)) if is_number(depth) else ""
-
-
-def blank_node_properties_group(blank_node: dict):
-    def property_group(property: dict):
-        return get_lookup_value(property.get("term", {}), "blankNodesGroup")
-
-    properties = blank_node.get("properties", [])
-    return "-".join(non_empty_list(map(property_group, properties)))
-
-
-def _blank_node_sum_groups(blank_node: dict, allow_sum_100: bool = True):
-    term = blank_node.get("term", {})
-    sum_below_100_group = get_lookup_value(term, "sumMax100Group")
-    sum_equal_100_group = get_lookup_value(term, "sumIs100Group")
-
-    return {
-        "sumMax100Group": sum_below_100_group
-        or (sum_equal_100_group if not allow_sum_100 else None),
-        "sumIs100Group": sum_equal_100_group if allow_sum_100 else None,
-    }
-
-
-def group_blank_nodes(nodes: list, by_sum: bool = True):
-    """
-    Group a list of blank nodes using:
-    - the `depthUpper`, `depthLower`, `startDate`, `endDate`, `dates`
-    - the lookup group `sumMax100Group` or `sumIs100Group` if specified
-    - the lookup group `blankNodesGroup` on properties if any
-
-    Parameters
-    ----------
-    nodes : list
-        List of blank nodes with their index.
-    by_sum : bool
-        Group blank nodes using the key to sum to 100% (`sumMax100Group` and `sumIs100Group`).
-    """
-
-    def group_by(group: dict, values: tuple):
-        index, blank_node = values
-        properties_group = blank_node_properties_group(blank_node)
-        # note: grouping of `properties` disables the grouping == 100
-        sum_groups = _blank_node_sum_groups(
-            blank_node, allow_sum_100=not properties_group
-        )
-        keys = non_empty_list(
-            [
-                _formatDepth(blank_node.get("depthUpper", "")),
-                _formatDepth(blank_node.get("depthLower", "")),
-                blank_node.get("startDate"),
-                blank_node.get("endDate"),
-                "-".join(blank_node.get("dates") or []),
-                properties_group,
-            ]
-            + (list(sum_groups.values()) if by_sum else [])
-        )
-        key = "-".join(keys) if len(keys) > 0 else "default"
-
-        if not by_sum or all(
-            [blank_node.get("value", []), any(list(sum_groups.values()))]
-        ):
-            group[key] = group.get(key, []) + [
-                {"index": index, "node": blank_node} | (sum_groups if by_sum else {})
-            ]
-
-        return group
-
-    return reduce(group_by, nodes, {})
-
-
-def is_permanent_crop(cycle: dict):
-    product = find_primary_product(cycle) or {}
-    return (
-        get_lookup_value(product.get("term", {}), "cropGroupingFAO")
-        == "Permanent crops"
-    )
-
-
-def term_valueType(term: dict):
-    return get_lookup_value(term, "valueType")
-
-
-VALUE_TYPE_MATCH = {"number": is_number, "boolean": is_boolean}
-
-
-def match_value_type(value_type: str, value):
-    values = non_empty_list(value if isinstance(value, list) else [value])
-    return all([VALUE_TYPE_MATCH.get(value_type, lambda _: True)(v) for v in values])
-
-
-def cycle_start_date(cycle: dict) -> datetime:
-    product = find_primary_product(cycle)
-    max_cycleDuration = (
-        cycle.get("cycleDuration")
-        or safe_parse_float(
-            value=get_lookup_value(product["term"], "maximumCycleDuration"), default=0
-        )
-        if product
-        else 0
-    )
-    return (
-        safe_parse_date(cycle.get("startDate"))
-        if cycle.get("startDate")
-        else (
-            safe_parse_date(cycle.get("endDate")) - (timedelta(days=max_cycleDuration))
-        )
-    )

package/hestia_earth/validation/validators/__init__.py

@@ -1,141 +0,0 @@
-import os
-from hestia_earth.schema import SchemaType
-from hestia_earth.utils.tools import flatten
-
-from ..log import logger
-from ..utils import update_error_path
-from .shared import validate_empty_fields, validate_nodes_duplicates
-from .cycle import validate_cycle
-from .impact_assessment import validate_impact_assessment
-from .organisation import validate_organisation
-from .site import validate_site
-from .source import validate_source
-
-# disable validation based on `@type`
-VALIDATE_EXISTING_NODES = os.getenv("VALIDATE_EXISTING_NODES", "false") == "true"
-VALIDATE_TYPE = {
-    SchemaType.CYCLE.value: lambda n, nodes: validate_cycle(n, nodes),
-    SchemaType.IMPACTASSESSMENT.value: lambda n, nodes: validate_impact_assessment(
-        n, nodes
-    ),
-    SchemaType.ORGANISATION.value: lambda n, nodes: validate_organisation(n, nodes),
-    SchemaType.SITE.value: lambda n, nodes: validate_site(n, nodes),
-    SchemaType.SOURCE.value: lambda n, nodes: validate_source(n, nodes),
-}
-SKIP_VALIDATE_DUPLICATES = [
-    SchemaType.ACTOR.value,
-    SchemaType.IMPACTASSESSMENT.value,
-    SchemaType.TERM.value,
-]
-
-
-def _has_keys(node: dict):
-    # ignore some keys that are automatically added to nested objects
-    ignore_keys = ["type", "id", "@type", "@id", "name", "description", "siteType"]
-    node_keys = [k for k in list(node.keys()) if k not in ignore_keys]
-    return len(node_keys) > 0
-
-
-def _should_run(ntype: str, node: dict):
-    return all([ntype in VALIDATE_TYPE, _has_keys(node)])
-
-
-def _validate_node_type(node_by_type: dict, node_by_hash: dict, ntype: str, node: dict):
-    should_run = _should_run(ntype, node)
-    if should_run:
-        logger.debug(
-            "Run validation on: type=%s, id=%s", ntype, node.get("id", node.get("@id"))
-        )
-    validator = VALIDATE_TYPE.get(ntype)
-    validations = validator(node, node_by_type) if should_run else []
-    return (
-        validations
-        + validate_empty_fields(node)
-        + (
-            []
-            if any([ntype in SKIP_VALIDATE_DUPLICATES, not should_run])
-            else validate_nodes_duplicates(node, node_by_hash)
-        )
-    )
-
-
-def _validate_node_children(node_by_type: dict, node_by_hash: dict, node: dict):
-    validations = []
-    for key, value in node.items():
-        if isinstance(value, list):
-            validations.extend(
-                [
-                    _validate_node_child(node_by_type, node_by_hash, key, value, index)
-                    for index, value in enumerate(value)
-                ]
-            )
-        if isinstance(value, dict):
-            validations.append(
-                _validate_node_child(node_by_type, node_by_hash, key, value)
-            )
-    return flatten(validations)
-
-
-def _validate_node_child(
-    node_by_type: dict, node_by_hash: dict, key: str, value: dict, index=None
-):
-    values = validate_node(node_by_type, node_by_hash)(value)
-    return list(
-        map(
-            lambda error: (
-                update_error_path(error, key, index)
-                if isinstance(error, dict)
-                else error
-            ),
-            values,
-        )
-    )
-
-
-def validate_node(node_by_type: dict = {}, node_by_hash: dict = {}):
-    def validate(node: dict):
-        """
-        Validates a single Node.
-
-        Parameters
-        ----------
-        node : dict
-            The JSON-Node to validate.
-
-        Returns
-        -------
-        List
-            The list of errors/warnings for the node, which can be empty if no errors/warnings detected.
-        """
-        try:
-            ntype = (
-                node.get("type", node.get("@type") if VALIDATE_EXISTING_NODES else None)
-                if isinstance(node, dict)
-                else None
-            )
-            return (
-                []
-                if ntype is None
-                else list(
-                    filter(
-                        lambda v: v is not True,
-                        flatten(
-                            _validate_node_type(node_by_type, node_by_hash, ntype, node)
-                            + (
-                                []
-                                if node.get("aggregated")
-                                else _validate_node_children(
-                                    node_by_type, node_by_hash, node
-                                )
-                            )
-                        ),
-                    )
-                )
-            )
-        except Exception as e:
-            logger.error(
-                f"Error validating {ntype} with id '{node.get('id', node.get('@id'))}': {str(e)}"
-            )
-            raise e
-
-    return validate

package/hestia_earth/validation/validators/aggregated_cycle.py

@@ -1,32 +0,0 @@
-from hestia_earth.utils.model import find_primary_product
-
-from hestia_earth.validation.utils import _filter_list_errors, get_lookup_value
-
-
-def validate_linked_impactAssessment(cycle: dict, list_key: str = "inputs"):
-    primary_product = find_primary_product(cycle) or {}
-    input_term_ids = (
-        get_lookup_value(primary_product.get("term"), "aggregationInputTermIds") or ""
-    ).split(";")
-
-    def validate(values: tuple):
-        index, blank_node = values
-        is_aggregation_input = blank_node.get("term", {}).get("@id") in input_term_ids
-        linked_id = blank_node.get("impactAssessment", {}).get("@id", "")
-        is_valid = all([linked_id, "world" not in linked_id])
-        return (
-            not is_aggregation_input
-            or is_valid
-            or {
-                "level": "error",
-                "dataPath": f".{list_key}[{index}]{'.impactAssessment' if linked_id else ''}",
-                "message": "must be linked to a verified country-level Impact Assessment",
-                "params": {"expected": blank_node.get("country"), "current": linked_id},
-            }
-        )
-
-    return (
-        _filter_list_errors(map(validate, enumerate(cycle.get(list_key, []))))
-        if input_term_ids
-        else True
-    )

package/hestia_earth/validation/validators/aggregated_shared.py

@@ -1,37 +0,0 @@
-def validate_quality_score_min(node: dict, max_diff: int = 2):
-    node_id = node.get("@id", node.get("id", ""))
-    key = "aggregatedQualityScore"
-    value = node.get(key, 0)
-    max_value = node.get(key + "Max", 0)
-    min_value = max_value - max_diff
-    # ignore nodes that are organic or irrigated as they should not block the upload of other nodes
-    level = (
-        "warning" if any(["organic" in node_id, "irrigated" in node_id]) else "error"
-    )
-    return value >= min_value or {
-        "level": level,
-        "dataPath": f".{key}",
-        "message": "must be at least equal to the minimum value",
-        "params": {
-            "expected": min_value,
-            "current": value,
-            "min": min_value,
-            "max": max_value,
-        },
-    }
-
-
-def validate_id(node: dict):
-    # make sure the ID contains a product ID, region ID, start and end year
-    node_id = node.get("id", "")
-    # there should be 4 different components to the id
-    node_id_length = len(node_id.split("-"))
-    return (
-        not node_id
-        or node_id_length >= 4
-        or {
-            "level": "error",
-            "dataPath": ".id",
-            "message": "aggregation id must contain a product, region, start, and end year",
-        }
-    )