@hestia-earth/data-validation 0.37.7

package/hestia_earth/validation/terms.py

@@ -0,0 +1,88 @@
+import os
+import json
+from enum import Enum
+from hestia_earth.schema import SchemaType, TermTermType
+from hestia_earth.utils.api import search
+
+CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
+RESULTS_PATH = os.path.join(CURRENT_DIR, "search-results.json")
+
+_CACHE = {}
+
+
+def _load_results(filepath: str) -> dict:
+    with open(filepath) as f:
+        return json.load(f)
+
+
+class TERMS_QUERY(Enum):
+    FUEL = TermTermType.FUEL.value
+    CROP_RESIDUE = TermTermType.CROPRESIDUE.value
+    MODEL = TermTermType.MODEL.value
+    FORAGE = TermTermType.FORAGE.value
+    RICE = "rice"
+
+
+_terms_query = {
+    TERMS_QUERY.FUEL: {
+        "should": [
+            {
+                "bool": {
+                    "must": [{"match": {"termType": TermTermType.FUEL.value}}],
+                    "should": [
+                        {"match": {"name": "gasoline"}},
+                        {"match": {"name": "petrol"}},
+                        {"match": {"name": "diesel"}},
+                    ],
+                    "minimum_should_match": 1,
+                }
+            }
+        ],
+        "minimum_should_match": 1,
+    },
+    TERMS_QUERY.CROP_RESIDUE: {
+        "should": [{"match": {"termType": TermTermType.CROPRESIDUE.value}}],
+        "minimum_should_match": 1,
+    },
+    TERMS_QUERY.MODEL: {
+        "should": [{"match": {"termType": TermTermType.MODEL.value}}],
+        "minimum_should_match": 1,
+    },
+    TERMS_QUERY.FORAGE: {
+        "should": [
+            {"match": {"termType.keyword": TermTermType.CROP.value}},
+            {"match": {"termType.keyword": TermTermType.FORAGE.value}},
+            {"match": {"name": "forage"}},
+        ],
+        "minimum_should_match": 2,
+    },
+    TERMS_QUERY.RICE: {
+        "should": [
+            {"match": {"termType.keyword": TermTermType.CROP.value}},
+            {"match": {"name": "rice"}},
+        ],
+        "minimum_should_match": 2,
+    },
+}
+
+
+def _exec_query(query: dict) -> list[str]:
+    terms = search(
+        {"bool": {"must": [{"match": {"@type": SchemaType.TERM.value}}]} | query},
+        limit=10000,
+    )
+    return list(map(lambda n: n["@id"], terms))
+
+
+def get_terms(query: TERMS_QUERY):
+    return _CACHE.get(query.value) or _exec_query(_terms_query[query])
+
+
+def get_all_terms():
+    return {key.value: _exec_query(value) for key, value in _terms_query.items()}
+
+
+def enable_mock(filepath: str):
+    global _CACHE  # noqa: F824
+    _CACHE = _load_results(filepath)
+    return _CACHE

package/hestia_earth/validation/utils.py

@@ -0,0 +1,444 @@
+import json
+from typing import List
+from functools import reduce, lru_cache
+from datetime import datetime, timedelta
+from hestia_earth.schema import NodeType, TermTermType, UNIQUENESS_FIELDS
+from hestia_earth.utils.api import download_hestia
+from hestia_earth.utils.lookup import download_lookup, get_table_value
+from hestia_earth.utils.tools import (
+    list_average,
+    safe_parse_date,
+    safe_parse_float,
+    non_empty_list,
+    is_number,
+    is_boolean,
+    get_dict_key,
+    flatten,
+)
+from hestia_earth.utils.model import filter_list_term_type, find_primary_product
+
+ANIMAL_TERM_TYPES = [TermTermType.LIVEANIMAL, TermTermType.LIVEAQUATICSPECIES]
+
+
+@lru_cache()
+def _get_term_lookup_value(term_id: str, term_type: str, column: str):
+    lookup = download_lookup(f"{term_type}.csv", keep_in_memory=True)
+    return get_table_value(lookup, "term.id", term_id, column)
+
+
+def get_lookup_value(lookup_term: dict, column: str):
+    value = (
+        _get_term_lookup_value(
+            lookup_term.get("@id"), lookup_term.get("termType"), column
+        )
+        if lookup_term
+        else None
+    )
+    return value
+
+
+def _next_error(values: list):
+    return next((x for x in values if x is not True), True)
+
+
+def _filter_list_errors(values: list, return_single=True):
+    values = list(filter(lambda x: x is not True, values))
+    return (
+        True
+        if return_single and len(values) == 0
+        else (values[0] if return_single and len(values) == 1 else values)
+    )
+
+
+def _flatten_errors(errors):
+    errors_list = [
+        [] if isinstance(v, bool) else v if isinstance(v, list) else [v] for v in errors
+    ]
+    return flatten(errors_list)
+
+
+def _list_except_item(values: list, item):
+    try:
+        idx = values.index(item)
+        return values[:idx] + values[idx + 1 :]
+    except ValueError:
+        return values
+
+
+def update_error_path(error: dict, key: str, index=None):
+    path = (
+        f".{key}[{index}]{error.get('dataPath')}"
+        if index is not None
+        else f".{key}{error.get('dataPath')}"
+    )
+    return error | {"dataPath": path}
+
+
+def _safe_cast(val, to_type, default=None):
+    try:
+        return to_type(val)
+    except (ValueError, TypeError):
+        return default
+
+
+def hash_dict(value: dict):
+    return json.dumps(value, sort_keys=True)
+
+
+def is_same_dict(a: dict, b: dict):
+    return hash_dict(a) == hash_dict(b)
+
+
+def _dict_without_key(a: dict, key: str):
+    no_key = a.copy()
+    if key in no_key:
+        no_key.pop(key)
+    return no_key
+
+
+_GROUP_NODE_TYPES = [
+    NodeType.CYCLE.value,
+    NodeType.IMPACTASSESSMENT.value,
+    NodeType.SITE.value,
+    NodeType.SOURCE.value,
+]
+
+
+def _is_node(value):
+    return (
+        isinstance(value, dict)
+        and (value.get("type") or value.get("@type")) in _GROUP_NODE_TYPES
+        and (value.get("id") or value.get("@id"))
+    )
+
+
+def _nodes_grouper(nodes: List[dict], grouping_func):
+    def group(groups: dict, node: dict):
+        if _is_node(node):
+            id = node.get("id") or node.get("@id")
+            type = node.get("type") or node.get("@type")
+            grouping_func(groups, node, type, id)
+        return groups
+
+    # check for nested nodes
+    nested_nodes = flatten(
+        [
+            v
+            for node in nodes
+            for v in node.values()
+            if _is_node(v) or (isinstance(v, list) and len(v) > 0 and _is_node(v[0]))
+        ]
+    )
+    data = reduce(group, nested_nodes, {})
+    return reduce(group, nodes, data)
+
+
+def _group_nodes(nodes: List[dict]):
+    def group_by(groups: dict, node: dict, type: str, id: str):
+        groups[type] = groups.get(type, {})
+        groups[type][id] = node
+
+    return _nodes_grouper(nodes, group_by)
+
+
+def _hash_nodes(nodes: List[dict]):
+    def group_by(groups: dict, node: dict, type: str, id: str):
+        # store the hash of the node without the `id` for uniqueness check
+        key = hash_dict(_dict_without_key(node, "id"))
+        groups[key] = groups.get(key, []) + [node]
+
+    return _nodes_grouper(nodes, group_by)
+
+
+def _list_sum(values: list, prop: str):
+    return sum(map(lambda v: _safe_cast(v.get(prop, 0), float, 0.0), values))
+
+
+def list_sum_terms(values: list, term_ids=[], default=None):
+    average_values = non_empty_list(
+        [
+            _value_average(node, default=default)
+            for node in values
+            if node.get("term", {}).get("@id") in term_ids
+        ]
+    )
+    return sum(average_values) if average_values else None
+
+
+def _compare_values(x, y):
+    return (
+        next((True for item in x if item in y), False)
+        if isinstance(x, list) and isinstance(y, list)
+        else x == y
+    )
+
+
+def _same_properties(value: dict, props: List[str]):
+    def identical(test: dict):
+        same_values = list(
+            filter(
+                lambda x: _compare_values(
+                    get_dict_key(value, x), get_dict_key(test, x)
+                ),
+                props,
+            )
+        )
+        return test if len(same_values) == len(props) else None
+
+    return identical
+
+
+def _value_average(node: dict, default=0, key="value"):
+    try:
+        value = node.get(key)
+        return (
+            list_average(value, default)
+            if isinstance(value, list)
+            else (value or default)
+        )
+    except Exception:
+        return default
+
+
+def term_id_prefix(term_id: str):
+    return term_id.split("Kg")[0]
+
+
+def _download_linked_node(node: dict):
+    data = (
+        download_hestia(node.get("@id"), node.get("@type"))
+        if node.get("@id") and node.get("@type")
+        else None
+    )
+    return data if (data or {}).get("@id") == node.get("@id") else None
+
+
+def find_linked_node(node_map: dict, node: dict):
+    """
+    Find the Node by type and id in the list of nodes.
+    """
+    return node_map.get(node.get("type"), {}).get(
+        node.get("id") or node.get("@id")
+    ) or _download_linked_node(node)
+
+
+def _value_as_array(data: dict, key: str):
+    value = data.get(key)
+    return value if isinstance(value, list) else non_empty_list([value])
+
+
+def find_related_nodes(
+    node_map: dict, node: dict, related_key: str, related_type: NodeType
+):
+    """
+    Find all nodes related to the same node via a key.
+    Example: find all Cycles related to a Site via the key "site".
+
+    Parameters
+    ----------
+    node_map : dict
+        The list of all nodes to do cross-validation, grouped by `type` and `id`.
+    node : dict
+        The node the other nodes should be related to.
+    related_key : str
+        How the other nodes are related to the `node`.
+    related_type : NodeType
+        The type of the related nodes.
+
+    Returns
+    -------
+    List[dict]
+        The list of nodes related to the `node`.
+    """
+    node_id = node.get("@id", node.get("id"))
+    nodes = node_map.get(related_type.value, {}).values()
+    return list(
+        {
+            n.get("@id", n.get("id")): n
+            for n in nodes
+            for related_node in _value_as_array(n, related_key)
+            if (related_node.get("@id", related_node.get("id")) == node_id)
+        }.values()
+    )
+
+
+def _is_before_today(date: str):
+    return safe_parse_date(date).date() <= datetime.now().date()
+
+
+def _node_year(node: dict):
+    date = node.get("endDate", node.get("startDate"))
+    date = safe_parse_date(date) if date else None
+    return date.year if date else None
+
+
+def is_live_animal_cycle(cycle: dict):
+    blank_nodes = cycle.get("animals", []) + cycle.get("products", [])
+    animals = filter_list_term_type(blank_nodes, ANIMAL_TERM_TYPES)
+    return len(animals) > 0
+
+
+def contains_grazing_animals(cycle: dict):
+    blank_nodes = cycle.get("animals", []) + cycle.get("products", [])
+    animals = filter_list_term_type(blank_nodes, ANIMAL_TERM_TYPES)
+    return any(
+        [v for v in animals if get_lookup_value(v.get("term", {}), "isGrazingAnimal")]
+    )
+
+
+def _match_list_el(source: list, dest: list, key: str):
+    src_values = non_empty_list([get_dict_key(x, key) for x in source])
+    dest_values = non_empty_list([get_dict_key(x, key) for x in dest])
+    return sorted(src_values) == sorted(dest_values)
+
+
+def _match_el(source: dict, dest: dict, fields: list):
+    def match(key: str):
+        keys = key.split(".")
+        is_list = len(keys) >= 2 and (
+            isinstance(get_dict_key(source, keys[0]), list)
+            or isinstance(get_dict_key(dest, keys[0]), list)
+        )
+        return (
+            _match_list_el(
+                get_dict_key(source, keys[0]) or [],
+                get_dict_key(dest, keys[0]) or [],
+                ".".join(keys[1:]),
+            )
+            if is_list
+            else get_dict_key(source, key) == get_dict_key(dest, key)
+        )
+
+    return all(map(match, fields))
+
+
+def find_by_unique_product(node: dict, product: dict, list_key: str = "products"):
+    """
+    Fallback to finding a product with unique keys if a single product has the same `term.@id`.
+    """
+    products = node.get(list_key, [])
+    products = [p for p in products if _match_el(p, product, ["term.@id"])]
+    return products[0] if len(products) == 1 else None
+
+
+def find_by_product(node: dict, product: dict, list_key: str = "products"):
+    keys = UNIQUENESS_FIELDS.get(node.get("type", node.get("@type")), {}).get(
+        list_key, ["term.@id"]
+    )
+    products = node.get(list_key, [])
+    return next((p for p in products if _match_el(p, product, keys)), None)
+
+
+def is_same_product(p1: dict, p2: dict):
+    return find_by_product({"type": NodeType.CYCLE.value, "products": [p1]}, p2)
+
+
+def _formatDepth(depth: str):
+    # handle float values
+    return str(int(depth)) if is_number(depth) else ""
+
+
+def blank_node_properties_group(blank_node: dict):
+    def property_group(property: dict):
+        return get_lookup_value(property.get("term", {}), "blankNodesGroup")
+
+    properties = blank_node.get("properties", [])
+    return "-".join(non_empty_list(map(property_group, properties)))
+
+
+def _blank_node_sum_groups(blank_node: dict, allow_sum_100: bool = True):
+    term = blank_node.get("term", {})
+    sum_below_100_group = get_lookup_value(term, "sumMax100Group")
+    sum_equal_100_group = get_lookup_value(term, "sumIs100Group")
+
+    return {
+        "sumMax100Group": sum_below_100_group
+        or (sum_equal_100_group if not allow_sum_100 else None),
+        "sumIs100Group": sum_equal_100_group if allow_sum_100 else None,
+    }
+
+
+def group_blank_nodes(nodes: list, by_sum: bool = True):
+    """
+    Group a list of blank nodes using:
+    - the `depthUpper`, `depthLower`, `startDate`, `endDate`, `dates`
+    - the lookup group `sumMax100Group` or `sumIs100Group` if specified
+    - the lookup group `blankNodesGroup` on properties if any
+
+    Parameters
+    ----------
+    nodes : list
+        List of blank nodes with their index.
+    by_sum : bool
+        Group blank nodes using the key to sum to 100% (`sumMax100Group` and `sumIs100Group`).
+    """
+
+    def group_by(group: dict, values: tuple):
+        index, blank_node = values
+        properties_group = blank_node_properties_group(blank_node)
+        # note: grouping of `properties` disables the grouping == 100
+        sum_groups = _blank_node_sum_groups(
+            blank_node, allow_sum_100=not properties_group
+        )
+        keys = non_empty_list(
+            [
+                _formatDepth(blank_node.get("depthUpper", "")),
+                _formatDepth(blank_node.get("depthLower", "")),
+                blank_node.get("startDate"),
+                blank_node.get("endDate"),
+                "-".join(blank_node.get("dates") or []),
+                properties_group,
+            ]
+            + (list(sum_groups.values()) if by_sum else [])
+        )
+        key = "-".join(keys) if len(keys) > 0 else "default"
+
+        if not by_sum or all(
+            [blank_node.get("value", []), any(list(sum_groups.values()))]
+        ):
+            group[key] = group.get(key, []) + [
+                {"index": index, "node": blank_node} | (sum_groups if by_sum else {})
+            ]
+
+        return group
+
+    return reduce(group_by, nodes, {})
+
+
+def is_permanent_crop(cycle: dict):
+    product = find_primary_product(cycle) or {}
+    return (
+        get_lookup_value(product.get("term", {}), "cropGroupingFAO")
+        == "Permanent crops"
+    )
+
+
+def term_valueType(term: dict):
+    return get_lookup_value(term, "valueType")
+
+
+VALUE_TYPE_MATCH = {"number": is_number, "boolean": is_boolean}
+
+
+def match_value_type(value_type: str, value):
+    values = non_empty_list(value if isinstance(value, list) else [value])
+    return all([VALUE_TYPE_MATCH.get(value_type, lambda _: True)(v) for v in values])
+
+
+def cycle_start_date(cycle: dict) -> datetime:
+    product = find_primary_product(cycle)
+    max_cycleDuration = (
+        cycle.get("cycleDuration")
+        or safe_parse_float(
+            value=get_lookup_value(product["term"], "maximumCycleDuration"), default=0
+        )
+        if product
+        else 0
+    )
+    return (
+        safe_parse_date(cycle.get("startDate"))
+        if cycle.get("startDate")
+        else (
+            safe_parse_date(cycle.get("endDate")) - (timedelta(days=max_cycleDuration))
+        )
+    )

package/hestia_earth/validation/validators/__init__.py

@@ -0,0 +1,141 @@
+import os
+from hestia_earth.schema import SchemaType
+from hestia_earth.utils.tools import flatten
+
+from ..log import logger
+from ..utils import update_error_path
+from .shared import validate_empty_fields, validate_nodes_duplicates
+from .cycle import validate_cycle
+from .impact_assessment import validate_impact_assessment
+from .organisation import validate_organisation
+from .site import validate_site
+from .source import validate_source
+
+# disable validation based on `@type`
+VALIDATE_EXISTING_NODES = os.getenv("VALIDATE_EXISTING_NODES", "false") == "true"
+VALIDATE_TYPE = {
+    SchemaType.CYCLE.value: lambda n, nodes: validate_cycle(n, nodes),
+    SchemaType.IMPACTASSESSMENT.value: lambda n, nodes: validate_impact_assessment(
+        n, nodes
+    ),
+    SchemaType.ORGANISATION.value: lambda n, nodes: validate_organisation(n, nodes),
+    SchemaType.SITE.value: lambda n, nodes: validate_site(n, nodes),
+    SchemaType.SOURCE.value: lambda n, nodes: validate_source(n, nodes),
+}
+SKIP_VALIDATE_DUPLICATES = [
+    SchemaType.ACTOR.value,
+    SchemaType.IMPACTASSESSMENT.value,
+    SchemaType.TERM.value,
+]
+
+
+def _has_keys(node: dict):
+    # ignore some keys that are automatically added to nested objects
+    ignore_keys = ["type", "id", "@type", "@id", "name", "description", "siteType"]
+    node_keys = [k for k in list(node.keys()) if k not in ignore_keys]
+    return len(node_keys) > 0
+
+
+def _should_run(ntype: str, node: dict):
+    return all([ntype in VALIDATE_TYPE, _has_keys(node)])
+
+
+def _validate_node_type(node_by_type: dict, node_by_hash: dict, ntype: str, node: dict):
+    should_run = _should_run(ntype, node)
+    if should_run:
+        logger.debug(
+            "Run validation on: type=%s, id=%s", ntype, node.get("id", node.get("@id"))
+        )
+    validator = VALIDATE_TYPE.get(ntype)
+    validations = validator(node, node_by_type) if should_run else []
+    return (
+        validations
+        + validate_empty_fields(node)
+        + (
+            []
+            if any([ntype in SKIP_VALIDATE_DUPLICATES, not should_run])
+            else validate_nodes_duplicates(node, node_by_hash)
+        )
+    )
+
+
+def _validate_node_children(node_by_type: dict, node_by_hash: dict, node: dict):
+    validations = []
+    for key, value in node.items():
+        if isinstance(value, list):
+            validations.extend(
+                [
+                    _validate_node_child(node_by_type, node_by_hash, key, value, index)
+                    for index, value in enumerate(value)
+                ]
+            )
+        if isinstance(value, dict):
+            validations.append(
+                _validate_node_child(node_by_type, node_by_hash, key, value)
+            )
+    return flatten(validations)
+
+
+def _validate_node_child(
+    node_by_type: dict, node_by_hash: dict, key: str, value: dict, index=None
+):
+    values = validate_node(node_by_type, node_by_hash)(value)
+    return list(
+        map(
+            lambda error: (
+                update_error_path(error, key, index)
+                if isinstance(error, dict)
+                else error
+            ),
+            values,
+        )
+    )
+
+
+def validate_node(node_by_type: dict = {}, node_by_hash: dict = {}):
+    def validate(node: dict):
+        """
+        Validates a single Node.
+
+        Parameters
+        ----------
+        node : dict
+            The JSON-Node to validate.
+
+        Returns
+        -------
+        List
+            The list of errors/warnings for the node, which can be empty if no errors/warnings detected.
+        """
+        try:
+            ntype = (
+                node.get("type", node.get("@type") if VALIDATE_EXISTING_NODES else None)
+                if isinstance(node, dict)
+                else None
+            )
+            return (
+                []
+                if ntype is None
+                else list(
+                    filter(
+                        lambda v: v is not True,
+                        flatten(
+                            _validate_node_type(node_by_type, node_by_hash, ntype, node)
+                            + (
+                                []
+                                if node.get("aggregated")
+                                else _validate_node_children(
+                                    node_by_type, node_by_hash, node
+                                )
+                            )
+                        ),
+                    )
+                )
+            )
+        except Exception as e:
+            logger.error(
+                f"Error validating {ntype} with id '{node.get('id', node.get('@id'))}': {str(e)}"
+            )
+            raise e
+
+    return validate

package/hestia_earth/validation/validators/aggregated_cycle.py

@@ -0,0 +1,32 @@
+from hestia_earth.utils.model import find_primary_product
+
+from hestia_earth.validation.utils import _filter_list_errors, get_lookup_value
+
+
+def validate_linked_impactAssessment(cycle: dict, list_key: str = "inputs"):
+    primary_product = find_primary_product(cycle) or {}
+    input_term_ids = (
+        get_lookup_value(primary_product.get("term"), "aggregationInputTermIds") or ""
+    ).split(";")
+
+    def validate(values: tuple):
+        index, blank_node = values
+        is_aggregation_input = blank_node.get("term", {}).get("@id") in input_term_ids
+        linked_id = blank_node.get("impactAssessment", {}).get("@id", "")
+        is_valid = all([linked_id, "world" not in linked_id])
+        return (
+            not is_aggregation_input
+            or is_valid
+            or {
+                "level": "error",
+                "dataPath": f".{list_key}[{index}]{'.impactAssessment' if linked_id else ''}",
+                "message": "must be linked to a verified country-level Impact Assessment",
+                "params": {"expected": blank_node.get("country"), "current": linked_id},
+            }
+        )
+
+    return (
+        _filter_list_errors(map(validate, enumerate(cycle.get(list_key, []))))
+        if input_term_ids
+        else True
+    )