@hestia-earth/data-validation 0.37.7

package/hestia_earth/validation/validators/site.py

@@ -0,0 +1,312 @@
+"""
+Site validation
+
+Here is the list of validations running on a [Site](/schema/Site).
+"""
+
+import os
+import json
+import re
+from hestia_earth.schema import NodeType, SiteSiteType, TermTermType
+from hestia_earth.utils.tools import flatten, safe_parse_date, non_empty_list
+
+from hestia_earth.validation.gee import get_cached_data
+from hestia_earth.validation.utils import find_linked_node, find_related_nodes
+from .shared import (
+    validate_dates,
+    validate_list_dates,
+    validate_list_dates_format,
+    validate_list_min_below_max,
+    validate_list_min_max_lookup,
+    validate_list_dates_after,
+    validate_country_region,
+    validate_coordinates,
+    need_validate_coordinates,
+    validate_area,
+    need_validate_area,
+    validate_list_term_percent,
+    validate_linked_source_privacy,
+    validate_list_date_lt_today,
+    validate_date_lt_today,
+    validate_boundary_area,
+    validate_region_size,
+    need_validate_region_size,
+    validate_private_has_source,
+    validate_list_value_between_min_max,
+    validate_list_sum_100_percent,
+    validate_list_percent_requires_value,
+    validate_list_valueType,
+)
+from .infrastructure import validate_lifespan
+from .measurement import (
+    validate_soilTexture,
+    validate_depths,
+    validate_required_depths,
+    validate_term_unique,
+    validate_require_startDate_endDate,
+    validate_with_models,
+    validate_value_length,
+    validate_water_measurements,
+    validate_water_salinity,
+)
+from .property import validate_all as validate_properties
+from .management import validate_fallow_dates, validate_cycles_overlap
+
+_VALIDATE_SITE_LINKED_NODES = os.getenv("VALIDATE_SITE_LINKED_NODES", "true") == "true"
+
+
+INLAND_TYPES = [
+    SiteSiteType.CROPLAND.value,
+    SiteSiteType.PERMANENT_PASTURE.value,
+    SiteSiteType.RIVER_OR_STREAM.value,
+    SiteSiteType.LAKE.value,
+    SiteSiteType.ANIMAL_HOUSING.value,
+    SiteSiteType.AGRI_FOOD_PROCESSOR.value,
+    SiteSiteType.FOOD_RETAILER.value,
+    SiteSiteType.FOREST.value,
+    SiteSiteType.OTHER_NATURAL_VEGETATION.value,
+]
+
+SITE_TYPES_VALID_VALUES = {
+    SiteSiteType.CROPLAND.value: [25, 35, 36],
+    SiteSiteType.FOREST.value: [10, 20, 25],
+}
+MEASUREMENT_REQUIRES_VALUE_TERM_TYPES = [
+    TermTermType.SOILTEXTURE,
+    TermTermType.SOILTYPE,
+    TermTermType.USDASOILTYPE,
+]
+
+
+def validate_site_dates(site: dict):
+    return validate_dates(site) or {
+        "level": "error",
+        "dataPath": ".endDate",
+        "message": "must be greater than startDate",
+    }
+
+
+def validate_site_coordinates(site: dict):
+    return need_validate_coordinates(site) and site.get("siteType") in INLAND_TYPES
+
+
+def validate_siteType(site: dict):
+    site_type = site.get("siteType")
+    values = SITE_TYPES_VALID_VALUES.get(site_type, [])
+    values_str = ", ".join(map(lambda v: str(v), values))
+
+    def validate():
+        value = get_cached_data(site, "siteType", 2019)
+        return value in values
+
+    return (
+        len(values) == 0
+        or validate()
+        or {
+            "level": "warning",
+            "dataPath": ".siteType",
+            "message": " ".join(
+                [
+                    "The coordinates you have provided are not in a known",
+                    site_type,
+                    f"area according to the MODIS Land Cover classification (MCD12Q1.006, LCCS2, bands {values_str}).",
+                ]
+            ),
+        }
+    )
+
+
+def _parse_cycle_date(date: str, is_end_date: bool = False):
+    value = (
+        date
+        if len(date) == 10
+        else (
+            f"{date}-01"
+            if len(date) == 7
+            else (f"{date}-12-31" if is_end_date else f"{date}-01-01")
+        )
+    )
+    return safe_parse_date(value)
+
+
+def validate_cycle_dates(cycles: list):
+    # compute list of all days existing between start and end date of the cycles
+    values = [
+        (
+            cycle,
+            (
+                _parse_cycle_date(cycle.get("startDate"))
+                if cycle.get("startDate")
+                else None
+            ),
+            _parse_cycle_date(cycle.get("endDate"), is_end_date=True),
+        )
+        for cycle in cycles
+    ]
+
+    day = 60 * 60 * 24  # 1 day step range
+    days = flatten(
+        (
+            list(
+                range(int(start_date.timestamp()), int(end_date.timestamp()) + day, day)
+            )
+            if start_date
+            else [int(end_date.timestamp())]
+        )
+        for _cycle, start_date, end_date in values
+    )
+
+    seen = set()
+    duplicated_dates = [x for x in days if x in seen or seen.add(x)]
+    duplicated_cycles = [
+        cycle
+        for cycle, start_date, end_date in values
+        if any(
+            [
+                (
+                    int(start_date.timestamp()) <= date <= int(end_date.timestamp())
+                    if start_date
+                    else date == int(end_date.timestamp())
+                )
+                for date in duplicated_dates
+            ]
+        )
+    ]
+
+    return not len(duplicated_dates) or {
+        "level": "error",
+        "dataPath": "",
+        "message": "multiple cycles on the same site cannot overlap",
+        "params": {"ids": [c.get("@id", c.get("id")) for c in duplicated_cycles]},
+    }
+
+
+def _extract_linked_data(cycle: dict, node_map: dict = {}):
+    cycle_str = json.dumps(cycle)
+    match = re.search(r'"impactAssessment":[\s]{([^}]*)}', cycle_str)
+    nodes = [json.loads("{" + v + "}") for v in match.groups()] if match else []
+    linked_impact_asessments = non_empty_list(
+        [
+            find_linked_node(
+                node_map,
+                {
+                    "type": NodeType.IMPACTASSESSMENT.value,
+                    "id": n.get("@id", n.get("id")),
+                },
+            )
+            for n in nodes
+        ]
+    )
+    linked_cycles = [n.get("cycle", {}) for n in linked_impact_asessments]
+    linked_cycle_ids = set([n.get("@id", n.get("id")) for n in linked_cycles])
+    return (cycle.get("@id", cycle.get("id")), linked_cycle_ids)
+
+
+def validate_cycles_linked_ia(cycles: list, node_map: dict = {}):
+    linked_data = [_extract_linked_data(cycle, node_map) for cycle in cycles]
+
+    cycle_ids = set([n.get("@id", n.get("id")) for n in cycles])
+    incorrect_cycle_ids = set(
+        flatten(
+            [
+                [cycle_id] + list(linked_cycle_ids.intersection(cycle_ids))
+                for cycle_id, linked_cycle_ids in linked_data
+                if linked_cycle_ids.intersection(cycle_ids)
+            ]
+        )
+    )
+
+    return not incorrect_cycle_ids or {
+        "level": "error",
+        "dataPath": "",
+        "message": "cycles linked together cannot be added to the same site",
+        "params": {"ids": list(incorrect_cycle_ids)},
+    }
+
+
+def validate_site(site: dict, node_map: dict = {}):
+    is_aggregated = site.get("aggregated", False)
+    cycles = find_related_nodes(
+        node_map, site, related_key="site", related_type=NodeType.CYCLE
+    ) + find_related_nodes(
+        node_map, site, related_key="otherSites", related_type=NodeType.CYCLE
+    )
+    return [
+        validate_site_dates(site),
+        validate_date_lt_today(site, "startDate"),
+        validate_date_lt_today(site, "endDate"),
+        validate_linked_source_privacy(site, "defaultSource", node_map),
+        validate_private_has_source(site, "defaultSource"),
+        validate_siteType(site) if need_validate_coordinates(site) else True,
+        validate_country_region(site),
+        validate_coordinates(site) if validate_site_coordinates(site) else True,
+        validate_area(site) if need_validate_area(site) else True,
+        validate_boundary_area(site),
+        validate_region_size(site) if need_validate_region_size(site) else True,
+        validate_cycle_dates(cycles) if cycles and not is_aggregated else True,
+        not _VALIDATE_SITE_LINKED_NODES
+        or (validate_cycles_linked_ia(cycles, node_map) if cycles else True),
+    ] + flatten(
+        (
+            [
+                validate_list_dates(site, "infrastructure"),
+                validate_list_dates_format(site, "infrastructure"),
+                validate_list_date_lt_today(
+                    site, "infrastructure", ["startDate", "endDate"]
+                ),
+                validate_lifespan(site.get("infrastructure")),
+            ]
+            if "infrastructure" in site
+            else []
+        )
+        + (
+            [
+                validate_list_dates(site, "measurements"),
+                validate_list_dates_after(
+                    site, "startDate", "measurements", ["startDate", "endDate", "dates"]
+                ),
+                validate_list_dates_format(site, "measurements"),
+                validate_list_date_lt_today(
+                    site, "measurements", ["startDate", "endDate"]
+                ),
+                validate_list_min_below_max(site, "measurements"),
+                validate_list_value_between_min_max(site, "measurements"),
+                validate_list_min_max_lookup(site, "measurements", "value"),
+                validate_list_min_max_lookup(site, "measurements", "min"),
+                validate_list_min_max_lookup(site, "measurements", "max"),
+                validate_list_term_percent(site, "measurements"),
+                validate_list_sum_100_percent(site, "measurements"),
+                validate_list_percent_requires_value(
+                    site, "measurements", MEASUREMENT_REQUIRES_VALUE_TERM_TYPES
+                ),
+                validate_list_valueType(site, "measurements"),
+                validate_soilTexture(site.get("measurements")),
+                validate_depths(site.get("measurements")),
+                validate_required_depths(site, "measurements"),
+                validate_term_unique(site.get("measurements")),
+                validate_properties(site, "measurements"),
+                validate_require_startDate_endDate(site, "measurements"),
+                validate_with_models(site, "measurements"),
+                validate_value_length(site, "measurements"),
+                validate_water_measurements(site, "measurements"),
+                validate_water_salinity(site, "measurements"),
+            ]
+            if len(site.get("measurements", [])) > 0
+            else []
+        )
+        + (
+            [
+                validate_list_sum_100_percent(site, "management"),
+                validate_list_valueType(site, "management"),
+                validate_properties(site, "management"),
+                validate_fallow_dates(site, "management"),
+                (
+                    validate_cycles_overlap(site, cycles, "management")
+                    if cycles and not is_aggregated
+                    else True
+                ),
+            ]
+            if len(site.get("management", [])) > 0
+            else []
+        )
+    )

package/hestia_earth/validation/validators/source.py

@@ -0,0 +1,20 @@
+from .shared import validate_date_lt_today
+
+
+def validate_source(source: dict, node_map: dict = {}):
+    """
+    Validates a single `Source`.
+
+    Parameters
+    ----------
+    organisation : dict
+        The `Source` to validate.
+    node_map : dict
+        The list of all nodes to do cross-validation, grouped by `type` and `id`.
+
+    Returns
+    -------
+    List
+        The list of errors for the `Source`, which can be empty if no errors detected.
+    """
+    return [validate_date_lt_today(source, "bibliography.year")]

package/hestia_earth/validation/validators/transformation.py

@@ -0,0 +1,250 @@
+from hestia_earth.schema import TermTermType
+from hestia_earth.utils.api import download_hestia
+from hestia_earth.utils.model import find_term_match, filter_list_term_type
+from hestia_earth.utils.tools import flatten, list_sum
+
+from hestia_earth.validation.utils import (
+    _filter_list_errors,
+    update_error_path,
+    term_id_prefix,
+)
+from hestia_earth.validation.validators.shared import is_value_different
+from .practice import validate_excretaManagement
+
+
+def _previous_transformation(
+    cycle: dict, list_key: str, transformation: dict, index: int
+):
+    tr_id = transformation.get("previousTransformationId")
+    transformations = cycle.get(list_key, [])
+    # previous transformation must be before the current transformation index
+    return next(
+        (
+            transformations[i]
+            for i in reversed(range(0, min(index, len(transformations))))
+            if transformations[i].get("transformationId") == tr_id and i < index
+        ),
+        None,
+    )
+
+
+def _validate_previous_term(
+    cycle: dict, list_key: str, transformation: dict, index: int
+):
+    previous_transformation = _previous_transformation(
+        cycle, list_key, transformation, index
+    )
+    tr_id = transformation.get("previousTransformationId")
+    return (
+        not tr_id
+        or previous_transformation is not None
+        or {
+            "level": "error",
+            "dataPath": f".{list_key}[{index}].previousTransformationId",
+            "message": "must point to a previous transformation in the list",
+        }
+    )
+
+
+def _cycle_has_product(cycle: dict, input: dict):
+    term_id = input.get("term", {}).get("@id")
+    return find_term_match(cycle.get("products", []), term_id, None) is not None
+
+
+def _validate_previous_input(
+    cycle: dict, list_key: str, transformation: dict, index: int
+):
+    has_previous_transformation = (
+        transformation.get("previousTransformationId") is not None
+    )
+    inputs = transformation.get("inputs", [])
+
+    def validate_in_cycle():
+        return (
+            any([len(cycle.get("products", [])) == 0, len(inputs) == 0])
+            or any([_cycle_has_product(cycle, i) for i in inputs])
+            or {
+                "level": "error",
+                "dataPath": f".{list_key}[{index}]",
+                "message": "at least one Input must be a Product of the Cycle",
+            }
+        )
+
+    def validate_in_previous_transformation():
+        previous_transformation = _previous_transformation(
+            cycle, list_key, transformation, index
+        )
+        return (
+            not previous_transformation
+            or any(
+                [
+                    len(previous_transformation.get("products", [])) == 0,
+                    len(inputs) == 0,
+                ]
+            )
+            or any([_cycle_has_product(previous_transformation, i) for i in inputs])
+            or {
+                "level": "error",
+                "dataPath": f".{list_key}[{index}]",
+                "message": "at least one Input must be a Product of the previous Transformation",
+            }
+        )
+
+    return (
+        validate_in_previous_transformation()
+        if has_previous_transformation
+        else validate_in_cycle()
+    )
+
+
+def _validate_previous_product_value(
+    cycle: dict, list_key: str, transformation: dict, index: int
+):
+    share = transformation.get("transformedShare")
+    inputs = transformation.get("inputs", [])
+    previous_transformation = _previous_transformation(
+        cycle, list_key, transformation, index
+    )
+    products = (previous_transformation or cycle).get("products", [])
+
+    def validate_input(input_index: int):
+        input = list_sum(inputs[input_index].get("value", []), None)
+        term_id = inputs[input_index].get("term", {}).get("@id")
+        product = list_sum(find_term_match(products, term_id).get("value", []), None)
+        return (
+            any([not input, not product])
+            or not is_value_different(input, product * share / 100, 0.01)
+            or {
+                "level": "error",
+                "dataPath": f".transformations[{index}].inputs[{input_index}].value",
+                "message": "must be equal to previous product multiplied by the share",
+            }
+        )
+
+    return any([len(products) == 0, share is None]) or _filter_list_errors(
+        flatten(map(validate_input, range(len(inputs))))
+    )
+
+
+def validate_previous_transformation(cycle: dict, list_key: str = "transformations"):
+    def validate(values: tuple):
+        index, transformation = values
+        return _filter_list_errors(
+            map(
+                lambda func: func(cycle, list_key, transformation, index),
+                [
+                    _validate_previous_term,
+                    _validate_previous_input,
+                    _validate_previous_product_value,
+                ],
+            )
+        )
+
+    return _filter_list_errors(map(validate, enumerate(cycle.get(list_key, []))))
+
+
+def validate_transformation_excretaManagement(
+    cycle: dict, list_key: str = "transformations"
+):
+    def validate(values: tuple):
+        index, transformation = values
+        practices = transformation.get("practices", []) + [
+            {"term": transformation.get("term")}
+        ]
+        error = validate_excretaManagement(transformation, practices)
+        return error is True or update_error_path(error, list_key, index)
+
+    return _filter_list_errors(map(validate, enumerate(cycle.get(list_key, []))))
+
+
+def validate_linked_emission(cycle: dict, list_key: str = "transformations"):
+    emissions = cycle.get("emissions", [])
+
+    def validate_emission(transformation_index: int, transformation: dict):
+        def validate(values: tuple):
+            index, emission = values
+            term_id = emission.get("term", {}).get("@id")
+            same_emissions = list(
+                filter(lambda e: e.get("term", {}).get("@id") == term_id, emissions)
+            )
+            linked_emission = next(
+                (
+                    e
+                    for e in same_emissions
+                    if all(
+                        [
+                            e.get("transformation", {}).get("@id")
+                            == transformation.get("term", {}).get("@id")
+                        ]
+                    )
+                ),
+                None,
+            )
+            return (
+                len(same_emissions) == 0
+                or linked_emission is not None
+                or {
+                    "level": "warning",
+                    "dataPath": f".{list_key}[{transformation_index}].emissions[{index}]",
+                    "message": "should be linked to an emission in the Cycle",
+                    "params": {"term": emission.get("term", {})},
+                }
+            )
+
+        return validate
+
+    def validate(values: tuple):
+        index, transformation = values
+        return _filter_list_errors(
+            map(
+                validate_emission(index, transformation),
+                enumerate(transformation.get("emissions", [])),
+            )
+        )
+
+    return len(emissions) == 0 or _filter_list_errors(
+        flatten(map(validate, enumerate(cycle.get(list_key, []))))
+    )
+
+
+def _is_generic_excreta(term_id: str):
+    return len((download_hestia(term_id) or {}).get("subClassOf", [])) == 0
+
+
+def validate_excreta_inputs_products(transformations: list):
+    def validate_product(transformation_index: int, input_prefix_ids: list):
+        def validate(values: tuple):
+            index, product = values
+            term = product.get("term", {})
+            term_id = term.get("@id", "")
+            is_excreta = term.get("termType", "") == TermTermType.EXCRETA.value
+            return (
+                not is_excreta
+                or _is_generic_excreta(term_id)
+                or term_id_prefix(term_id) in input_prefix_ids
+                or {
+                    "level": "error",
+                    "dataPath": f".transformations[{transformation_index}].products[{index}]",
+                    "message": "must be included as an Input",
+                    "params": {"term": term, "expected": input_prefix_ids},
+                }
+            )
+
+        return validate
+
+    def validate(values: tuple):
+        index, transformation = values
+        excreta_inputs = filter_list_term_type(
+            transformation.get("inputs", []), TermTermType.EXCRETA
+        )
+        input_prefix_ids = list(
+            set([term_id_prefix(v.get("term", {}).get("@id")) for v in excreta_inputs])
+        )
+        return len(input_prefix_ids) == 0 or _filter_list_errors(
+            map(
+                validate_product(index, input_prefix_ids),
+                enumerate(transformation.get("products", [])),
+            )
+        )
+
+    return _filter_list_errors(flatten(map(validate, enumerate(transformations))))

package/hestia_earth/validation/version.py

@@ -0,0 +1 @@
+VERSION = "0.37.7"

package/layer/build.sh

@@ -0,0 +1,34 @@
+#!/bin/sh
+
+# exit when any command fails
+set -e
+
+export PKG_DIR="python"
+
+LIBRARY="validation"
+PKG_PATH="./layer/$PKG_DIR/lib/python3.12/site-packages/"
+
+rm -rf "./layer/${PKG_DIR}" && mkdir -p "./layer/${PKG_DIR}"
+
+PACKAGE_PATH="package.json"
+PACKAGE_VERSION=$(cat $PACKAGE_PATH \
+  | grep version \
+  | head -1 \
+  | awk -F: '{ print $2 }' \
+  | sed 's/[",]//g' \
+  | tr -d '[[:space:]]')
+
+# create requirements.txt
+cp requirements.txt layer/requirements.txt
+
+# remove packages included in layers
+sed -i '/hestia-earth-utils.*/d' layer/requirements.txt || sed -i '' '/hestia-earth-utils.*/d' layer/requirements.txt
+
+docker run --rm -v $(pwd):/var/task public.ecr.aws/sam/build-python3.12 pip install -r layer/requirements.txt -t $PKG_PATH
+
+# copy the current library
+cp -R "./hestia_earth/${LIBRARY}/" "${PKG_PATH}hestia_earth/${LIBRARY}"
+
+# Removing nonessential files
+find ${PKG_PATH} -type f -name '*.md' -delete
+rm -rf ${PKG_PATH}*.dist-info

package/layer/deploy.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# exit when any command fails
+set -e
+
+STAGE=${1:-"dev"}
+
+cd ./layer/
+
+rm -rf layer.zip
+zip -r layer.zip python
+
+aws lambda publish-layer-version \
+    --region $AWS_REGION \
+    --layer-name "hestia-$STAGE-python312-data-validation" \
+    --description "Data Validation running on python 3.12" \
+    --zip-file "fileb://layer.zip" \
+    --compatible-runtimes python3.12

package/package.json

@@ -0,0 +1,59 @@
+{
+  "name": "@hestia-earth/data-validation",
+  "version": "0.37.7",
+  "description": "HESTIA Data Validation library",
+  "main": "index.js",
+  "directories": {
+    "test": "tests"
+  },
+  "scripts": {
+    "build:module": "rm -rf dist && tsc -p tsconfig.dist.json",
+    "test": "hestia-validate-jsonld tests/fixtures",
+    "lint": "tsc --noEmit && eslint .",
+    "lint:fix": "npm run lint -- --fix",
+    "release": "standard-version -a",
+    "postrelease": "git push origin master --follow-tags"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+ssh://git@gitlab.com/hestia-earth/hestia-data-validation.git"
+  },
+  "keywords": [
+    "hestia",
+    "data",
+    "validaiton",
+    "error",
+    "warning"
+  ],
+  "author": "Guillaume Royer <guillaume@hestia.earth>",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://gitlab.com/hestia-earth/hestia-data-validation/issues"
+  },
+  "homepage": "https://gitlab.com/hestia-earth/hestia-data-validation#readme",
+  "devDependencies": {
+    "@commitlint/cli": "^17.8.1",
+    "@commitlint/config-conventional": "^17.8.1",
+    "@hestia-earth/eslint-config": "^0.1.1",
+    "@hestia-earth/json-schema": "^33.7.11",
+    "@hestia-earth/schema": "^33.7.11",
+    "@hestia-earth/schema-validation": "^33.7.11",
+    "@typescript-eslint/eslint-plugin": "^5.62.0",
+    "@typescript-eslint/parser": "^5.62.0",
+    "eslint": "^7.32.0",
+    "eslint-plugin-jsdoc": "^30.7.13",
+    "husky": "~4.3.8",
+    "standard-version": "^9.5.0",
+    "typescript": "^5.9.3"
+  },
+  "standard-version": {
+    "scripts": {
+      "postchangelog": "node scripts/update-package-version.js && git add hestia_earth/validation/version.py && git add requirements-ci.txt"
+    }
+  },
+  "husky": {
+    "hooks": {
+      "commit-msg": "commitlint -E HUSKY_GIT_PARAMS"
+    }
+  }
+}