@hestia-earth/data-validation 0.37.7 → 0.37.9

package/hestia_earth/validation/validators/site.py

@@ -1,312 +0,0 @@
-"""
-Site validation
-
-Here is the list of validations running on a [Site](/schema/Site).
-"""
-
-import os
-import json
-import re
-from hestia_earth.schema import NodeType, SiteSiteType, TermTermType
-from hestia_earth.utils.tools import flatten, safe_parse_date, non_empty_list
-
-from hestia_earth.validation.gee import get_cached_data
-from hestia_earth.validation.utils import find_linked_node, find_related_nodes
-from .shared import (
-    validate_dates,
-    validate_list_dates,
-    validate_list_dates_format,
-    validate_list_min_below_max,
-    validate_list_min_max_lookup,
-    validate_list_dates_after,
-    validate_country_region,
-    validate_coordinates,
-    need_validate_coordinates,
-    validate_area,
-    need_validate_area,
-    validate_list_term_percent,
-    validate_linked_source_privacy,
-    validate_list_date_lt_today,
-    validate_date_lt_today,
-    validate_boundary_area,
-    validate_region_size,
-    need_validate_region_size,
-    validate_private_has_source,
-    validate_list_value_between_min_max,
-    validate_list_sum_100_percent,
-    validate_list_percent_requires_value,
-    validate_list_valueType,
-)
-from .infrastructure import validate_lifespan
-from .measurement import (
-    validate_soilTexture,
-    validate_depths,
-    validate_required_depths,
-    validate_term_unique,
-    validate_require_startDate_endDate,
-    validate_with_models,
-    validate_value_length,
-    validate_water_measurements,
-    validate_water_salinity,
-)
-from .property import validate_all as validate_properties
-from .management import validate_fallow_dates, validate_cycles_overlap
-
-_VALIDATE_SITE_LINKED_NODES = os.getenv("VALIDATE_SITE_LINKED_NODES", "true") == "true"
-
-
-INLAND_TYPES = [
-    SiteSiteType.CROPLAND.value,
-    SiteSiteType.PERMANENT_PASTURE.value,
-    SiteSiteType.RIVER_OR_STREAM.value,
-    SiteSiteType.LAKE.value,
-    SiteSiteType.ANIMAL_HOUSING.value,
-    SiteSiteType.AGRI_FOOD_PROCESSOR.value,
-    SiteSiteType.FOOD_RETAILER.value,
-    SiteSiteType.FOREST.value,
-    SiteSiteType.OTHER_NATURAL_VEGETATION.value,
-]
-
-SITE_TYPES_VALID_VALUES = {
-    SiteSiteType.CROPLAND.value: [25, 35, 36],
-    SiteSiteType.FOREST.value: [10, 20, 25],
-}
-MEASUREMENT_REQUIRES_VALUE_TERM_TYPES = [
-    TermTermType.SOILTEXTURE,
-    TermTermType.SOILTYPE,
-    TermTermType.USDASOILTYPE,
-]
-
-
-def validate_site_dates(site: dict):
-    return validate_dates(site) or {
-        "level": "error",
-        "dataPath": ".endDate",
-        "message": "must be greater than startDate",
-    }
-
-
-def validate_site_coordinates(site: dict):
-    return need_validate_coordinates(site) and site.get("siteType") in INLAND_TYPES
-
-
-def validate_siteType(site: dict):
-    site_type = site.get("siteType")
-    values = SITE_TYPES_VALID_VALUES.get(site_type, [])
-    values_str = ", ".join(map(lambda v: str(v), values))
-
-    def validate():
-        value = get_cached_data(site, "siteType", 2019)
-        return value in values
-
-    return (
-        len(values) == 0
-        or validate()
-        or {
-            "level": "warning",
-            "dataPath": ".siteType",
-            "message": " ".join(
-                [
-                    "The coordinates you have provided are not in a known",
-                    site_type,
-                    f"area according to the MODIS Land Cover classification (MCD12Q1.006, LCCS2, bands {values_str}).",
-                ]
-            ),
-        }
-    )
-
-
-def _parse_cycle_date(date: str, is_end_date: bool = False):
-    value = (
-        date
-        if len(date) == 10
-        else (
-            f"{date}-01"
-            if len(date) == 7
-            else (f"{date}-12-31" if is_end_date else f"{date}-01-01")
-        )
-    )
-    return safe_parse_date(value)
-
-
-def validate_cycle_dates(cycles: list):
-    # compute list of all days existing between start and end date of the cycles
-    values = [
-        (
-            cycle,
-            (
-                _parse_cycle_date(cycle.get("startDate"))
-                if cycle.get("startDate")
-                else None
-            ),
-            _parse_cycle_date(cycle.get("endDate"), is_end_date=True),
-        )
-        for cycle in cycles
-    ]
-
-    day = 60 * 60 * 24  # 1 day step range
-    days = flatten(
-        (
-            list(
-                range(int(start_date.timestamp()), int(end_date.timestamp()) + day, day)
-            )
-            if start_date
-            else [int(end_date.timestamp())]
-        )
-        for _cycle, start_date, end_date in values
-    )
-
-    seen = set()
-    duplicated_dates = [x for x in days if x in seen or seen.add(x)]
-    duplicated_cycles = [
-        cycle
-        for cycle, start_date, end_date in values
-        if any(
-            [
-                (
-                    int(start_date.timestamp()) <= date <= int(end_date.timestamp())
-                    if start_date
-                    else date == int(end_date.timestamp())
-                )
-                for date in duplicated_dates
-            ]
-        )
-    ]
-
-    return not len(duplicated_dates) or {
-        "level": "error",
-        "dataPath": "",
-        "message": "multiple cycles on the same site cannot overlap",
-        "params": {"ids": [c.get("@id", c.get("id")) for c in duplicated_cycles]},
-    }
-
-
-def _extract_linked_data(cycle: dict, node_map: dict = {}):
-    cycle_str = json.dumps(cycle)
-    match = re.search(r'"impactAssessment":[\s]{([^}]*)}', cycle_str)
-    nodes = [json.loads("{" + v + "}") for v in match.groups()] if match else []
-    linked_impact_asessments = non_empty_list(
-        [
-            find_linked_node(
-                node_map,
-                {
-                    "type": NodeType.IMPACTASSESSMENT.value,
-                    "id": n.get("@id", n.get("id")),
-                },
-            )
-            for n in nodes
-        ]
-    )
-    linked_cycles = [n.get("cycle", {}) for n in linked_impact_asessments]
-    linked_cycle_ids = set([n.get("@id", n.get("id")) for n in linked_cycles])
-    return (cycle.get("@id", cycle.get("id")), linked_cycle_ids)
-
-
-def validate_cycles_linked_ia(cycles: list, node_map: dict = {}):
-    linked_data = [_extract_linked_data(cycle, node_map) for cycle in cycles]
-
-    cycle_ids = set([n.get("@id", n.get("id")) for n in cycles])
-    incorrect_cycle_ids = set(
-        flatten(
-            [
-                [cycle_id] + list(linked_cycle_ids.intersection(cycle_ids))
-                for cycle_id, linked_cycle_ids in linked_data
-                if linked_cycle_ids.intersection(cycle_ids)
-            ]
-        )
-    )
-
-    return not incorrect_cycle_ids or {
-        "level": "error",
-        "dataPath": "",
-        "message": "cycles linked together cannot be added to the same site",
-        "params": {"ids": list(incorrect_cycle_ids)},
-    }
-
-
-def validate_site(site: dict, node_map: dict = {}):
-    is_aggregated = site.get("aggregated", False)
-    cycles = find_related_nodes(
-        node_map, site, related_key="site", related_type=NodeType.CYCLE
-    ) + find_related_nodes(
-        node_map, site, related_key="otherSites", related_type=NodeType.CYCLE
-    )
-    return [
-        validate_site_dates(site),
-        validate_date_lt_today(site, "startDate"),
-        validate_date_lt_today(site, "endDate"),
-        validate_linked_source_privacy(site, "defaultSource", node_map),
-        validate_private_has_source(site, "defaultSource"),
-        validate_siteType(site) if need_validate_coordinates(site) else True,
-        validate_country_region(site),
-        validate_coordinates(site) if validate_site_coordinates(site) else True,
-        validate_area(site) if need_validate_area(site) else True,
-        validate_boundary_area(site),
-        validate_region_size(site) if need_validate_region_size(site) else True,
-        validate_cycle_dates(cycles) if cycles and not is_aggregated else True,
-        not _VALIDATE_SITE_LINKED_NODES
-        or (validate_cycles_linked_ia(cycles, node_map) if cycles else True),
-    ] + flatten(
-        (
-            [
-                validate_list_dates(site, "infrastructure"),
-                validate_list_dates_format(site, "infrastructure"),
-                validate_list_date_lt_today(
-                    site, "infrastructure", ["startDate", "endDate"]
-                ),
-                validate_lifespan(site.get("infrastructure")),
-            ]
-            if "infrastructure" in site
-            else []
-        )
-        + (
-            [
-                validate_list_dates(site, "measurements"),
-                validate_list_dates_after(
-                    site, "startDate", "measurements", ["startDate", "endDate", "dates"]
-                ),
-                validate_list_dates_format(site, "measurements"),
-                validate_list_date_lt_today(
-                    site, "measurements", ["startDate", "endDate"]
-                ),
-                validate_list_min_below_max(site, "measurements"),
-                validate_list_value_between_min_max(site, "measurements"),
-                validate_list_min_max_lookup(site, "measurements", "value"),
-                validate_list_min_max_lookup(site, "measurements", "min"),
-                validate_list_min_max_lookup(site, "measurements", "max"),
-                validate_list_term_percent(site, "measurements"),
-                validate_list_sum_100_percent(site, "measurements"),
-                validate_list_percent_requires_value(
-                    site, "measurements", MEASUREMENT_REQUIRES_VALUE_TERM_TYPES
-                ),
-                validate_list_valueType(site, "measurements"),
-                validate_soilTexture(site.get("measurements")),
-                validate_depths(site.get("measurements")),
-                validate_required_depths(site, "measurements"),
-                validate_term_unique(site.get("measurements")),
-                validate_properties(site, "measurements"),
-                validate_require_startDate_endDate(site, "measurements"),
-                validate_with_models(site, "measurements"),
-                validate_value_length(site, "measurements"),
-                validate_water_measurements(site, "measurements"),
-                validate_water_salinity(site, "measurements"),
-            ]
-            if len(site.get("measurements", [])) > 0
-            else []
-        )
-        + (
-            [
-                validate_list_sum_100_percent(site, "management"),
-                validate_list_valueType(site, "management"),
-                validate_properties(site, "management"),
-                validate_fallow_dates(site, "management"),
-                (
-                    validate_cycles_overlap(site, cycles, "management")
-                    if cycles and not is_aggregated
-                    else True
-                ),
-            ]
-            if len(site.get("management", [])) > 0
-            else []
-        )
-    )

package/hestia_earth/validation/validators/source.py

@@ -1,20 +0,0 @@
-from .shared import validate_date_lt_today
-
-
-def validate_source(source: dict, node_map: dict = {}):
-    """
-    Validates a single `Source`.
-
-    Parameters
-    ----------
-    organisation : dict
-        The `Source` to validate.
-    node_map : dict
-        The list of all nodes to do cross-validation, grouped by `type` and `id`.
-
-    Returns
-    -------
-    List
-        The list of errors for the `Source`, which can be empty if no errors detected.
-    """
-    return [validate_date_lt_today(source, "bibliography.year")]

package/hestia_earth/validation/validators/transformation.py

@@ -1,250 +0,0 @@
-from hestia_earth.schema import TermTermType
-from hestia_earth.utils.api import download_hestia
-from hestia_earth.utils.model import find_term_match, filter_list_term_type
-from hestia_earth.utils.tools import flatten, list_sum
-
-from hestia_earth.validation.utils import (
-    _filter_list_errors,
-    update_error_path,
-    term_id_prefix,
-)
-from hestia_earth.validation.validators.shared import is_value_different
-from .practice import validate_excretaManagement
-
-
-def _previous_transformation(
-    cycle: dict, list_key: str, transformation: dict, index: int
-):
-    tr_id = transformation.get("previousTransformationId")
-    transformations = cycle.get(list_key, [])
-    # previous transformation must be before the current transformation index
-    return next(
-        (
-            transformations[i]
-            for i in reversed(range(0, min(index, len(transformations))))
-            if transformations[i].get("transformationId") == tr_id and i < index
-        ),
-        None,
-    )
-
-
-def _validate_previous_term(
-    cycle: dict, list_key: str, transformation: dict, index: int
-):
-    previous_transformation = _previous_transformation(
-        cycle, list_key, transformation, index
-    )
-    tr_id = transformation.get("previousTransformationId")
-    return (
-        not tr_id
-        or previous_transformation is not None
-        or {
-            "level": "error",
-            "dataPath": f".{list_key}[{index}].previousTransformationId",
-            "message": "must point to a previous transformation in the list",
-        }
-    )
-
-
-def _cycle_has_product(cycle: dict, input: dict):
-    term_id = input.get("term", {}).get("@id")
-    return find_term_match(cycle.get("products", []), term_id, None) is not None
-
-
-def _validate_previous_input(
-    cycle: dict, list_key: str, transformation: dict, index: int
-):
-    has_previous_transformation = (
-        transformation.get("previousTransformationId") is not None
-    )
-    inputs = transformation.get("inputs", [])
-
-    def validate_in_cycle():
-        return (
-            any([len(cycle.get("products", [])) == 0, len(inputs) == 0])
-            or any([_cycle_has_product(cycle, i) for i in inputs])
-            or {
-                "level": "error",
-                "dataPath": f".{list_key}[{index}]",
-                "message": "at least one Input must be a Product of the Cycle",
-            }
-        )
-
-    def validate_in_previous_transformation():
-        previous_transformation = _previous_transformation(
-            cycle, list_key, transformation, index
-        )
-        return (
-            not previous_transformation
-            or any(
-                [
-                    len(previous_transformation.get("products", [])) == 0,
-                    len(inputs) == 0,
-                ]
-            )
-            or any([_cycle_has_product(previous_transformation, i) for i in inputs])
-            or {
-                "level": "error",
-                "dataPath": f".{list_key}[{index}]",
-                "message": "at least one Input must be a Product of the previous Transformation",
-            }
-        )
-
-    return (
-        validate_in_previous_transformation()
-        if has_previous_transformation
-        else validate_in_cycle()
-    )
-
-
-def _validate_previous_product_value(
-    cycle: dict, list_key: str, transformation: dict, index: int
-):
-    share = transformation.get("transformedShare")
-    inputs = transformation.get("inputs", [])
-    previous_transformation = _previous_transformation(
-        cycle, list_key, transformation, index
-    )
-    products = (previous_transformation or cycle).get("products", [])
-
-    def validate_input(input_index: int):
-        input = list_sum(inputs[input_index].get("value", []), None)
-        term_id = inputs[input_index].get("term", {}).get("@id")
-        product = list_sum(find_term_match(products, term_id).get("value", []), None)
-        return (
-            any([not input, not product])
-            or not is_value_different(input, product * share / 100, 0.01)
-            or {
-                "level": "error",
-                "dataPath": f".transformations[{index}].inputs[{input_index}].value",
-                "message": "must be equal to previous product multiplied by the share",
-            }
-        )
-
-    return any([len(products) == 0, share is None]) or _filter_list_errors(
-        flatten(map(validate_input, range(len(inputs))))
-    )
-
-
-def validate_previous_transformation(cycle: dict, list_key: str = "transformations"):
-    def validate(values: tuple):
-        index, transformation = values
-        return _filter_list_errors(
-            map(
-                lambda func: func(cycle, list_key, transformation, index),
-                [
-                    _validate_previous_term,
-                    _validate_previous_input,
-                    _validate_previous_product_value,
-                ],
-            )
-        )
-
-    return _filter_list_errors(map(validate, enumerate(cycle.get(list_key, []))))
-
-
-def validate_transformation_excretaManagement(
-    cycle: dict, list_key: str = "transformations"
-):
-    def validate(values: tuple):
-        index, transformation = values
-        practices = transformation.get("practices", []) + [
-            {"term": transformation.get("term")}
-        ]
-        error = validate_excretaManagement(transformation, practices)
-        return error is True or update_error_path(error, list_key, index)
-
-    return _filter_list_errors(map(validate, enumerate(cycle.get(list_key, []))))
-
-
-def validate_linked_emission(cycle: dict, list_key: str = "transformations"):
-    emissions = cycle.get("emissions", [])
-
-    def validate_emission(transformation_index: int, transformation: dict):
-        def validate(values: tuple):
-            index, emission = values
-            term_id = emission.get("term", {}).get("@id")
-            same_emissions = list(
-                filter(lambda e: e.get("term", {}).get("@id") == term_id, emissions)
-            )
-            linked_emission = next(
-                (
-                    e
-                    for e in same_emissions
-                    if all(
-                        [
-                            e.get("transformation", {}).get("@id")
-                            == transformation.get("term", {}).get("@id")
-                        ]
-                    )
-                ),
-                None,
-            )
-            return (
-                len(same_emissions) == 0
-                or linked_emission is not None
-                or {
-                    "level": "warning",
-                    "dataPath": f".{list_key}[{transformation_index}].emissions[{index}]",
-                    "message": "should be linked to an emission in the Cycle",
-                    "params": {"term": emission.get("term", {})},
-                }
-            )
-
-        return validate
-
-    def validate(values: tuple):
-        index, transformation = values
-        return _filter_list_errors(
-            map(
-                validate_emission(index, transformation),
-                enumerate(transformation.get("emissions", [])),
-            )
-        )
-
-    return len(emissions) == 0 or _filter_list_errors(
-        flatten(map(validate, enumerate(cycle.get(list_key, []))))
-    )
-
-
-def _is_generic_excreta(term_id: str):
-    return len((download_hestia(term_id) or {}).get("subClassOf", [])) == 0
-
-
-def validate_excreta_inputs_products(transformations: list):
-    def validate_product(transformation_index: int, input_prefix_ids: list):
-        def validate(values: tuple):
-            index, product = values
-            term = product.get("term", {})
-            term_id = term.get("@id", "")
-            is_excreta = term.get("termType", "") == TermTermType.EXCRETA.value
-            return (
-                not is_excreta
-                or _is_generic_excreta(term_id)
-                or term_id_prefix(term_id) in input_prefix_ids
-                or {
-                    "level": "error",
-                    "dataPath": f".transformations[{transformation_index}].products[{index}]",
-                    "message": "must be included as an Input",
-                    "params": {"term": term, "expected": input_prefix_ids},
-                }
-            )
-
-        return validate
-
-    def validate(values: tuple):
-        index, transformation = values
-        excreta_inputs = filter_list_term_type(
-            transformation.get("inputs", []), TermTermType.EXCRETA
-        )
-        input_prefix_ids = list(
-            set([term_id_prefix(v.get("term", {}).get("@id")) for v in excreta_inputs])
-        )
-        return len(input_prefix_ids) == 0 or _filter_list_errors(
-            map(
-                validate_product(index, input_prefix_ids),
-                enumerate(transformation.get("products", [])),
-            )
-        )
-
-    return _filter_list_errors(flatten(map(validate, enumerate(transformations))))

package/hestia_earth/validation/version.py

@@ -1 +0,0 @@
-VERSION = "0.37.7"

package/layer/build.sh

@@ -1,34 +0,0 @@
-#!/bin/sh
-
-# exit when any command fails
-set -e
-
-export PKG_DIR="python"
-
-LIBRARY="validation"
-PKG_PATH="./layer/$PKG_DIR/lib/python3.12/site-packages/"
-
-rm -rf "./layer/${PKG_DIR}" && mkdir -p "./layer/${PKG_DIR}"
-
-PACKAGE_PATH="package.json"
-PACKAGE_VERSION=$(cat $PACKAGE_PATH \
-  | grep version \
-  | head -1 \
-  | awk -F: '{ print $2 }' \
-  | sed 's/[",]//g' \
-  | tr -d '[[:space:]]')
-
-# create requirements.txt
-cp requirements.txt layer/requirements.txt
-
-# remove packages included in layers
-sed -i '/hestia-earth-utils.*/d' layer/requirements.txt || sed -i '' '/hestia-earth-utils.*/d' layer/requirements.txt
-
-docker run --rm -v $(pwd):/var/task public.ecr.aws/sam/build-python3.12 pip install -r layer/requirements.txt -t $PKG_PATH
-
-# copy the current library
-cp -R "./hestia_earth/${LIBRARY}/" "${PKG_PATH}hestia_earth/${LIBRARY}"
-
-# Removing nonessential files
-find ${PKG_PATH} -type f -name '*.md' -delete
-rm -rf ${PKG_PATH}*.dist-info

package/layer/deploy.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-# exit when any command fails
-set -e
-
-STAGE=${1:-"dev"}
-
-cd ./layer/
-
-rm -rf layer.zip
-zip -r layer.zip python
-
-aws lambda publish-layer-version \
-    --region $AWS_REGION \
-    --layer-name "hestia-$STAGE-python312-data-validation" \
-    --description "Data Validation running on python 3.12" \
-    --zip-file "fileb://layer.zip" \
-    --compatible-runtimes python3.12

package/release.sh

@@ -1,11 +0,0 @@
-#!/bin/sh
-
-FROM_BRANCH="develop"
-TO_BRANCH="master"
-
-git checkout $TO_BRANCH
-git merge $FROM_BRANCH
-npm run release
-git checkout $FROM_BRANCH
-git merge $TO_BRANCH
-git push

package/requirements-ci.txt

@@ -1,6 +0,0 @@
--r requirements.txt
-boto3
-botocore
-hestia-earth-models>=0.75.1
-hestia-earth-earth-engine>=0.6.0
-hestia-earth-distribution>=0.3.1

package/requirements-test.txt

@@ -1,4 +0,0 @@
--r requirements-ci.txt
-pytest-cov
-pytest-xdist
-coverage

package/requirements.txt

@@ -1,2 +0,0 @@
-hestia-earth-schema>=35.0.1
-hestia-earth-utils>=0.16.9

package/run-docker-test.sh

@@ -1,7 +0,0 @@
-#!/bin/sh
-docker build -t hestia-data-validation:test -f tests/Dockerfile .
-
-docker run --rm \
-  --env-file .env \
-  -v ${PWD}:/app \
-  hestia-data-validation:test "$@"

package/run-docker.sh

@@ -1,9 +0,0 @@
-#!/bin/sh
-
-docker build --progress=plain -t hestia-data-validation:latest .
-
-docker run --rm \
-  --name hestia-data-validation \
-  -v ${PWD}:/app \
-  -v ${PWD}/samples:/app/samples \
-  hestia-data-validation:latest python run.py "$@"