@hestia-earth/data-validation 0.37.7 → 0.37.9

package/hestia_earth/validation/validators/measurement.py

@@ -1,368 +0,0 @@
-import importlib
-from hestia_earth.schema import SiteSiteType
-from hestia_earth.utils.lookup import download_lookup, lookup_term_ids
-from hestia_earth.utils.model import find_term_match
-from hestia_earth.utils.tools import (
-    non_empty_list,
-    safe_parse_float,
-    flatten,
-    to_precision,
-)
-from hestia_earth.utils.blank_node import get_node_value
-
-from hestia_earth.validation.utils import (
-    group_blank_nodes,
-    _filter_list_errors,
-    _value_average,
-    _node_year,
-    get_lookup_value,
-)
-from hestia_earth.validation.models import (
-    is_enabled as models_is_enabled,
-    value_from_model,
-)
-from .shared import need_validate_coordinates, value_difference, _parse_node_value
-
-
-SOIL_TEXTURE_IDS = ["sandContent", "siltContent", "clayContent"]
-MEASUREMENTS_MODELS = {
-    "precipitationAnnual": {"model": "geospatialDatabase", "runByYear": True},
-    "precipitationLongTermAnnualMean": {
-        "model": "geospatialDatabase",
-        "runByYear": False,
-    },
-    "temperatureAnnual": {"model": "geospatialDatabase", "runByYear": True},
-    "temperatureLongTermAnnualMean": {
-        "model": "geospatialDatabase",
-        "runByYear": False,
-    },
-}
-WATER_SITE_TYPES = [
-    SiteSiteType.POND.value,
-    SiteSiteType.RIVER_OR_STREAM.value,
-    SiteSiteType.LAKE.value,
-    SiteSiteType.SEA_OR_OCEAN.value,
-]
-
-
-def _group_measurement_key(measurement: dict):
-    keys = non_empty_list(
-        [
-            str(measurement.get("depthUpper", "")),
-            str(measurement.get("depthLower", "")),
-            measurement.get("startDate"),
-            measurement.get("endDate"),
-        ]
-    )
-    return "-".join(keys) if len(keys) > 0 else "default"
-
-
-def validate_soilTexture(measurements: list):
-    lookup = download_lookup("soilTexture.csv")
-    soil_texture_ids = lookup_term_ids(lookup)
-
-    def validate_single(measurements: list, texture: dict, measurement_id: str):
-        term = texture["node"].get("term", {})
-        min = safe_parse_float(
-            get_lookup_value(term, f"{measurement_id}Min"), default=0
-        )
-        max = safe_parse_float(
-            get_lookup_value(term, f"{measurement_id}Max"), default=100
-        )
-        # set default value to min so if no value then passes validation
-        measurement = next(
-            (
-                v
-                for v in measurements
-                if v["node"].get("term", {}).get("@id") == measurement_id
-            ),
-            {},
-        )
-        texture_value = _value_average(measurement.get("node"), min)
-        return min <= texture_value <= max or {
-            "level": "error",
-            "dataPath": f".measurements[{measurement['index']}].value",
-            "message": "is outside the allowed range",
-            "params": {
-                "term": measurement["node"].get("term", {}),
-                "range": {"min": min, "max": max},
-            },
-        }
-
-    def validate(values: list):
-        texture_ids = list(
-            filter(
-                lambda v: v["node"].get("term", {}).get("@id") in soil_texture_ids,
-                values,
-            )
-        )
-        return len(texture_ids) == 0 or flatten(
-            map(
-                lambda texture: list(
-                    map(
-                        lambda id: validate_single(values, texture, id),
-                        SOIL_TEXTURE_IDS,
-                    )
-                ),
-                texture_ids,
-            )
-        )
-
-    groupped_values = group_blank_nodes(enumerate(measurements), by_sum=False).values()
-    return _filter_list_errors(flatten(map(validate, groupped_values)))
-
-
-def validate_depths(measurements: list):
-    def validate(values: tuple):
-        index, measurement = values
-        depthUpper = measurement.get("depthUpper")
-        depthLower = measurement.get("depthLower")
-        return (
-            any([depthUpper is None, depthLower is None])
-            or depthUpper <= depthLower
-            or {
-                "level": "error",
-                "dataPath": f".measurements[{index}].depthLower",
-                "message": "must be greater than or equal to depthUpper",
-            }
-        )
-
-    return _filter_list_errors(map(validate, enumerate(measurements)))
-
-
-def validate_required_depths(site: dict, list_key: str):
-    def validate(values: tuple):
-        index, measurement = values
-        term = measurement.get("term", {})
-        depth_error = get_lookup_value(term, "depthSensitive")
-        depth_warning = get_lookup_value(term, "recommendAddingDepth")
-        has_depths = (
-            measurement.get("depthUpper") is not None
-            and measurement.get("depthLower") is not None
-        )
-        return has_depths or (
-            {
-                "level": "error",
-                "dataPath": f".{list_key}[{index}]",
-                "message": "must set both depthUpper and depthLower",
-            }
-            if depth_error
-            else (
-                {
-                    "level": "warning",
-                    "dataPath": f".{list_key}[{index}]",
-                    "message": "should set both depthUpper and depthLower",
-                }
-                if depth_warning
-                else True
-            )
-        )
-
-    return _filter_list_errors(map(validate, enumerate(site.get(list_key, []))))
-
-
-def validate_term_unique(measurements: list):
-    def count_same_term(term_id: str):
-        return len(
-            list(
-                filter(lambda x: x.get("term", {}).get("@id") == term_id, measurements)
-            )
-        )
-
-    def validate(values: tuple):
-        index, measurement = values
-        term = measurement.get("term", {})
-        term_id = term.get("@id")
-        unique = get_lookup_value(term, "oneMeasurementPerSite")
-        unique = False if unique is None or unique == "-" else bool(unique)
-        return (
-            not unique
-            or count_same_term(term_id) == 1
-            or {
-                "level": "error",
-                "dataPath": f".measurements[{index}].term.name",
-                "message": "must be unique",
-            }
-        )
-
-    return _filter_list_errors(map(validate, enumerate(measurements)))
-
-
-def validate_require_startDate_endDate(site: dict, list_key: str):
-    site_start_date = site.get("startDate")
-    site_end_date = site.get("endDate")
-
-    def validate(values: tuple):
-        index, measurement = values
-        term = measurement.get("term", {})
-        start_date = measurement.get("startDate")
-        end_date = measurement.get("endDate")
-        required = get_lookup_value(term, "needStartDateEndDate")
-        return any(
-            [
-                not required,
-                start_date is not None and end_date is not None,
-                site_start_date is not None and start_date == site_start_date,
-                site_end_date is not None and end_date == site_end_date,
-            ]
-        ) or list(
-            map(
-                lambda k: {
-                    "level": "error",
-                    "dataPath": f".{list_key}[{index}]",
-                    "message": f"should have required property '{k}'",
-                    "params": {"missingProperty": k},
-                },
-                ["startDate", "endDate"],
-            )
-        )
-
-    return _filter_list_errors(
-        flatten(map(validate, enumerate(site.get(list_key, []))))
-    )
-
-
-def _run_from_model(site: dict, blank_node: dict):
-    term_id = blank_node.get("term", {}).get("@id")
-    params = MEASUREMENTS_MODELS.get(term_id, {})
-    year = _node_year(blank_node)
-    should_run = all(
-        [
-            need_validate_coordinates(site),
-            not params.get("runByYear", False) or year is not None,
-        ]
-    )
-    model_run = (
-        importlib.import_module(
-            f"hestia_earth.models.{params.get('model')}.{term_id}"
-        )._run
-        if params.get("model")
-        else None
-    )
-    return (
-        (model_run(site, year) if params.get("runByYear", False) else model_run(site))
-        if all([params.get("model"), should_run])
-        else None
-    )
-
-
-def validate_with_models(site: dict, list_key: str):
-    threshold = 0.25
-
-    def validate(values: tuple):
-        index, blank_node = values
-        value = _parse_node_value(blank_node)
-        expected_node = _run_from_model(site, blank_node) or {}
-        expected_method_model = (
-            (expected_node[0] or {})
-            if isinstance(expected_node, list)
-            else expected_node
-        ).get("methodModel", {})
-        expected_value = value_from_model(expected_node) if expected_node else None
-        delta = value_difference(value or 0, expected_value)
-        data_path = "" if blank_node.get("value") is None else ".value"
-        return delta < threshold or {
-            "level": "warning",
-            "dataPath": f".{list_key}[{index}]{data_path}",
-            "message": "the measurement provided might be in error",
-            "params": {
-                "term": blank_node.get("term", {}),
-                "model": expected_method_model,
-                "current": value,
-                "expected": expected_value,
-                "delta": to_precision(delta * 100, 4),
-                "threshold": threshold,
-            },
-        }
-
-    nodes = site.get(list_key, []) if models_is_enabled() else []
-    return _filter_list_errors(flatten(map(validate, enumerate(nodes))))
-
-
-def validate_value_length(site: dict, list_key: str):
-    def validate(values: tuple):
-        index, blank_node = values
-        term = blank_node.get("term", {})
-        array_type = get_lookup_value(term, "arrayTreatment")
-        value_length = len(blank_node.get("value", []))
-        return (
-            array_type != "arrayNotAllowed"
-            or value_length <= 1
-            or {
-                "level": "error",
-                "dataPath": f".measurements[{index}].value",
-                "message": "must not contain more than 1 value",
-            }
-        )
-
-    return _filter_list_errors(
-        flatten(map(validate, enumerate(site.get(list_key, []))))
-    )
-
-
-def validate_water_measurements(site: dict, list_key: str):
-    site_type_valid = site.get("siteType") in WATER_SITE_TYPES
-    nodes = site.get(list_key, [])
-    term_ids = ["salineWater", "freshWater", "brackishWater", "waterSalinity"]
-    has_node = any([find_term_match(nodes, term_id) for term_id in term_ids])
-    return (
-        not site_type_valid
-        or has_node
-        or {
-            "level": "error",
-            "dataPath": f".{list_key}",
-            "message": "must specify water type for ponds",
-            "params": {"ids": term_ids},
-        }
-    )
-
-
-_ALLOWED_SALINITY = {
-    "brackishWater": lambda value: 500 <= value <= 18000,
-    "freshWater": lambda value: value < 500,
-    "salineWater": lambda value: value > 18000,
-}
-
-
-def validate_water_salinity(site: dict, list_key: str):
-    site_type_valid = site.get("siteType") in WATER_SITE_TYPES
-    nodes = site.get(list_key, [])
-    waterSalinity = get_node_value(
-        find_term_match(nodes, "waterSalinity", {}), default=None
-    )
-    valid_saline_ids = (
-        non_empty_list(
-            [
-                term_id
-                for term_id, validator in _ALLOWED_SALINITY.items()
-                if validator(waterSalinity)
-            ]
-        )
-        if waterSalinity is not None
-        else []
-    )
-    invalid_saline_nodes = (
-        non_empty_list(
-            [
-                find_term_match(nodes, term_id)
-                for term_id, validator in _ALLOWED_SALINITY.items()
-                if not validator(waterSalinity)
-            ]
-        )
-        if waterSalinity is not None
-        else []
-    )
-    return (
-        not site_type_valid
-        or waterSalinity is None
-        or not invalid_saline_nodes
-        or {
-            "level": "error",
-            "dataPath": f".{list_key}",
-            "message": "invalid water salinity",
-            "params": {
-                "current": invalid_saline_nodes[0].get("term", {}).get("@id"),
-                "expected": valid_saline_ids[0] if valid_saline_ids else None,
-            },
-        }
-    )

package/hestia_earth/validation/validators/organisation.py

@@ -1,43 +0,0 @@
-"""
-Organisation validation
-
-Here is the list of validations running on a [Organisation](/schema/Organisation).
-"""
-
-from .shared import (
-    validate_dates,
-    need_validate_coordinates,
-    validate_coordinates,
-    validate_area,
-    need_validate_area,
-    validate_date_lt_today,
-    validate_country_region,
-)
-
-
-def validate_organisation_dates(organisation: dict):
-    """
-    Validate Organisation Dates
-
-    This validation ensures that the `endDate` must be later than `startDate`.
-    """
-    return validate_dates(organisation) or {
-        "level": "error",
-        "dataPath": ".endDate",
-        "message": "must be greater than startDate",
-    }
-
-
-def validate_organisation(organisation: dict, node_map: dict = {}):
-    return [
-        validate_organisation_dates(organisation),
-        validate_date_lt_today(organisation, "startDate"),
-        validate_date_lt_today(organisation, "endDate"),
-        validate_country_region(organisation),
-        (
-            validate_coordinates(organisation)
-            if need_validate_coordinates(organisation)
-            else True
-        ),
-        validate_area(organisation) if need_validate_area(organisation) else True,
-    ]