@hestia-earth/data-validation 0.37.7

package/tests/fixtures/transformation/previousTransformationId/invalid-product-input.json

@@ -0,0 +1,106 @@
+{
+  "id": "1",
+  "type": "Cycle",
+  "functionalUnit": "1 ha",
+  "site": {
+    "id": "1",
+    "type": "Site"
+  },
+  "dataPrivate": true,
+  "endDate": "2008-12",
+  "completeness": {
+    "type": "Completeness",
+    "electricityFuel": true,
+    "material": true,
+    "fertiliser": true,
+    "water": true,
+    "cropResidue": false,
+    "animalFeed": true,
+    "transport": false,
+    "excreta": true,
+    "waste": true,
+    "seed": true,
+    "pesticideVeterinaryDrug": true,
+    "liveAnimalInput": true,
+    "operation": true,
+    "soilAmendment": true,
+    "product": true,
+    "otherChemical": false,
+    "freshForage": false,
+    "ingredient": false,
+    "animalPopulation": false
+  },
+  "defaultMethodClassification": "modelled",
+  "defaultMethodClassificationDescription": "description",
+  "transformations": [
+    {
+      "@type": "Transformation",
+      "transformationId": "tr-1",
+      "term": {
+        "@id": "dryLot",
+        "@type": "Term",
+        "name": "Dry lot",
+        "termType": "excretaManagement"
+      },
+      "transformationDuration": 300,
+      "products": [
+        {
+          "@type": "Product",
+          "term": {
+            "@id": "excretaBeefCattleFeedlotFedKgN",
+            "@type": "Term",
+            "name": "Excreta, beef cattle, feedlot fed (kg N)",
+            "units": "kg N",
+            "termType": "excreta"
+          },
+          "value": [
+            100
+          ]
+        }
+      ]
+    },
+    {
+      "@type": "Transformation",
+      "transformationId": "tr-2",
+      "term": {
+        "@id": "compostingInVessel",
+        "@type": "Term",
+        "name": "Composting - In Vessel",
+        "termType": "excretaManagement"
+      },
+      "transformationDuration": 300,
+      "previousTransformationId": "tr-1",
+      "inputs": [
+        {
+          "@type": "Input",
+          "term": {
+            "@id": "excretaBeefCattleFeedlotFedKgN",
+            "@type": "Term",
+            "name": "Excreta, beef cattle, feedlot fed (kg N)",
+            "units": "kg N",
+            "termType": "excreta"
+          },
+          "value": [
+            150
+          ],
+          "fromCycle": true
+        },
+        {
+          "@type": "Input",
+          "term": {
+            "@id": "rapeseedStraw",
+            "@type": "Term",
+            "name": "Rapeseed, straw",
+            "units": "kg",
+            "termType": "crop"
+          },
+          "value": [
+            25
+          ],
+          "fromCycle": true
+        }
+      ],
+      "transformedShare": 90
+    }
+  ]
+}

package/tests/fixtures/transformation/previousTransformationId/invalid-wrong-order.json

@@ -0,0 +1,136 @@
+{
+  "id": "1",
+  "type": "Cycle",
+  "functionalUnit": "1 ha",
+  "site": {
+    "id": "1",
+    "type": "Site"
+  },
+  "dataPrivate": true,
+  "endDate": "2008-12",
+  "completeness": {
+    "type": "Completeness",
+    "electricityFuel": true,
+    "material": true,
+    "fertiliser": true,
+    "water": true,
+    "cropResidue": false,
+    "animalFeed": true,
+    "transport": false,
+    "excreta": true,
+    "waste": true,
+    "seed": true,
+    "pesticideVeterinaryDrug": true,
+    "liveAnimalInput": true,
+    "operation": true,
+    "soilAmendment": true,
+    "product": true,
+    "otherChemical": false,
+    "freshForage": false,
+    "ingredient": false,
+    "animalPopulation": false
+  },
+  "defaultMethodClassification": "modelled",
+  "defaultMethodClassificationDescription": "description",
+  "transformations": [
+    {
+      "@type": "Transformation",
+      "transformationId": "tr-1",
+      "term": {
+        "@id": "dailySpread",
+        "@type": "Term",
+        "name": "Daily spread",
+        "termType": "excretaManagement"
+      },
+      "transformationDuration": 300
+    },
+    {
+      "@type": "Transformation",
+      "transformationId": "tr-2",
+      "term": {
+        "@id": "compostingInVessel",
+        "@type": "Term"
+      },
+      "transformationDuration": 300,
+      "previousTransformationId": "tr-3",
+      "inputs": [
+        {
+          "@type": "Input",
+          "term": {
+            "@id": "excretaBeefCattleFeedlotFedKgN",
+            "@type": "Term",
+            "name": "Excreta, beef cattle, feedlot fed (kg N)",
+            "units": "kg N",
+            "termType": "excreta"
+          },
+          "value": [
+            150
+          ],
+          "fromCycle": true
+        },
+        {
+          "@type": "Input",
+          "term": {
+            "@id": "rapeseedStraw",
+            "@type": "Term",
+            "name": "Rapeseed, straw",
+            "units": "kg",
+            "termType": "crop"
+          },
+          "value": [
+            25
+          ],
+          "fromCycle": true
+        }
+      ],
+      "emissions": [
+        {
+          "@type": "Emission",
+          "term": {
+            "@id": "ch4ToAirExcretaManagement",
+            "@type": "Term",
+            "name": "CH4, to air, manure management",
+            "units": "kg CH4",
+            "termType": "emission"
+          },
+          "value": [
+            8.7
+          ],
+          "methodModel": {
+            "@id": "emeaEea2019",
+            "@type": "Term",
+            "name": "EMEA-EEA (2019)",
+            "termType": "model"
+          },
+          "methodTier": "tier 2"
+        }
+      ],
+      "products": [
+        {
+          "@type": "Product",
+          "term": {
+            "@id": "excretaMixturesKgN",
+            "@type": "Term",
+            "name": "Excreta mixtures (kg N)",
+            "units": "kg N",
+            "termType": "excreta"
+          },
+          "value": [
+            160
+          ]
+        }
+      ]
+    },
+    {
+      "@type": "Transformation",
+      "transformationId": "tr-3",
+      "term": {
+        "@id": "dryLot",
+        "@type": "Term",
+        "name": "Dry lot",
+        "termType": "excretaManagement"
+      },
+      "transformationDuration": 300
+    }
+  ]
+}

package/tests/fixtures/transformation/previousTransformationId/valid.json

@@ -0,0 +1,171 @@
+{
+  "id": "1",
+  "type": "Cycle",
+  "functionalUnit": "1 ha",
+  "site": {
+    "id": "1",
+    "type": "Site"
+  },
+  "dataPrivate": true,
+  "endDate": "2008-12",
+  "completeness": {
+    "type": "Completeness",
+    "electricityFuel": true,
+    "material": true,
+    "fertiliser": true,
+    "water": true,
+    "cropResidue": false,
+    "animalFeed": true,
+    "transport": false,
+    "excreta": true,
+    "waste": true,
+    "seed": true,
+    "pesticideVeterinaryDrug": true,
+    "liveAnimalInput": true,
+    "operation": true,
+    "soilAmendment": true,
+    "product": true,
+    "otherChemical": false,
+    "freshForage": false,
+    "ingredient": false,
+    "animalPopulation": false
+  },
+  "defaultMethodClassification": "modelled",
+  "defaultMethodClassificationDescription": "description",
+  "products": [
+    {
+      "type": "Product",
+      "term": {
+        "@type": "Term",
+        "name": "Generic Crop, seed",
+        "@id": "genericCropSeed"
+      },
+      "value": [
+        1000
+      ]
+    }
+  ],
+  "transformations": [
+    {
+      "@type": "Transformation",
+      "transformationId": "tr-1",
+      "term": {
+        "@id": "dryLot",
+        "@type": "Term",
+        "name": "Dry lot",
+        "termType": "excretaManagement"
+      },
+      "transformationDuration": 300,
+      "transformedShare": 80,
+      "inputs": [
+        {
+          "type": "Input",
+          "term": {
+            "@type": "Term",
+            "name": "Generic Crop, seed",
+            "@id": "genericCropSeed"
+          },
+          "value": [
+            800
+          ],
+          "fromCycle": true
+        }
+      ],
+      "products": [
+        {
+          "@type": "Product",
+          "term": {
+            "@id": "excretaBeefCattleFeedlotFedKgN",
+            "@type": "Term",
+            "name": "Excreta, beef cattle, feedlot fed (kg N)",
+            "units": "kg N",
+            "termType": "excreta"
+          },
+          "value": [
+            150
+          ]
+        }
+      ]
+    },
+    {
+      "@type": "Transformation",
+      "transformationId": "tr-2",
+      "term": {
+        "@id": "compostingInVessel",
+        "@type": "Term",
+        "name": "Composting - In Vessel",
+        "termType": "excretaManagement"
+      },
+      "transformationDuration": 300,
+      "transformedShare": 100,
+      "previousTransformationId": "tr-1",
+      "inputs": [
+        {
+          "@type": "Input",
+          "term": {
+            "@id": "excretaBeefCattleFeedlotFedKgN",
+            "@type": "Term",
+            "name": "Excreta, beef cattle, feedlot fed (kg N)",
+            "units": "kg N",
+            "termType": "excreta"
+          },
+          "value": [
+            150
+          ],
+          "fromCycle": true
+        },
+        {
+          "@type": "Input",
+          "term": {
+            "@id": "rapeseedStraw",
+            "@type": "Term",
+            "name": "Rapeseed, straw",
+            "units": "kg",
+            "termType": "crop"
+          },
+          "value": [
+            25
+          ],
+          "fromCycle": true
+        }
+      ],
+      "emissions": [
+        {
+          "@type": "Emission",
+          "term": {
+            "@id": "ch4ToAirExcretaManagement",
+            "@type": "Term",
+            "name": "CH4, to air, manure management",
+            "units": "kg CH4",
+            "termType": "emission"
+          },
+          "value": [
+            8.7
+          ],
+          "methodModel": {
+            "@id": "emeaEea2019",
+            "@type": "Term",
+            "name": "EMEA-EEA (2019)",
+            "termType": "model"
+          },
+          "methodTier": "tier 2"
+        }
+      ],
+      "products": [
+        {
+          "@type": "Product",
+          "term": {
+            "@id": "excretaMixturesKgN",
+            "@type": "Term",
+            "name": "Excreta mixtures (kg N)",
+            "units": "kg N",
+            "termType": "excreta"
+          },
+          "value": [
+            160
+          ]
+        }
+      ]
+    }
+  ]
+}

package/tests/integration/test_product.py

@@ -0,0 +1,17 @@
+import os
+import json
+from tests.utils import fixtures_path
+from hestia_earth.validation.validators.product import validate_product_yield
+
+fixtures_folder = os.path.join(fixtures_path, "integration", "distribution")
+
+
+def test_validate_product_yield():
+    with open(os.path.join(fixtures_folder, "product-yield-invalid.json")) as f:
+        cycle = json.load(f)
+
+    result = validate_product_yield(cycle, cycle.get("site"))
+    assert result["level"] == "warning"
+    assert result["dataPath"] == ".products[0].value"
+    assert result["message"] == "is outside confidence interval"
+    assert result["params"]["outliers"] == [1000]

package/tests/test_gee.py

@@ -0,0 +1,10 @@
+from hestia_earth.validation.gee import id_to_level
+
+
+def test_id_to_level():
+    assert id_to_level("GADM-ITA") == 0
+    assert id_to_level("GADM-ITA.16_1") == 1
+    assert id_to_level("GADM-ITA.16.10_1") == 2
+    assert id_to_level("GADM-ITA.16.10.3_1") == 3
+    assert id_to_level("GADM-RWA.5.3.10.4_1") == 4
+    assert id_to_level("GADM-RWA.5.3.10.4.3_1") == 5

package/tests/test_utils.py

@@ -0,0 +1,36 @@
+import pytest
+
+from hestia_earth.validation.utils import match_value_type, _group_nodes
+
+
+@pytest.mark.parametrize(
+    "value,value_type,is_valid",
+    [
+        (10, "number", True),
+        (True, "number", False),
+        ([10, 20], "number", True),
+        ([10, True], "number", False),
+        ([True, False], "boolean", True),
+        (True, "boolean", True),
+        ([10, 20], "boolean", False),
+    ],
+)
+def test_match_value_type(value, value_type: str, is_valid: bool):
+    assert match_value_type(value_type, value) == is_valid, value
+
+
+def test_group_nodes():
+    site1 = {"@type": "Site", "@id": "1"}
+    site2 = {"@type": "Site", "@id": "2"}
+    site3 = {"@type": "Site", "@id": "3"}
+    site4 = {"@type": "Site", "@id": "4"}
+    cycle1 = {"@type": "Cycle", "@id": "1"}
+    cycle2 = {"@type": "Cycle", "@id": "2", "site": site2}
+    cycle3 = {"@type": "Cycle", "@id": "3", "otherSites": [site4]}
+    impact1 = {"@type": "ImpactAssessment", "@id": "1", "cycle": cycle1, "site": site1}
+    nodes = [impact1, cycle2, cycle3, site3]
+    assert _group_nodes(nodes) == {
+        "ImpactAssessment": {"1": impact1},
+        "Cycle": {"1": cycle1, "2": cycle2, "3": cycle3},
+        "Site": {"1": site1, "2": site2, "3": site3, "4": site4},
+    }

package/tests/test_validation.py

@@ -0,0 +1,11 @@
+from unittest.mock import patch
+
+from hestia_earth.validation import validate
+
+
+@patch("hestia_earth.validation.init_gee_by_nodes")
+@patch("hestia_earth.validation.validate_node")
+def test_validate_call_validate_node(mock_validate_node, *args):
+    node = {"type": "Site"}
+    validate([node])
+    mock_validate_node.assert_called_once()

package/tests/utils.py

@@ -0,0 +1,28 @@
+import os
+
+from hestia_earth.validation.terms import TERMS_QUERY
+
+fixtures_path = os.path.abspath("tests/fixtures")
+
+FUEL_TERM_IDS = ["petrol", "diesel"]
+CROP_RESIDUE_TERM_IDS = [
+    "aboveGroundCropResidueRemoved",
+    "aboveGroundCropResidueIncorporated",
+    "aboveGroundCropResidueTotal",
+    "aboveGroundCropResidueLeftOnField",
+    "aboveGroundCropResidueBurnt",
+    "belowGroundCropResidue",
+]
+FORAGE_TERM_IDS = ["wheatFreshForage"]
+RICE_TERM_IDS = ["riceMeal", "riceGrainInHuskUpland"]
+MODEL_TERM_IDS = ["ipcc2007"]
+
+
+def fake_get_terms(query):
+    return {
+        TERMS_QUERY.CROP_RESIDUE: CROP_RESIDUE_TERM_IDS,
+        TERMS_QUERY.FORAGE: FORAGE_TERM_IDS,
+        TERMS_QUERY.FUEL: FUEL_TERM_IDS,
+        TERMS_QUERY.MODEL: MODEL_TERM_IDS,
+        TERMS_QUERY.RICE: RICE_TERM_IDS,
+    }.get(query, [])

package/tests/validators/test_aggregated_cycle.py

@@ -0,0 +1,44 @@
+import os
+import json
+
+from tests.utils import fixtures_path
+from hestia_earth.validation.validators.aggregated_cycle import (
+    validate_linked_impactAssessment,
+)
+
+class_path = "hestia_earth.validation.validators.aggregated_cycle"
+fixtures_folder = os.path.join(fixtures_path, "aggregated", "cycle")
+
+
+def test_validate_linked_impactAssessment_valid():
+    # no inputs should be valid
+    assert validate_linked_impactAssessment({}) is True
+
+    with open(f"{fixtures_folder}/inputs-impactAssessment/valid.json") as f:
+        data = json.load(f)
+    assert validate_linked_impactAssessment(data, "inputs") is True
+
+
+def test_validate_linked_impactAssessment_invalid():
+    with open(f"{fixtures_folder}/inputs-impactAssessment/invalid-world.json") as f:
+        data = json.load(f)
+    assert validate_linked_impactAssessment(data, "inputs") == {
+        "level": "error",
+        "dataPath": ".inputs[0].impactAssessment",
+        "message": "must be linked to a verified country-level Impact Assessment",
+        "params": {
+            "expected": {"@type": "Term", "@id": "GADM-BRA"},
+            "current": "oilPalmFruit-world-2010-2025",
+        },
+    }
+
+    with open(
+        f"{fixtures_folder}/inputs-impactAssessment/invalid-no-impactAssessment.json"
+    ) as f:
+        data = json.load(f)
+    assert validate_linked_impactAssessment(data, "inputs") == {
+        "level": "error",
+        "dataPath": ".inputs[0]",
+        "message": "must be linked to a verified country-level Impact Assessment",
+        "params": {"expected": None, "current": ""},
+    }

package/tests/validators/test_aggregated_shared.py

@@ -0,0 +1,63 @@
+from hestia_earth.validation.validators.aggregated_shared import (
+    validate_quality_score_min,
+    validate_id,
+)
+
+
+def test_validate_quality_score_min_valid():
+    assert validate_quality_score_min({}) is True
+
+    node = {
+        "id": "wheatGrain-united-kingdom-2000-2009",
+        "aggregatedQualityScore": 3,
+        "aggregatedQualityScoreMax": 5,
+    }
+    assert validate_quality_score_min(node) is True
+
+    node = {
+        "id": "wheatGrain-united-kingdom-2000-2009",
+        "aggregatedQualityScore": 5,
+        "aggregatedQualityScoreMax": 5,
+    }
+    assert validate_quality_score_min(node) is True
+
+
+def test_validate_quality_score_min_error():
+    node = {
+        "id": "wheatGrain-united-kingdom-2000-2009",
+        "aggregatedQualityScore": 2,
+        "aggregatedQualityScoreMax": 5,
+    }
+    assert validate_quality_score_min(node) == {
+        "level": "error",
+        "dataPath": ".aggregatedQualityScore",
+        "message": "must be at least equal to the minimum value",
+        "params": {"expected": 3, "current": 2, "min": 3, "max": 5},
+    }
+
+
+def test_validate_quality_score_min_warning():
+    node = {
+        "id": "wheatGrain-united-kingdom-organic-non-irrigated-2000-2009",
+        "aggregatedQualityScore": 2,
+        "aggregatedQualityScoreMax": 5,
+    }
+    assert validate_quality_score_min(node) == {
+        "level": "warning",
+        "dataPath": ".aggregatedQualityScore",
+        "message": "must be at least equal to the minimum value",
+        "params": {"expected": 3, "current": 2, "min": 3, "max": 5},
+    }
+
+
+def test_validate_id_valid():
+    assert validate_id({"id": "sugarBeetRoot-region-world-2010-2025"}) is True
+    assert validate_id({"id": "sugarBeetRoot-GADM-FRA-2010-2025"}) is True
+
+
+def test_validate_id_invalid():
+    assert validate_id({"id": "sugarBeetRoot-2010-2025"}) == {
+        "level": "error",
+        "dataPath": ".id",
+        "message": "aggregation id must contain a product, region, start, and end year",
+    }