@hestia-earth/data-validation 0.37.7

package/tests/validators/test_site.py

@@ -0,0 +1,151 @@
+from unittest.mock import patch
+import os
+import json
+from hestia_earth.schema import SiteSiteType
+
+from tests.utils import fixtures_path
+from hestia_earth.validation.utils import _group_nodes
+from hestia_earth.validation.validators.site import (
+    validate_site,
+    validate_site_dates,
+    validate_site_coordinates,
+    validate_siteType,
+    validate_cycle_dates,
+    validate_cycles_linked_ia,
+)
+
+fixtures_folder = os.path.join(fixtures_path, "site")
+class_path = "hestia_earth.validation.validators.site"
+
+
+@patch(f"{class_path}.validate_boundary_area", return_value=True)
+@patch(f"{class_path}.validate_region_size", return_value=True)
+@patch(f"{class_path}.validate_coordinates", return_value=True)
+def test_validate_site_valid(*args):
+    with open(f"{fixtures_folder}/valid.json") as f:
+        node = json.load(f)
+    results = validate_site(node)
+    assert all([r is True for r in results])
+
+
+def test_validate_site_dates_valid():
+    site = {"startDate": "2020-01-01", "endDate": "2020-01-02"}
+    assert validate_site_dates(site) is True
+
+
+def test_validate_site_dates_invalid():
+    site = {"startDate": "2020-01-02", "endDate": "2020-01-01"}
+    assert validate_site_dates(site) == {
+        "level": "error",
+        "dataPath": ".endDate",
+        "message": "must be greater than startDate",
+    }
+
+
+@patch(f"{class_path}.need_validate_coordinates", return_value=True)
+def test_validate_site_coordinates(*args):
+    site = {"siteType": SiteSiteType.CROPLAND.value}
+    assert validate_site_coordinates(site)
+
+    site["siteType"] = SiteSiteType.SEA_OR_OCEAN.value
+    assert not validate_site_coordinates(site)
+
+
+@patch(f"{class_path}.get_cached_data", return_value=25)
+def test_validate_siteType_valid(*args):
+    site = {
+        "siteType": SiteSiteType.FOREST.value,
+        "latitude": 44.18753,
+        "longitude": -0.62521,
+    }
+    assert validate_siteType(site) is True
+
+    site = {
+        "siteType": SiteSiteType.CROPLAND.value,
+        "latitude": 44.5096,
+        "longitude": 0.40749,
+    }
+    assert validate_siteType(site) is True
+
+
+@patch(f"{class_path}.get_cached_data", return_value=30)
+def test_validate_siteType_invalid(*args):
+    site = {
+        "siteType": SiteSiteType.CROPLAND.value,
+        "latitude": 44.18753,
+        "longitude": -0.62521,
+    }
+    assert validate_siteType(site) == {
+        "level": "warning",
+        "dataPath": ".siteType",
+        "message": "The coordinates you have provided are not in a known cropland "
+        "area according to the MODIS Land Cover classification (MCD12Q1.006, LCCS2, bands 25, 35, 36).",
+    }
+
+    site = {
+        "siteType": SiteSiteType.FOREST.value,
+        "latitude": 44.5096,
+        "longitude": 0.40749,
+    }
+    assert validate_siteType(site) == {
+        "level": "warning",
+        "dataPath": ".siteType",
+        "message": "The coordinates you have provided are not in a known forest "
+        "area according to the MODIS Land Cover classification (MCD12Q1.006, LCCS2, bands 10, 20, 25).",
+    }
+
+
+def test_validate_cycle_dates_valid():
+    cycles = [{"endDate": "2020"}, {"endDate": "2021"}]
+    assert validate_cycle_dates(cycles) is True
+
+    cycles = [
+        {"startDate": "2020-01-01", "endDate": "2020-06-30"},
+        {"startDate": "2020-07-01", "endDate": "2020-12-31"},
+    ]
+    assert validate_cycle_dates(cycles) is True
+
+
+def test_validate_cycle_dates_invalid():
+    cycles = [
+        {"id": "cycle1", "endDate": "2020"},
+        {"id": "cycle2", "endDate": "2020"},
+        {"id": "cycle3", "endDate": "2021"},
+    ]
+    assert validate_cycle_dates(cycles) == {
+        "level": "error",
+        "dataPath": "",
+        "message": "multiple cycles on the same site cannot overlap",
+        "params": {"ids": ["cycle1", "cycle2"]},
+    }
+
+    cycles = [
+        {"id": "cycle1", "startDate": "2020-01-01", "endDate": "2020-07-02"},
+        {"id": "cycle2", "startDate": "2020-07-01", "endDate": "2020-12-31"},
+        {"id": "cycle3", "startDate": "2021-01-01", "endDate": "2021-12-31"},
+    ]
+    assert validate_cycle_dates(cycles) == {
+        "level": "error",
+        "dataPath": "",
+        "message": "multiple cycles on the same site cannot overlap",
+        "params": {"ids": ["cycle1", "cycle2"]},
+    }
+
+
+def test_validate_cycles_linked_ia_valid():
+    with open(f"{fixtures_folder}/cycles-linked-ia/valid.json") as f:
+        nodes = json.load(f).get("nodes")
+    cycles = [c for c in nodes if c.get("type") == "Cycle"]
+    assert validate_cycles_linked_ia(cycles, _group_nodes(nodes)) is True
+
+
+def test_validate_cycles_linked_ia_invalid():
+    with open(f"{fixtures_folder}/cycles-linked-ia/invalid.json") as f:
+        nodes = json.load(f).get("nodes")
+    cycles = [c for c in nodes if c.get("type") == "Cycle"]
+    assert validate_cycles_linked_ia(cycles, _group_nodes(nodes)) == {
+        "level": "error",
+        "dataPath": "",
+        "message": "cycles linked together cannot be added to the same site",
+        "params": {"ids": ["2"]},
+    }

package/tests/validators/test_source.py

@@ -0,0 +1,15 @@
+import os
+import json
+
+from tests.utils import fixtures_path
+from hestia_earth.validation.validators.source import validate_source
+
+fixtures_folder = os.path.join(fixtures_path, "source")
+class_path = "hestia_earth.validation.validators.source"
+
+
+def test_validate_valid():
+    with open(f"{fixtures_path}/source/valid.json") as f:
+        node = json.load(f)
+    results = validate_source(node)
+    assert all([r is True for r in results])

package/tests/validators/test_transformation.py

@@ -0,0 +1,151 @@
+import json
+from unittest.mock import patch
+
+from tests.utils import fixtures_path
+from hestia_earth.validation.validators.transformation import (
+    validate_previous_transformation,
+    validate_transformation_excretaManagement,
+    validate_linked_emission,
+    validate_excreta_inputs_products,
+)
+
+class_path = "hestia_earth.validation.validators.transformation"
+fixtures_folder = f"{fixtures_path}/transformation"
+
+
+def test_validate_previous_transformation_valid():
+    # no transformations should be valid
+    assert validate_previous_transformation({}, "transformations") is True
+
+    with open(f"{fixtures_folder}/previousTransformationId/valid.json") as f:
+        cycle = json.load(f)
+    assert validate_previous_transformation(cycle) is True
+
+
+def test_validate_previous_transformation_invalid():
+    with open(
+        f"{fixtures_folder}/previousTransformationId/invalid-wrong-order.json"
+    ) as f:
+        cycle = json.load(f)
+    assert validate_previous_transformation(cycle) == {
+        "level": "error",
+        "dataPath": ".transformations[1].previousTransformationId",
+        "message": "must point to a previous transformation in the list",
+    }
+
+    with open(
+        f"{fixtures_folder}/previousTransformationId/invalid-no-previous.json"
+    ) as f:
+        cycle = json.load(f)
+    assert validate_previous_transformation(cycle) == {
+        "level": "error",
+        "dataPath": ".transformations[1].previousTransformationId",
+        "message": "must point to a previous transformation in the list",
+    }
+
+    with open(
+        f"{fixtures_folder}/previousTransformationId/invalid-product-input.json"
+    ) as f:
+        cycle = json.load(f)
+    assert validate_previous_transformation(cycle) == {
+        "level": "error",
+        "dataPath": ".transformations[1].inputs[0].value",
+        "message": "must be equal to previous product multiplied by the share",
+    }
+
+    with open(
+        f"{fixtures_folder}/previousTransformationId/invalid-previous-input.json"
+    ) as f:
+        cycle = json.load(f)
+    assert validate_previous_transformation(cycle) == [
+        {
+            "level": "error",
+            "dataPath": ".transformations[0]",
+            "message": "at least one Input must be a Product of the Cycle",
+        },
+        {
+            "level": "error",
+            "dataPath": ".transformations[1]",
+            "message": "at least one Input must be a Product of the previous Transformation",
+        },
+    ]
+
+
+def test_validate_transformation_excretaManagement_valid():
+    # no transformations should be valid
+    assert validate_transformation_excretaManagement({}, "transformations")
+
+    with open(f"{fixtures_folder}/excretaManagement/valid.json") as f:
+        cycle = json.load(f)
+    assert validate_transformation_excretaManagement(cycle) is True
+
+
+def test_validate_transformation_excretaManagement_invalid():
+    with open(f"{fixtures_folder}/excretaManagement/invalid.json") as f:
+        cycle = json.load(f)
+    assert validate_transformation_excretaManagement(cycle) == {
+        "level": "error",
+        "dataPath": ".transformations[0].practices",
+        "message": "an excreta input is required when using an excretaManagement practice",
+    }
+
+
+def test_validate_linked_emission_valid():
+    # no emissions should be valid
+    assert validate_linked_emission({}, "transformations") is True
+
+    with open(f"{fixtures_folder}/linked-emission/valid.json") as f:
+        data = json.load(f)
+    assert validate_linked_emission(data, "transformations") is True
+
+
+def test_validate_linked_emission_invalid():
+    with open(f"{fixtures_folder}/linked-emission/invalid.json") as f:
+        data = json.load(f)
+    assert validate_linked_emission(data, "transformations") == {
+        "level": "warning",
+        "dataPath": ".transformations[0].emissions[0]",
+        "message": "should be linked to an emission in the Cycle",
+        "params": {
+            "term": {
+                "@type": "Term",
+                "@id": "ch4ToAirEntericFermentation",
+                "termType": "emission",
+            }
+        },
+    }
+
+
+def test_validate_excreta_inputs_products_valid():
+    # no transformations should be valid
+    assert validate_excreta_inputs_products([]) is True
+
+    with open(f"{fixtures_folder}/inputs-products/valid.json") as f:
+        data = json.load(f)
+    assert validate_excreta_inputs_products(data) is True
+
+
+def fake_download_excreta(term_id: str, *args):
+    return {
+        "processedExcretaKgVs": {},
+        "excretaBeefCattleExceptFeedlotFedKgMass": {"subClassOf": [{}]},
+    }[term_id]
+
+
+@patch(f"{class_path}.download_hestia", side_effect=fake_download_excreta)
+def test_validate_excreta_inputs_products_invalid(*args):
+    with open(f"{fixtures_folder}/inputs-products/invalid.json") as f:
+        data = json.load(f)
+    assert validate_excreta_inputs_products(data) == {
+        "level": "error",
+        "dataPath": ".transformations[0].products[1]",
+        "message": "must be included as an Input",
+        "params": {
+            "term": {
+                "@type": "Term",
+                "@id": "excretaBeefCattleExceptFeedlotFedKgMass",
+                "termType": "excreta",
+            },
+            "expected": ["excretaDairyCattle"],
+        },
+    }

package/tests/validators/test_validators.py

@@ -0,0 +1,74 @@
+from unittest.mock import patch
+import json
+from hestia_earth.schema import NodeType
+
+from tests.utils import fixtures_path
+from hestia_earth.validation.validators import validate_node, _should_run
+
+class_path = "hestia_earth.validation.validators"
+
+
+def test_should_run():
+    ntype = NodeType.CYCLE.value
+    node = {"@type": ntype, "@id": "cycle"}
+    assert not _should_run(ntype, node)
+
+    node = {"@type": ntype, "@id": "cycle", "name": "Cycle 1"}
+    assert not _should_run(ntype, node)
+
+    node = {"@type": ntype, "@id": "cycle", "name": "Cycle 1", "endDate": "2020-01-01"}
+    assert _should_run(ntype, node) is True
+
+
+@patch(f"{class_path}.validate_site")
+def test_validate_node_type_validation(mock_validate_site):
+    node = {
+        "type": NodeType.SITE.value,
+        "id": "id",
+        "siteType": "cropland",
+        "country": {"@id": "GADM-GBR"},
+    }
+    validate_node()(node)
+    mock_validate_site.assert_called_once()
+
+    # no validation on existing nodes
+    mock_validate_site.reset_mock()
+    node = {"@type": NodeType.SITE.value}
+    validate_node()(node)
+    mock_validate_site.assert_not_called()
+
+
+def test_validate_node_no_validation():
+    # no validation on uploaded Actor
+    node = {"type": NodeType.ACTOR.value}
+    assert validate_node([])(node) == []
+
+
+@patch(f"{class_path}.validate_cycle")
+@patch(f"{class_path}.validate_impact_assessment")
+@patch(f"{class_path}.validate_site")
+def test_validate_nested(
+    mock_validate_site, mock_validate_impact_assessment, mock_validate_cycle
+):
+    with open(f"{fixtures_path}/impactAssessment/valid.json") as f:
+        node = json.load(f)
+    assert validate_node()(node) == []
+    assert mock_validate_cycle.call_count == 1
+    assert mock_validate_impact_assessment.call_count == 1
+    assert mock_validate_site.call_count == 2
+
+
+@patch(f"{class_path}.validate_cycle")
+@patch(f"{class_path}.validate_impact_assessment")
+@patch(f"{class_path}.validate_site")
+def test_validate_nested_aggregated(
+    mock_validate_site, mock_validate_impact_assessment, mock_validate_cycle
+):
+    with open(f"{fixtures_path}/impactAssessment/valid.json") as f:
+        node = json.load(f)
+
+    node["aggregated"] = True
+    assert validate_node()(node) == []
+    assert mock_validate_cycle.call_count == 0
+    assert mock_validate_impact_assessment.call_count == 1
+    assert mock_validate_site.call_count == 0

package/tsconfig.dist.json

@@ -0,0 +1,9 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./dist",
+    "sourceMap": false
+  },
+  "include": ["./src/index.ts"],
+  "exclude": ["./src/**/*.spec.ts"]
+}

package/tsconfig.json

@@ -0,0 +1,25 @@
+{
+  "compilerOptions": {
+    "target": "es5",
+    "module": "commonjs",
+    "stripInternal": true,
+    "emitDecoratorMetadata": true,
+    "experimentalDecorators": true,
+    "removeComments": false,
+    "noImplicitAny": false,
+    "skipLibCheck": true,
+    "lib": ["es2019"],
+    "baseUrl": ".",
+    "rootDir": "./src",
+    "outDir": "./dist",
+    "sourceMap": true,
+    "declaration": true,
+    "downlevelIteration": true,
+    "resolveJsonModule": true
+  },
+  "include": ["src/**/*.ts"],
+  "exclude": ["node_modules"],
+  "atom": {
+    "rewriteTsconfig": false
+  }
+}