@hestia-earth/data-validation 0.37.7

package/docs/conf.py

@@ -0,0 +1,74 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath(".."))
+
+
+# -- Project information -----------------------------------------------------
+
+project = "HESTIA Data Validation"
+copyright = "2020, HESTIA Team"
+author = "Guillaume Royer"
+
+# The full version, including alpha/beta/rc tags
+release = "0.0.1"
+
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    "recommonmark",
+    "sphinx.ext.autodoc",
+    "sphinx.ext.napoleon",
+    "sphinx.ext.autosummary",
+    "sphinx.ext.viewcode",
+    "sphinx_autodoc_typehints",
+]
+
+autosummary_generate = True
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ["_templates"]
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = "sphinx_rtd_theme"
+
+# -- Options for HTML output -------------------------------------------------
+
+# on_rtd is whether on readthedocs.org, this line of code grabbed from docs.readthedocs.org...
+on_rtd = os.getenv("READTHEDOCS", None) == "True"
+if not on_rtd:  # only import and set the theme if we're building docs locally
+    import sphinx_rtd_theme
+
+    html_theme = "sphinx_rtd_theme"
+    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ["_static"]

package/docs/index.rst

@@ -0,0 +1,42 @@
+Welcome to HESTIA Data Validation's documentation!
+========================================
+
+This package contains the Data Validation module developed by HESTIA.
+
+Installation
+------------
+
+Install from `PyPI <https://pypi.python.org/pypi>`_ using `pip <http://www.pip-installer.org/en/latest/>`_, a
+package manager for Python.
+
+.. code-block:: bash
+
+    pip install hestia_earth.validation
+
+
+Requirements
+============
+
+- `hestia_earth.schema <https://pypi.org/project/hestia-earth.schema/>`_
+- `requests <https://pypi.org/project/requests/>`_
+- `python-dateutil <https://pypi.org/project/python-dateutil/>`_
+- `area <https://pypi.org/project/area/>`_
+
+Contents
+--------
+
+.. autosummary::
+   :toctree: _autosummary
+   :caption: API Reference
+   :template: custom-module-template.rst
+   :recursive:
+
+   hestia_earth.validation
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`

package/docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

package/docs/requirements.txt

@@ -0,0 +1,13 @@
+hestia_earth.validation
+docutils==0.17.1
+Jinja2==3.0.3
+alabaster==0.7.13
+sphinx-rtd-theme==0.5.0
+sphinxcontrib-applehelp==1.0.4
+sphinxcontrib-devhelp==1.0.2
+sphinxcontrib-qthelp==1.0.3
+sphinxcontrib-htmlhelp==2.0.1
+sphinxcontrib-serializinghtml==1.1.5
+sphinx-autodoc-typehints==1.12.0
+sphinx==3.3.1
+recommonmark

package/envs/.develop.env

@@ -0,0 +1 @@
+export STAGE='dev'

package/envs/.master.env

@@ -0,0 +1 @@
+export STAGE='prod'

package/hestia_earth/validation/README.md

@@ -0,0 +1,5 @@
+# HESTIA Data Validation
+
+See the complete list of validations available on the HESTIA format.
+
+Each validation is divided by node type: `Cycle`, `Site`, `ImpactAssessment`, and `Organisation`.

package/hestia_earth/validation/__init__.py

@@ -0,0 +1,32 @@
+from typing import List
+from concurrent.futures import ThreadPoolExecutor
+from hestia_earth.utils.tools import current_time_ms
+
+from .log import logger
+from .validators import validate_node
+from .utils import _group_nodes, _hash_nodes
+from .gee import init_gee_by_nodes
+
+
+def validate(nodes: List[dict]):
+    """
+    Validates a list of HESTIA JSON-Nodes against a list of rules.
+
+    Parameters
+    ----------
+    nodes : List[dict]
+        The list of JSON-Nodes to validate.
+
+    Returns
+    -------
+    List
+        The list of errors for each node, which can be empty if no errors detected.
+    """
+    now = current_time_ms()
+    nodes = init_gee_by_nodes(nodes)
+    nodes_by_type = _group_nodes(nodes)
+    nodes_by_hash = _hash_nodes(nodes)
+    with ThreadPoolExecutor() as executor:
+        results = list(executor.map(validate_node(nodes_by_type, nodes_by_hash), nodes))
+    logger.info("time=%s, unit=ms", current_time_ms() - now)
+    return results

package/hestia_earth/validation/distribution.py

@@ -0,0 +1,22 @@
+import os
+from functools import lru_cache
+
+from .log import logger
+
+ENABLED = os.getenv("VALIDATE_DISTRIBUTION", "true") == "true"
+
+
+@lru_cache()
+def is_enabled():
+    if ENABLED:
+        try:
+            from hestia_earth.distribution.version import VERSION
+
+            logger.debug("Using distribution version %s", VERSION)
+            return True
+        except ImportError:
+            logger.error(
+                "Run `pip install hestia-earth-validation[distribution]` to use distribution validation"
+            )
+
+    return False

package/hestia_earth/validation/gee.py

@@ -0,0 +1,162 @@
+import os
+from hestia_earth.utils.tools import current_time_ms
+from hestia_earth.schema import NodeType
+
+from .log import logger
+
+_ENABLED = os.getenv("VALIDATE_SPATIAL", "true") == "true"
+_ENABLE_TYPES = [NodeType.SITE.value, NodeType.ORGANISATION.value]
+MAX_AREA_SIZE = int(os.getenv("MAX_AREA_SIZE", "5000"))
+_CACHE_BATCH_SIZE = int(os.getenv("CACHE_SITES_BATCH_SIZE", "5000"))
+
+_caching = {}
+_RASTERS = [
+    {
+        "name": "siteType",
+        "collection": "MODIS/006/MCD12Q1",
+        "band_name": "LC_Prop2",
+        "year": "2019",
+    }
+]
+_VECTORS = [
+    {
+        "name": f"region-{level}",
+        "collection": f"users/hestiaplatform/gadm36_{level}",
+        "fields": f"GID_{level}",
+    }
+    for level in range(0, 6)
+]
+
+
+def _caching_key(func_name: str, args: dict):
+    return "-".join([func_name, str(args)])
+
+
+def _run_with_cache(func_name: str, args: dict, func):
+    global _caching  # noqa: F824
+    key = _caching_key(func_name, args)
+    _caching[key] = _caching.get(key, func())
+    return _caching[key]
+
+
+def _should_cache_node(node: dict):
+    return all(
+        [
+            node.get("@type", node.get("type")) in _ENABLE_TYPES,
+            not node.get("aggregated", False),
+            "latitude" in node and "longitude" in node,
+        ]
+    )
+
+
+def _node_key(node: dict):
+    return "/".join(
+        [node.get("type", node.get("@type")), node.get("id", node.get("@id"))]
+    )
+
+
+def _pop_items(values: list, nb_items: int):
+    if len(values) < nb_items:
+        removed_items = values[:]  # Get a copy of the entire array
+        values.clear()  # Remove all items from the original array
+    else:
+        removed_items = values[:nb_items]  # Get the first N items
+        del values[:nb_items]  # Remove the first N items from the original array
+
+    return removed_items
+
+
+def _cache_nodes(nodes: list, batch_size: int):
+    from hestia_earth.models.cache_sites import ParamType, _run_values
+
+    now = current_time_ms()
+
+    nodes_mapping = {_node_key(n): n for n in nodes}
+
+    cache_nodes = list(filter(_should_cache_node, nodes))
+    while len(cache_nodes) > 0:
+        batch_values = _pop_items(cache_nodes, batch_size)
+        logger.info(f"Caching {len(batch_values)} nodes. {len(cache_nodes)} remaining.")
+        results = _run_values(
+            [(n, 0) for n in batch_values],  # expecting tuple with area_size
+            ParamType.COORDINATES,
+            _RASTERS,
+            _VECTORS,
+            years=[],
+        )
+
+        for result in results:
+            nodes_mapping[_node_key(result)] = result
+
+    logger.info("Done caching in %sms", current_time_ms() - now)
+
+    return list(nodes_mapping.values())
+
+
+def _init_gee_by_nodes(nodes: list):
+    from hestia_earth.earth_engine import init_gee
+
+    init_gee()
+    try:
+        return _cache_nodes(nodes, _CACHE_BATCH_SIZE)
+    except Exception as e:
+        logger.error(f"An error occured while caching nodes on EE: {str(e)}")
+        if "User memory limit exceeded" in str(e) or "query aborted" in str(e):
+            return _cache_nodes(nodes, 100)
+
+
+def init_gee_by_nodes(nodes: list):
+    # need to validate for non-aggregated Site or Oganisation with coordinates
+    enabled_nodes = list(filter(_should_cache_node, nodes))
+    should_init = len(enabled_nodes) > 0
+    return _init_gee_by_nodes(nodes) if should_init and is_enabled() else nodes
+
+
+def is_enabled():
+    if _ENABLED:
+        try:
+            from hestia_earth.earth_engine.version import VERSION
+
+            return isinstance(VERSION, str)
+        except ImportError:
+            logger.error(
+                "Run `pip install hestia_earth.earth_engine` to use geospatial validation"
+            )
+
+    return False
+
+
+def id_to_level(id: str):
+    return id.count(".")
+
+
+def get_cached_data(site: dict, key: str, year: int = None):
+    from hestia_earth.models.geospatialDatabase.utils import _cached_value
+
+    value = _cached_value(site, key)
+    return value.get(str(year)) if value and year else value
+
+
+def get_region_id(node: dict):
+    level = id_to_level(node.get("region", node.get("country")).get("@id"))
+    id = get_cached_data(node, f"region-{level}")
+    return None if id is None else f"GADM-{id}"
+
+
+def get_region_distance(gid: str, latitude: float, longitude: float):
+    def exec_func():
+        return round(
+            get_distance_to_coordinates(gid, latitude=latitude, longitude=longitude)
+            / 1000
+        )
+
+    try:
+        from hestia_earth.earth_engine.gadm import get_distance_to_coordinates
+
+        return _run_with_cache(
+            "get_region_distance",
+            {"gid": gid, "latitude": latitude, "longitude": longitude},
+            exec_func,
+        )
+    except Exception:
+        return None

package/hestia_earth/validation/log.py

@@ -0,0 +1,44 @@
+import os
+import logging
+
+
+def _add_handler(
+    logger: logging.Logger, formatter: logging.Formatter, handler: logging.Handler
+):
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+
+
+LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO")
+logger = logging.getLogger("hestia_earth.validation")
+logger.setLevel(logging.getLevelName(LOG_LEVEL))
+
+
+def log_to_file(filepath: str):
+    """
+    By default, all logs are saved into a file with path stored in the env variable `LOG_FILENAME`.
+    If you do not set the environment variable `LOG_FILENAME`, you can use this function with the file path.
+
+    Parameters
+    ----------
+    filepath : str
+        Path of the file.
+    """
+    formatter = logging.Formatter(
+        '{"timestamp": "%(asctime)s", "level": "%(levelname)s", "logger": "%(name)s", '
+        '"filename": "%(filename)s", "message": "%(message)s"}',
+        "%Y-%m-%dT%H:%M:%S%z",
+    )
+    handler = logging.FileHandler(filepath, encoding="utf-8")
+    handler.setLevel(logging.getLevelName(LOG_LEVEL))
+    _add_handler(logger, formatter, handler)
+
+
+_formatter = logging.Formatter(
+    "%(asctime)s - %(name)s - %(levelname)s - %(filename)s - %(message)s"
+)
+_add_handler(logger, _formatter, logging.StreamHandler())
+
+LOG_FILENAME = os.getenv("LOG_FILENAME")
+if LOG_FILENAME is not None:
+    log_to_file(LOG_FILENAME)

package/hestia_earth/validation/models.py

@@ -0,0 +1,141 @@
+import importlib
+import os
+from functools import lru_cache
+from hestia_earth.utils.tools import list_sum
+
+from .log import logger
+
+ENABLED = os.getenv("VALIDATE_MODELS", "true") == "true"
+
+
+@lru_cache()
+def is_enabled():
+    if ENABLED:
+        try:
+            from hestia_earth.models.version import VERSION
+
+            logger.debug("Using models version %s", VERSION)
+            return True
+        except ImportError:
+            logger.error(
+                "Run `pip install hestia-earth-validation[models]` to use models validation"
+            )
+
+    return False
+
+
+def _model_value_from_list(results: list, default_value: float):
+    return (
+        results[0].get("value", [default_value]) if len(results) > 0 else default_value
+    )
+
+
+def _model_value(result, default_value=0):
+    return (
+        default_value
+        if result is None
+        else (
+            _model_value_from_list(result, default_value)
+            if isinstance(result, list)
+            else (
+                result.get("value", [default_value])
+                if isinstance(result, dict)
+                else default_value
+            )
+        )
+    )
+
+
+def value_from_model(result):
+    value = _model_value(result)
+    try:
+        # fix numpy.float64
+        # TODO: find a better way to handle this
+        return list_sum(value, float(value))
+    except Exception:
+        return list_sum(value, value)
+
+
+def method_tier_from_model(result):
+    return (
+        None
+        if result is None
+        else (
+            method_tier_from_model(result[0])
+            if isinstance(result, list) and len(result) > 0
+            else (result.get("methodTier") if isinstance(result, dict) else None)
+        )
+    )
+
+
+def _import_model(name: str):
+    return importlib.import_module(f"hestia_earth.models.{name}").run
+
+
+def run_model(model: str, term_id: str, data: dict):
+    """
+    Run a HESTIA model from the engine models library.
+
+    Parameters
+    ----------
+    model : str
+        The name of the model to run.
+    term_id : str
+        The term to run the model on.
+    data : dict
+        The data used to run the model.
+
+    Returns
+    -------
+    Any
+        The result of the model, which can be a single `dict` or a list of `dict`s.
+    """
+    return _import_model(model)(term_id, data)
+
+
+def run_model_from_node(node: dict, data: dict):
+    """
+    Run a HESTIA model from the engine models library.
+    To use this function, you need to use a Blank Node that contains a `methodModel` and a `term`,
+    otherwise you need to use the `run_model` method.
+
+    Parameters
+    ----------
+    node : dict
+        The Blank Node containing a `methodModel` and a `Term`.
+    data : dict
+        The data used to run the model.
+
+    Returns
+    -------
+    Any
+        The result of the model, which can be a single `dict` or a list of `dict`s.
+    """
+    methodModel = node.get("methodModel", {}).get("@id")
+    term_id = node.get("term", {}).get("@id")
+    return run_model(methodModel, term_id, data)
+
+
+def run_models(node: dict, models: list):
+    """
+    Run a list of models, using the orchestrator, and return the updated node.
+
+    Parameters
+    ----------
+    node : dict
+        The Blank Node containing a `methodModel` and a `Term`.
+    models : list
+        List of models as defined in the orchestrator configurations.
+
+    Returns
+    -------
+    Any
+        The recalculated node after running all models
+    """
+    try:
+        from hestia_earth.orchestrator import run
+
+        return run(node, {"models": models})
+    except Exception as e:
+        logger.error(f"Error running models: {str(e)}")
+        return node

package/hestia_earth/validation/preload_requests.py

@@ -0,0 +1,61 @@
+"""
+Preload all search requests to avoid making the same searches many times while running models.
+"""
+
+import json
+import os
+from hestia_earth.utils.storage import _load_from_storage
+
+from .log import logger
+from .terms import RESULTS_PATH, get_all_terms, enable_mock
+
+
+def _write_results(filepath: str, data: dict):
+    with open(filepath, "w") as f:
+        f.write(json.dumps(data, ensure_ascii=False))
+
+
+def _load_data_from_glossary():
+    try:
+        return json.loads(
+            _load_from_storage(
+                os.path.join("glossary", "validation-search-results.json"),
+                glossary=True,
+            )
+        )
+    except Exception:
+        return None
+
+
+def enable_preload(
+    filepath: str = RESULTS_PATH,
+    overwrite_existing: bool = True,
+    use_glossary: bool = False,
+):
+    """
+    Prefetch calls to HESTIA API in a local file.
+
+    Parameters
+    ----------
+    filepath : str
+        The path of the file containing the search results. Defaults to current library folder.
+    overwrite_existing : bool
+        Optional - If the file already exists, the file can be used instead of generating it again.
+        Will overwrite by default.
+    use_glossary : bool
+        Optional - Try to fetch search results from the glossary.
+        Only available with access to HESTIA infrastructure.
+    """
+    should_generate = overwrite_existing or not os.path.exists(filepath)
+
+    if should_generate:
+        logger.debug("Preloading search results and storing in %s", filepath)
+
+        # build the search results
+        data = (_load_data_from_glossary() if use_glossary else None) or get_all_terms()
+
+        # store in file
+        _write_results(filepath, data)
+
+    # enable mock search results from file
+    enable_mock(filepath=filepath)