@hestia-earth/data-validation 0.37.7 → 0.37.9

package/docs/index.rst

@@ -1,42 +0,0 @@
-Welcome to HESTIA Data Validation's documentation!
-========================================
-
-This package contains the Data Validation module developed by HESTIA.
-
-Installation
-------------
-
-Install from `PyPI <https://pypi.python.org/pypi>`_ using `pip <http://www.pip-installer.org/en/latest/>`_, a
-package manager for Python.
-
-.. code-block:: bash
-
-    pip install hestia_earth.validation
-
-
-Requirements
-============
-
-- `hestia_earth.schema <https://pypi.org/project/hestia-earth.schema/>`_
-- `requests <https://pypi.org/project/requests/>`_
-- `python-dateutil <https://pypi.org/project/python-dateutil/>`_
-- `area <https://pypi.org/project/area/>`_
-
-Contents
---------
-
-.. autosummary::
-   :toctree: _autosummary
-   :caption: API Reference
-   :template: custom-module-template.rst
-   :recursive:
-
-   hestia_earth.validation
-
-
-Indices and tables
-==================
-
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`

package/docs/make.bat

@@ -1,35 +0,0 @@
-@ECHO OFF
-
-pushd %~dp0
-
-REM Command file for Sphinx documentation
-
-if "%SPHINXBUILD%" == "" (
-	set SPHINXBUILD=sphinx-build
-)
-set SOURCEDIR=.
-set BUILDDIR=_build
-
-if "%1" == "" goto help
-
-%SPHINXBUILD% >NUL 2>NUL
-if errorlevel 9009 (
-	echo.
-	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
-	echo.installed, then set the SPHINXBUILD environment variable to point
-	echo.to the full path of the 'sphinx-build' executable. Alternatively you
-	echo.may add the Sphinx directory to PATH.
-	echo.
-	echo.If you don't have Sphinx installed, grab it from
-	echo.http://sphinx-doc.org/
-	exit /b 1
-)
-
-%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
-goto end
-
-:help
-%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
-
-:end
-popd

package/docs/requirements.txt

@@ -1,13 +0,0 @@
-hestia_earth.validation
-docutils==0.17.1
-Jinja2==3.0.3
-alabaster==0.7.13
-sphinx-rtd-theme==0.5.0
-sphinxcontrib-applehelp==1.0.4
-sphinxcontrib-devhelp==1.0.2
-sphinxcontrib-qthelp==1.0.3
-sphinxcontrib-htmlhelp==2.0.1
-sphinxcontrib-serializinghtml==1.1.5
-sphinx-autodoc-typehints==1.12.0
-sphinx==3.3.1
-recommonmark

package/envs/.develop.env

@@ -1 +0,0 @@
-export STAGE='dev'

package/envs/.master.env

@@ -1 +0,0 @@
-export STAGE='prod'

package/hestia_earth/validation/README.md

@@ -1,5 +0,0 @@
-# HESTIA Data Validation
-
-See the complete list of validations available on the HESTIA format.
-
-Each validation is divided by node type: `Cycle`, `Site`, `ImpactAssessment`, and `Organisation`.

package/hestia_earth/validation/__init__.py

@@ -1,32 +0,0 @@
-from typing import List
-from concurrent.futures import ThreadPoolExecutor
-from hestia_earth.utils.tools import current_time_ms
-
-from .log import logger
-from .validators import validate_node
-from .utils import _group_nodes, _hash_nodes
-from .gee import init_gee_by_nodes
-
-
-def validate(nodes: List[dict]):
-    """
-    Validates a list of HESTIA JSON-Nodes against a list of rules.
-
-    Parameters
-    ----------
-    nodes : List[dict]
-        The list of JSON-Nodes to validate.
-
-    Returns
-    -------
-    List
-        The list of errors for each node, which can be empty if no errors detected.
-    """
-    now = current_time_ms()
-    nodes = init_gee_by_nodes(nodes)
-    nodes_by_type = _group_nodes(nodes)
-    nodes_by_hash = _hash_nodes(nodes)
-    with ThreadPoolExecutor() as executor:
-        results = list(executor.map(validate_node(nodes_by_type, nodes_by_hash), nodes))
-    logger.info("time=%s, unit=ms", current_time_ms() - now)
-    return results

package/hestia_earth/validation/distribution.py

@@ -1,22 +0,0 @@
-import os
-from functools import lru_cache
-
-from .log import logger
-
-ENABLED = os.getenv("VALIDATE_DISTRIBUTION", "true") == "true"
-
-
-@lru_cache()
-def is_enabled():
-    if ENABLED:
-        try:
-            from hestia_earth.distribution.version import VERSION
-
-            logger.debug("Using distribution version %s", VERSION)
-            return True
-        except ImportError:
-            logger.error(
-                "Run `pip install hestia-earth-validation[distribution]` to use distribution validation"
-            )
-
-    return False

package/hestia_earth/validation/gee.py

@@ -1,162 +0,0 @@
-import os
-from hestia_earth.utils.tools import current_time_ms
-from hestia_earth.schema import NodeType
-
-from .log import logger
-
-_ENABLED = os.getenv("VALIDATE_SPATIAL", "true") == "true"
-_ENABLE_TYPES = [NodeType.SITE.value, NodeType.ORGANISATION.value]
-MAX_AREA_SIZE = int(os.getenv("MAX_AREA_SIZE", "5000"))
-_CACHE_BATCH_SIZE = int(os.getenv("CACHE_SITES_BATCH_SIZE", "5000"))
-
-_caching = {}
-_RASTERS = [
-    {
-        "name": "siteType",
-        "collection": "MODIS/006/MCD12Q1",
-        "band_name": "LC_Prop2",
-        "year": "2019",
-    }
-]
-_VECTORS = [
-    {
-        "name": f"region-{level}",
-        "collection": f"users/hestiaplatform/gadm36_{level}",
-        "fields": f"GID_{level}",
-    }
-    for level in range(0, 6)
-]
-
-
-def _caching_key(func_name: str, args: dict):
-    return "-".join([func_name, str(args)])
-
-
-def _run_with_cache(func_name: str, args: dict, func):
-    global _caching  # noqa: F824
-    key = _caching_key(func_name, args)
-    _caching[key] = _caching.get(key, func())
-    return _caching[key]
-
-
-def _should_cache_node(node: dict):
-    return all(
-        [
-            node.get("@type", node.get("type")) in _ENABLE_TYPES,
-            not node.get("aggregated", False),
-            "latitude" in node and "longitude" in node,
-        ]
-    )
-
-
-def _node_key(node: dict):
-    return "/".join(
-        [node.get("type", node.get("@type")), node.get("id", node.get("@id"))]
-    )
-
-
-def _pop_items(values: list, nb_items: int):
-    if len(values) < nb_items:
-        removed_items = values[:]  # Get a copy of the entire array
-        values.clear()  # Remove all items from the original array
-    else:
-        removed_items = values[:nb_items]  # Get the first N items
-        del values[:nb_items]  # Remove the first N items from the original array
-
-    return removed_items
-
-
-def _cache_nodes(nodes: list, batch_size: int):
-    from hestia_earth.models.cache_sites import ParamType, _run_values
-
-    now = current_time_ms()
-
-    nodes_mapping = {_node_key(n): n for n in nodes}
-
-    cache_nodes = list(filter(_should_cache_node, nodes))
-    while len(cache_nodes) > 0:
-        batch_values = _pop_items(cache_nodes, batch_size)
-        logger.info(f"Caching {len(batch_values)} nodes. {len(cache_nodes)} remaining.")
-        results = _run_values(
-            [(n, 0) for n in batch_values],  # expecting tuple with area_size
-            ParamType.COORDINATES,
-            _RASTERS,
-            _VECTORS,
-            years=[],
-        )
-
-        for result in results:
-            nodes_mapping[_node_key(result)] = result
-
-    logger.info("Done caching in %sms", current_time_ms() - now)
-
-    return list(nodes_mapping.values())
-
-
-def _init_gee_by_nodes(nodes: list):
-    from hestia_earth.earth_engine import init_gee
-
-    init_gee()
-    try:
-        return _cache_nodes(nodes, _CACHE_BATCH_SIZE)
-    except Exception as e:
-        logger.error(f"An error occured while caching nodes on EE: {str(e)}")
-        if "User memory limit exceeded" in str(e) or "query aborted" in str(e):
-            return _cache_nodes(nodes, 100)
-
-
-def init_gee_by_nodes(nodes: list):
-    # need to validate for non-aggregated Site or Oganisation with coordinates
-    enabled_nodes = list(filter(_should_cache_node, nodes))
-    should_init = len(enabled_nodes) > 0
-    return _init_gee_by_nodes(nodes) if should_init and is_enabled() else nodes
-
-
-def is_enabled():
-    if _ENABLED:
-        try:
-            from hestia_earth.earth_engine.version import VERSION
-
-            return isinstance(VERSION, str)
-        except ImportError:
-            logger.error(
-                "Run `pip install hestia_earth.earth_engine` to use geospatial validation"
-            )
-
-    return False
-
-
-def id_to_level(id: str):
-    return id.count(".")
-
-
-def get_cached_data(site: dict, key: str, year: int = None):
-    from hestia_earth.models.geospatialDatabase.utils import _cached_value
-
-    value = _cached_value(site, key)
-    return value.get(str(year)) if value and year else value
-
-
-def get_region_id(node: dict):
-    level = id_to_level(node.get("region", node.get("country")).get("@id"))
-    id = get_cached_data(node, f"region-{level}")
-    return None if id is None else f"GADM-{id}"
-
-
-def get_region_distance(gid: str, latitude: float, longitude: float):
-    def exec_func():
-        return round(
-            get_distance_to_coordinates(gid, latitude=latitude, longitude=longitude)
-            / 1000
-        )
-
-    try:
-        from hestia_earth.earth_engine.gadm import get_distance_to_coordinates
-
-        return _run_with_cache(
-            "get_region_distance",
-            {"gid": gid, "latitude": latitude, "longitude": longitude},
-            exec_func,
-        )
-    except Exception:
-        return None

package/hestia_earth/validation/log.py

@@ -1,44 +0,0 @@
-import os
-import logging
-
-
-def _add_handler(
-    logger: logging.Logger, formatter: logging.Formatter, handler: logging.Handler
-):
-    handler.setFormatter(formatter)
-    logger.addHandler(handler)
-
-
-LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO")
-logger = logging.getLogger("hestia_earth.validation")
-logger.setLevel(logging.getLevelName(LOG_LEVEL))
-
-
-def log_to_file(filepath: str):
-    """
-    By default, all logs are saved into a file with path stored in the env variable `LOG_FILENAME`.
-    If you do not set the environment variable `LOG_FILENAME`, you can use this function with the file path.
-
-    Parameters
-    ----------
-    filepath : str
-        Path of the file.
-    """
-    formatter = logging.Formatter(
-        '{"timestamp": "%(asctime)s", "level": "%(levelname)s", "logger": "%(name)s", '
-        '"filename": "%(filename)s", "message": "%(message)s"}',
-        "%Y-%m-%dT%H:%M:%S%z",
-    )
-    handler = logging.FileHandler(filepath, encoding="utf-8")
-    handler.setLevel(logging.getLevelName(LOG_LEVEL))
-    _add_handler(logger, formatter, handler)
-
-
-_formatter = logging.Formatter(
-    "%(asctime)s - %(name)s - %(levelname)s - %(filename)s - %(message)s"
-)
-_add_handler(logger, _formatter, logging.StreamHandler())
-
-LOG_FILENAME = os.getenv("LOG_FILENAME")
-if LOG_FILENAME is not None:
-    log_to_file(LOG_FILENAME)

package/hestia_earth/validation/models.py

@@ -1,141 +0,0 @@
-import importlib
-import os
-from functools import lru_cache
-from hestia_earth.utils.tools import list_sum
-
-from .log import logger
-
-ENABLED = os.getenv("VALIDATE_MODELS", "true") == "true"
-
-
-@lru_cache()
-def is_enabled():
-    if ENABLED:
-        try:
-            from hestia_earth.models.version import VERSION
-
-            logger.debug("Using models version %s", VERSION)
-            return True
-        except ImportError:
-            logger.error(
-                "Run `pip install hestia-earth-validation[models]` to use models validation"
-            )
-
-    return False
-
-
-def _model_value_from_list(results: list, default_value: float):
-    return (
-        results[0].get("value", [default_value]) if len(results) > 0 else default_value
-    )
-
-
-def _model_value(result, default_value=0):
-    return (
-        default_value
-        if result is None
-        else (
-            _model_value_from_list(result, default_value)
-            if isinstance(result, list)
-            else (
-                result.get("value", [default_value])
-                if isinstance(result, dict)
-                else default_value
-            )
-        )
-    )
-
-
-def value_from_model(result):
-    value = _model_value(result)
-    try:
-        # fix numpy.float64
-        # TODO: find a better way to handle this
-        return list_sum(value, float(value))
-    except Exception:
-        return list_sum(value, value)
-
-
-def method_tier_from_model(result):
-    return (
-        None
-        if result is None
-        else (
-            method_tier_from_model(result[0])
-            if isinstance(result, list) and len(result) > 0
-            else (result.get("methodTier") if isinstance(result, dict) else None)
-        )
-    )
-
-
-def _import_model(name: str):
-    return importlib.import_module(f"hestia_earth.models.{name}").run
-
-
-def run_model(model: str, term_id: str, data: dict):
-    """
-    Run a HESTIA model from the engine models library.
-
-    Parameters
-    ----------
-    model : str
-        The name of the model to run.
-    term_id : str
-        The term to run the model on.
-    data : dict
-        The data used to run the model.
-
-    Returns
-    -------
-    Any
-        The result of the model, which can be a single `dict` or a list of `dict`s.
-    """
-    return _import_model(model)(term_id, data)
-
-
-def run_model_from_node(node: dict, data: dict):
-    """
-    Run a HESTIA model from the engine models library.
-    To use this function, you need to use a Blank Node that contains a `methodModel` and a `term`,
-    otherwise you need to use the `run_model` method.
-
-    Parameters
-    ----------
-    node : dict
-        The Blank Node containing a `methodModel` and a `Term`.
-    data : dict
-        The data used to run the model.
-
-    Returns
-    -------
-    Any
-        The result of the model, which can be a single `dict` or a list of `dict`s.
-    """
-    methodModel = node.get("methodModel", {}).get("@id")
-    term_id = node.get("term", {}).get("@id")
-    return run_model(methodModel, term_id, data)
-
-
-def run_models(node: dict, models: list):
-    """
-    Run a list of models, using the orchestrator, and return the updated node.
-
-    Parameters
-    ----------
-    node : dict
-        The Blank Node containing a `methodModel` and a `Term`.
-    models : list
-        List of models as defined in the orchestrator configurations.
-
-    Returns
-    -------
-    Any
-        The recalculated node after running all models
-    """
-    try:
-        from hestia_earth.orchestrator import run
-
-        return run(node, {"models": models})
-    except Exception as e:
-        logger.error(f"Error running models: {str(e)}")
-        return node

package/hestia_earth/validation/preload_requests.py

@@ -1,61 +0,0 @@
-"""
-Preload all search requests to avoid making the same searches many times while running models.
-"""
-
-import json
-import os
-from hestia_earth.utils.storage import _load_from_storage
-
-from .log import logger
-from .terms import RESULTS_PATH, get_all_terms, enable_mock
-
-
-def _write_results(filepath: str, data: dict):
-    with open(filepath, "w") as f:
-        f.write(json.dumps(data, ensure_ascii=False))
-
-
-def _load_data_from_glossary():
-    try:
-        return json.loads(
-            _load_from_storage(
-                os.path.join("glossary", "validation-search-results.json"),
-                glossary=True,
-            )
-        )
-    except Exception:
-        return None
-
-
-def enable_preload(
-    filepath: str = RESULTS_PATH,
-    overwrite_existing: bool = True,
-    use_glossary: bool = False,
-):
-    """
-    Prefetch calls to HESTIA API in a local file.
-
-    Parameters
-    ----------
-    filepath : str
-        The path of the file containing the search results. Defaults to current library folder.
-    overwrite_existing : bool
-        Optional - If the file already exists, the file can be used instead of generating it again.
-        Will overwrite by default.
-    use_glossary : bool
-        Optional - Try to fetch search results from the glossary.
-        Only available with access to HESTIA infrastructure.
-    """
-    should_generate = overwrite_existing or not os.path.exists(filepath)
-
-    if should_generate:
-        logger.debug("Preloading search results and storing in %s", filepath)
-
-        # build the search results
-        data = (_load_data_from_glossary() if use_glossary else None) or get_all_terms()
-
-        # store in file
-        _write_results(filepath, data)
-
-    # enable mock search results from file
-    enable_mock(filepath=filepath)

package/hestia_earth/validation/terms.py

@@ -1,88 +0,0 @@
-import os
-import json
-from enum import Enum
-from hestia_earth.schema import SchemaType, TermTermType
-from hestia_earth.utils.api import search
-
-CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
-RESULTS_PATH = os.path.join(CURRENT_DIR, "search-results.json")
-
-_CACHE = {}
-
-
-def _load_results(filepath: str) -> dict:
-    with open(filepath) as f:
-        return json.load(f)
-
-
-class TERMS_QUERY(Enum):
-    FUEL = TermTermType.FUEL.value
-    CROP_RESIDUE = TermTermType.CROPRESIDUE.value
-    MODEL = TermTermType.MODEL.value
-    FORAGE = TermTermType.FORAGE.value
-    RICE = "rice"
-
-
-_terms_query = {
-    TERMS_QUERY.FUEL: {
-        "should": [
-            {
-                "bool": {
-                    "must": [{"match": {"termType": TermTermType.FUEL.value}}],
-                    "should": [
-                        {"match": {"name": "gasoline"}},
-                        {"match": {"name": "petrol"}},
-                        {"match": {"name": "diesel"}},
-                    ],
-                    "minimum_should_match": 1,
-                }
-            }
-        ],
-        "minimum_should_match": 1,
-    },
-    TERMS_QUERY.CROP_RESIDUE: {
-        "should": [{"match": {"termType": TermTermType.CROPRESIDUE.value}}],
-        "minimum_should_match": 1,
-    },
-    TERMS_QUERY.MODEL: {
-        "should": [{"match": {"termType": TermTermType.MODEL.value}}],
-        "minimum_should_match": 1,
-    },
-    TERMS_QUERY.FORAGE: {
-        "should": [
-            {"match": {"termType.keyword": TermTermType.CROP.value}},
-            {"match": {"termType.keyword": TermTermType.FORAGE.value}},
-            {"match": {"name": "forage"}},
-        ],
-        "minimum_should_match": 2,
-    },
-    TERMS_QUERY.RICE: {
-        "should": [
-            {"match": {"termType.keyword": TermTermType.CROP.value}},
-            {"match": {"name": "rice"}},
-        ],
-        "minimum_should_match": 2,
-    },
-}
-
-
-def _exec_query(query: dict) -> list[str]:
-    terms = search(
-        {"bool": {"must": [{"match": {"@type": SchemaType.TERM.value}}]} | query},
-        limit=10000,
-    )
-    return list(map(lambda n: n["@id"], terms))
-
-
-def get_terms(query: TERMS_QUERY):
-    return _CACHE.get(query.value) or _exec_query(_terms_query[query])
-
-
-def get_all_terms():
-    return {key.value: _exec_query(value) for key, value in _terms_query.items()}
-
-
-def enable_mock(filepath: str):
-    global _CACHE  # noqa: F824
-    _CACHE = _load_results(filepath)
-    return _CACHE