@hestia-earth/data-validation 0.37.7 → 0.37.8

package/.gitlab/issue_templates/new validation.md

@@ -1,82 +0,0 @@
-# New Validation
-
-## Validation
-
-<!-- Please describe what the validation does -->
-
-### Validation Conditions
-
-<!--
-
-Please describe the conditions when the validation will return the error/warning message.
-
-Example:
-- If the Cycle `functionalUnit` is equal to `1 ha`
-- If the primary Product `units` is equal to `ha`
-- If the product `value` is equal to `0`
-- Then show error
-
--->
-
-### Validation Level
-
-<!-- Please select the validation level by entering an "x" instead of the space in the brackets -->
-
-- [ ] Warning: this might be an error, but we will still allow the upload to be validated.
-- [ ] Error: this is an error and must be fixed to validate the upload.
-
-## Example
-
-<!-- 
-
-Please provide an example of data to validate and the validation message below.
-
-Note: you can write the example in a Excel (export to CSV) or directly a CSV file, and use the [Convert Files tool](https://www.hestia.earth/files/tools) on the platform to convert to JSON.
-
--->
-
-### Input Data
-
-<!--
-Example:
-
-```
-{
-  "@type": "Cycle",
-  "functionalUnit: "1ha",
-  "products": [
-    {
-      "@type": "Product",
-      "term": {
-        "units": "ha"
-      },
-      "value": [
-        0
-      ]
-    }
-  ]
-}
-```
--->
-
-
-### Output error/warning message
-
-<!--
-Use the following structure:
-- Describe error.
-- Say why important to fix (especially for warnings where they are optional).
-- Describe how to fix (if not obvious).
-
-```
-The value for the a Product in ha must be above `0` or we can not calculate an impact.
-```
--->
-
-## Additional Notes
-
-<!-- Add further notes if necessary -->
-
-/label ~"Type::feature"
-/label ~"Priority::LOW"
-/label ~"Tracking::Triage"

package/.gitlab-ci.yml

@@ -1,216 +0,0 @@
-default:
-  image: python:3.12
-  before_script:
-    - pip install -r requirements.txt
-
-stages:
-  - test
-  - build
-  - deploy
-  - post-deploy
-  # needs to happen after deploy version
-  - build-layer
-  - deploy-layer
-
-lint-commit:
-  image: node:22
-  stage: test
-  before_script:
-    - npm install
-  script:
-    # only fetch the last 100 commits on develop to check against
-    - git fetch origin develop --depth=100
-    - npx commitlint --from=origin/develop
-  except:
-    - tags
-    - develop
-    - master
-
-lint:
-  stage: test
-  before_script:
-    - pip install flake8 black
-  script:
-    - flake8
-    - black --check .
-  except:
-    - tags
-
-validate-jsonld:
-  image: node:22
-  stage: test
-  before_script:
-    - npm install
-  script:
-    - npm test
-  except:
-    - tags
-
-test:
-  stage: test
-  variables:
-    AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID_RO
-    AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY_RO
-  before_script:
-    - pip install -r requirements-test.txt
-  script:
-    - pytest -n 5 --cov-config=.coveragerc --cov-report term --cov-report html --cov-report xml --cov=./ --cov-fail-under=95
-  coverage: /Total coverage\s*:\s*([^%]+)/
-  artifacts:
-    paths:
-      - coverage/
-    reports:
-      coverage_report:
-        coverage_format: cobertura
-        path: coverage.xml
-  except:
-    - tags
-
-build-mocking:
-  stage: build
-  script:
-    - python build_mocking.py search-results.json
-  artifacts:
-    paths:
-      - search-results.json
-  only:
-    - develop
-    - tags
-
-build-validation-json:
-  stage: build
-  script:
-    - python scripts/build_validation_list.py --input-folder hestia_earth/validation/validators --output-file validation.json
-  artifacts:
-    paths:
-      - validation.json
-  only:
-    - develop
-    - tags
-
-deploy-dev:
-  stage: deploy
-  script:
-    - cp search-results.json hestia_earth/validation/.
-    - pip install twine setuptools wheel
-    - python setup.py sdist bdist_wheel
-    - twine upload -u $PYPI_USERNAME -p $PYPI_TOKEN_TEST --repository-url https://test.pypi.org/legacy/ dist/* || true
-  only:
-    - develop
-
-deploy:
-  stage: deploy
-  script:
-    - cp search-results.json hestia_earth/validation/.
-    - pip install twine setuptools wheel
-    - python setup.py sdist bdist_wheel
-    - twine upload -u $PYPI_USERNAME -p $PYPI_TOKEN --skip-existing dist/*
-  only:
-    - tags
-
-docs-python:
-  stage: deploy
-  artifacts:
-    paths:
-      - public/
-  before_script:
-    - cd docs
-    - pip install -r requirements.txt
-  script:
-    - make html
-    - cd ../
-    - mv docs/_build/html public
-  only:
-    - develop
-    - tags
-
-build-layer:
-  image: docker:stable
-  stage: build-layer
-  services:
-    - docker:dind
-  artifacts:
-    paths:
-      - layer/python
-  before_script:
-    - chmod +x layer/build.sh
-  script:
-    - ./layer/build.sh
-  only:
-    - develop
-    - master
-
-deploy-layer:
-  stage: deploy-layer
-  before_script:
-    - source envs/.${CI_COMMIT_REF_NAME}.env
-    - pip install awscli
-    - apt-get update && apt-get install -y zip
-    - chmod +x layer/deploy.sh
-  script:
-    - ./layer/deploy.sh $STAGE
-  only:
-    - develop
-    - master
-
-deploy-guide:
-  stage: deploy
-  trigger:
-    project: hestia-earth/hestia-guide
-    branch: ${CI_COMMIT_REF_NAME}
-  only:
-    - develop
-    - master
-
-.delete-guide-dev:
-  before_script:
-    - BRANCH_NAME=$(echo "hestia-data-validation${CI_COMMIT_BRANCH}" | sed 's/\//-/g')
-    - curl --request DELETE --header "PRIVATE-TOKEN:${GITLAB_TOKEN_GUIDE}" "https://gitlab.com/api/v4/projects/${GITLAB_GUIDE_PROJECT_ID}/repository/branches/${BRANCH_NAME}"
-
-deploy-guide-dev:
-  stage: deploy
-  environment:
-    name: $CI_COMMIT_BRANCH
-    url: https://www-dev.hestia.earth/guide?branch=hestia-data-validation${CI_COMMIT_BRANCH}
-    on_stop: remove-guide-dev
-  extends:
-   - .delete-guide-dev
-  script:
-    - apt-get update && apt-get install -y jq
-    - ./scripts/guide-create-branch.sh "${BRANCH_NAME}" "${GITLAB_TOKEN_GUIDE}"
-  only:
-    - branches
-  except:
-    - develop
-    - master
-
-remove-guide-dev:
-  stage: post-deploy
-  when: manual
-  variables:
-    GIT_STRATEGY: none
-  environment:
-    name: $CI_COMMIT_BRANCH
-    action: stop
-  extends:
-   - .delete-guide-dev
-  script:
-    - echo "Done"
-  only:
-    - branches
-  except:
-    - develop
-    - master
-
-publish-js:
-  image: node:22
-  stage: deploy
-  before_script:
-    - npm install
-  script:
-    - npm run build:module
-    - npm config set //registry.npmjs.org/:_authToken=${NPM_TOKEN}
-    - rm *.md
-    - npm publish --access public --verbose
-  only:
-    - tags

package/.readthedocs.yml

@@ -1,24 +0,0 @@
-# .readthedocs.yml
-# Read the Docs configuration file
-# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
-
-# Required
-version: 2
-
-# Build documentation in the docs/ directory with Sphinx
-sphinx:
-  configuration: docs/conf.py
-
-# Build documentation with MkDocs
-#mkdocs:
-#  configuration: mkdocs.yml
-
-# Optionally build your docs in additional formats such as PDF
-#formats:
-#  - pdf
-
-# Optionally set the version of Python and requirements required to build your docs
-python:
-  version: 3.7
-  install:
-    - requirements: docs/requirements.txt

package/CODEOWNERS

@@ -1,11 +0,0 @@
-# admins
-CODEOWNERS @mebibou
-/envs/ @mebibou
-.gitlab-ci.yml @mebibou
-setup.py @mebibouu
-layer/* @hestia-earth @mebibou
-
-# hestia
-CONTRIBUTING.md @hestia-earth
-LICENSE @hestia-earth
-README.md @hestia-earth

package/Dockerfile

@@ -1,13 +0,0 @@
-FROM python:3.12
-
-WORKDIR /app
-
-RUN pip install python-dotenv
-
-COPY requirements.txt /app/requirements.txt
-COPY requirements-ci.txt /app/requirements-ci.txt
-RUN pip install -r requirements-ci.txt
-
-COPY . /app
-
-CMD run.py

package/MANIFEST.in

@@ -1,2 +0,0 @@
-include README.md
-include hestia_earth/validation/mocking/*.json

package/bin/hestia-validate-data

@@ -1,80 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import sys
-import glob
-import json
-from hestia_earth.utils.tools import flatten
-from hestia_earth.validation.preload_requests import enable_preload
-from hestia_earth.validation import validate
-import numpy as np
-
-
-# fix error "Object of type int64 is not JSON serializable"
-class NpEncoder(json.JSONEncoder):
-    def default(self, obj):
-        if isinstance(obj, np.integer):
-            return int(obj)
-        if isinstance(obj, np.floating):
-            return float(obj)
-        if isinstance(obj, np.ndarray):
-            return obj.tolist()
-        return super(NpEncoder, self).default(obj)
-
-
-# disable GEE_API
-os.environ['VALIDATE_SPATIAL'] = 'false'
-# enable validation on all nodes
-os.environ['VALIDATE_EXISTING_NODES'] = 'true'
-EXTENSIONS = [
-  'jsonld',
-  'json',
-  'hestia'
-]
-
-
-def list_files(folder: str):
-    return flatten([
-        list(glob.iglob(f"{folder}/**/*.{ext}", recursive=True)) for ext in EXTENSIONS
-    ])
-
-
-def filter_by_level(values: list, level: str):
-    return list(filter(lambda v: isinstance(v, dict) and v.get('level', 'error') == level, values))
-
-
-def error_to_string(errors: list, warnings: list):
-    return json.dumps({'errors': errors, 'warnings': warnings}, indent=2, cls=NpEncoder)
-
-
-def validate_file(filepath: str):
-    with open(filepath) as f:
-        data = json.load(f)
-
-    print('Validating file', filepath)
-    nodes = data if isinstance(data, list) else data.get('nodes', [data])
-    results = validate(nodes)
-    errors = list(map(lambda v: filter_by_level(v, 'error'), results))
-    warnings = list(map(lambda v: filter_by_level(v, 'warning'), results))
-    has_errors = next((True for x in errors if len(x) > 0), False)
-    return has_errors, errors, warnings
-
-
-def main(args: list):
-    folder, = args
-    files = list_files(folder)
-    exit_code = ''
-
-    enable_preload(use_glossary=True)
-
-    for file in files:
-        has_errors, errors, warnings = validate_file(file)
-        if has_errors:
-            error = error_to_string(errors, warnings)
-            exit_code += f"Validation of {file} failed:\n{error}\n"
-
-    return 0 if exit_code == '' else exit_code
-
-
-if __name__ == '__main__':
-    sys.exit(main(sys.argv[1:]))

package/build_mocking.py

@@ -1,14 +0,0 @@
-import sys
-import json
-from hestia_earth.validation.terms import RESULTS_PATH, get_all_terms
-
-
-def main(args: list):
-    filepath = args[0] if len(args) > 0 else RESULTS_PATH
-    data = get_all_terms()
-    with open(filepath, "w") as f:
-        f.write(json.dumps(data, indent=2, ensure_ascii=False))
-
-
-if __name__ == "__main__":
-    main(sys.argv[1:])

package/commitlint.config.js

@@ -1 +0,0 @@
-module.exports = {extends: ['@commitlint/config-conventional']}

package/docs/Makefile

@@ -1,20 +0,0 @@
-# Minimal makefile for Sphinx documentation
-#
-
-# You can set these variables from the command line, and also
-# from the environment for the first two.
-SPHINXOPTS    ?=
-SPHINXBUILD   ?= sphinx-build
-SOURCEDIR     = .
-BUILDDIR      = _build
-
-# Put it first so that "make" without argument is like "make help".
-help:
-	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
-
-.PHONY: help Makefile
-
-# Catch-all target: route all unknown targets to Sphinx using the new
-# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
-%: Makefile
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

package/docs/_static/styles.css

@@ -1,4 +0,0 @@
-.wy-nav-content {
-  max-width: none;
-  min-height: 100vh;
-}

package/docs/_templates/custom-class-template.rst

@@ -1,34 +0,0 @@
-{{ fullname | escape | underline}}
-
-.. currentmodule:: {{ module }}
-
-.. autoclass:: {{ objname }}
-   :members:
-   :show-inheritance:
-   :inherited-members:
-   :special-members: __call__, __add__, __mul__
-
-   {% block methods %}
-   {% if methods %}
-   .. rubric:: {{ _('Methods') }}
-
-   .. autosummary::
-      :nosignatures:
-   {% for item in methods %}
-      {%- if not item.startswith('_') %}
-      ~{{ name }}.{{ item }}
-      {%- endif -%}
-   {%- endfor %}
-   {% endif %}
-   {% endblock %}
-
-   {% block attributes %}
-   {% if attributes %}
-   .. rubric:: {{ _('Attributes') }}
-
-   .. autosummary::
-   {% for item in attributes %}
-      ~{{ name }}.{{ item }}
-   {%- endfor %}
-   {% endif %}
-   {% endblock %}

package/docs/_templates/custom-module-template.rst

@@ -1,66 +0,0 @@
-{{ fullname | escape | underline}}
-
-.. automodule:: {{ fullname }}
-
-   {% block attributes %}
-   {% if attributes %}
-   .. rubric:: Module attributes
-
-   .. autosummary::
-      :toctree:
-   {% for item in attributes %}
-      {{ item }}
-   {%- endfor %}
-   {% endif %}
-   {% endblock %}
-
-   {% block functions %}
-   {% if functions %}
-   .. rubric:: {{ _('Functions') }}
-
-   .. autosummary::
-      :toctree:
-      :nosignatures:
-   {% for item in functions %}
-      {{ item }}
-   {%- endfor %}
-   {% endif %}
-   {% endblock %}
-
-   {% block classes %}
-   {% if classes %}
-   .. rubric:: {{ _('Classes') }}
-
-   .. autosummary::
-      :toctree:
-      :template: custom-class-template.rst
-      :nosignatures:
-   {% for item in classes %}
-      {{ item }}
-   {%- endfor %}
-   {% endif %}
-   {% endblock %}
-
-   {% block exceptions %}
-   {% if exceptions %}
-   .. rubric:: {{ _('Exceptions') }}
-
-   .. autosummary::
-      :toctree:
-   {% for item in exceptions %}
-      {{ item }}
-   {%- endfor %}
-   {% endif %}
-   {% endblock %}
-
-{% block modules %}
-{% if modules %}
-.. autosummary::
-   :toctree:
-   :template: custom-module-template.rst
-   :recursive:
-{% for item in modules %}
-   {{ item }}
-{%- endfor %}
-{% endif %}
-{% endblock %}

package/docs/_templates/layout.html

@@ -1,4 +0,0 @@
-{% extends "!layout.html" %}
-{% block extrahead %}
-    <link href="{{ pathto("_static/styles.css", True) }}" rel="stylesheet" type="text/css">
-{% endblock %}

package/docs/conf.py

@@ -1,74 +0,0 @@
-# Configuration file for the Sphinx documentation builder.
-#
-# This file only contains a selection of the most common options. For a full
-# list see the documentation:
-# https://www.sphinx-doc.org/en/master/usage/configuration.html
-
-# -- Path setup --------------------------------------------------------------
-
-# If extensions (or modules to document with autodoc) are in another directory,
-# add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
-#
-import os
-import sys
-
-sys.path.insert(0, os.path.abspath(".."))
-
-
-# -- Project information -----------------------------------------------------
-
-project = "HESTIA Data Validation"
-copyright = "2020, HESTIA Team"
-author = "Guillaume Royer"
-
-# The full version, including alpha/beta/rc tags
-release = "0.0.1"
-
-
-# -- General configuration ---------------------------------------------------
-
-# Add any Sphinx extension module names here, as strings. They can be
-# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
-# ones.
-extensions = [
-    "recommonmark",
-    "sphinx.ext.autodoc",
-    "sphinx.ext.napoleon",
-    "sphinx.ext.autosummary",
-    "sphinx.ext.viewcode",
-    "sphinx_autodoc_typehints",
-]
-
-autosummary_generate = True
-
-# Add any paths that contain templates here, relative to this directory.
-templates_path = ["_templates"]
-
-# List of patterns, relative to source directory, that match files and
-# directories to ignore when looking for source files.
-# This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
-
-
-# -- Options for HTML output -------------------------------------------------
-
-# The theme to use for HTML and HTML Help pages.  See the documentation for
-# a list of builtin themes.
-#
-html_theme = "sphinx_rtd_theme"
-
-# -- Options for HTML output -------------------------------------------------
-
-# on_rtd is whether on readthedocs.org, this line of code grabbed from docs.readthedocs.org...
-on_rtd = os.getenv("READTHEDOCS", None) == "True"
-if not on_rtd:  # only import and set the theme if we're building docs locally
-    import sphinx_rtd_theme
-
-    html_theme = "sphinx_rtd_theme"
-    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
-
-# Add any paths that contain custom static files (such as style sheets) here,
-# relative to this directory. They are copied after the builtin static files,
-# so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ["_static"]

package/docs/index.rst

@@ -1,42 +0,0 @@
-Welcome to HESTIA Data Validation's documentation!
-========================================
-
-This package contains the Data Validation module developed by HESTIA.
-
-Installation
-------------
-
-Install from `PyPI <https://pypi.python.org/pypi>`_ using `pip <http://www.pip-installer.org/en/latest/>`_, a
-package manager for Python.
-
-.. code-block:: bash
-
-    pip install hestia_earth.validation
-
-
-Requirements
-============
-
-- `hestia_earth.schema <https://pypi.org/project/hestia-earth.schema/>`_
-- `requests <https://pypi.org/project/requests/>`_
-- `python-dateutil <https://pypi.org/project/python-dateutil/>`_
-- `area <https://pypi.org/project/area/>`_
-
-Contents
---------
-
-.. autosummary::
-   :toctree: _autosummary
-   :caption: API Reference
-   :template: custom-module-template.rst
-   :recursive:
-
-   hestia_earth.validation
-
-
-Indices and tables
-==================
-
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`