@hestia-earth/data-validation 0.37.7

package/.coveragerc

@@ -0,0 +1,14 @@
+[run]
+omit =
+    setup.py
+    build_mocking.py
+    validateData.py
+    run.py
+    **/version.py
+    tests/*
+    hestia_earth/validation/log.py
+    hestia_earth/validation/gee.py
+    hestia_earth/validation/mocking/*
+
+[html]
+directory=coverage

package/.dockerignore

@@ -0,0 +1,19 @@
+bin/
+coverage/
+dist/
+docs/
+node_modules/
+samples/
+scripts/
+tmp/
+envs/
+.coveragerc
+.flake8
+*.json
+*.md
+*.sh
+*.yml
+.git
+.gitlab
+layer/python/
+*.zip

package/.eslintignore

@@ -0,0 +1,17 @@
+.eslintrc.js
+coverage
+build
+build-lambdas
+commitlint.config.js
+.mocharc.js
+alembic/
+app/
+data/
+dist/
+doc/
+layer/
+load_testing/
+rabbit/
+samples/
+scripts/
+jest.config.js

package/.eslintrc.js

@@ -0,0 +1,11 @@
+module.exports = {
+  env: {
+    node: true
+  },
+  extends: ['@hestia-earth/eslint-config'],
+  parser: '@typescript-eslint/parser',
+  parserOptions: {
+    project: 'tsconfig.json',
+    sourceType: 'module'
+  }
+};

package/.flake8

@@ -0,0 +1,5 @@
+[flake8]
+exclude = layer, scripts, run.py
+ignore = E402, E704, E203, W503, E226
+max-line-length = 120
+max-complexity = 4

package/.gitlab/issue_templates/new validation.md

@@ -0,0 +1,82 @@
+# New Validation
+
+## Validation
+
+<!-- Please describe what the validation does -->
+
+### Validation Conditions
+
+<!--
+
+Please describe the conditions when the validation will return the error/warning message.
+
+Example:
+- If the Cycle `functionalUnit` is equal to `1 ha`
+- If the primary Product `units` is equal to `ha`
+- If the product `value` is equal to `0`
+- Then show error
+
+-->
+
+### Validation Level
+
+<!-- Please select the validation level by entering an "x" instead of the space in the brackets -->
+
+- [ ] Warning: this might be an error, but we will still allow the upload to be validated.
+- [ ] Error: this is an error and must be fixed to validate the upload.
+
+## Example
+
+<!-- 
+
+Please provide an example of data to validate and the validation message below.
+
+Note: you can write the example in a Excel (export to CSV) or directly a CSV file, and use the [Convert Files tool](https://www.hestia.earth/files/tools) on the platform to convert to JSON.
+
+-->
+
+### Input Data
+
+<!--
+Example:
+
+```
+{
+  "@type": "Cycle",
+  "functionalUnit: "1ha",
+  "products": [
+    {
+      "@type": "Product",
+      "term": {
+        "units": "ha"
+      },
+      "value": [
+        0
+      ]
+    }
+  ]
+}
+```
+-->
+
+
+### Output error/warning message
+
+<!--
+Use the following structure:
+- Describe error.
+- Say why important to fix (especially for warnings where they are optional).
+- Describe how to fix (if not obvious).
+
+```
+The value for the a Product in ha must be above `0` or we can not calculate an impact.
+```
+-->
+
+## Additional Notes
+
+<!-- Add further notes if necessary -->
+
+/label ~"Type::feature"
+/label ~"Priority::LOW"
+/label ~"Tracking::Triage"

package/.gitlab-ci.yml

@@ -0,0 +1,216 @@
+default:
+  image: python:3.12
+  before_script:
+    - pip install -r requirements.txt
+
+stages:
+  - test
+  - build
+  - deploy
+  - post-deploy
+  # needs to happen after deploy version
+  - build-layer
+  - deploy-layer
+
+lint-commit:
+  image: node:22
+  stage: test
+  before_script:
+    - npm install
+  script:
+    # only fetch the last 100 commits on develop to check against
+    - git fetch origin develop --depth=100
+    - npx commitlint --from=origin/develop
+  except:
+    - tags
+    - develop
+    - master
+
+lint:
+  stage: test
+  before_script:
+    - pip install flake8 black
+  script:
+    - flake8
+    - black --check .
+  except:
+    - tags
+
+validate-jsonld:
+  image: node:22
+  stage: test
+  before_script:
+    - npm install
+  script:
+    - npm test
+  except:
+    - tags
+
+test:
+  stage: test
+  variables:
+    AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID_RO
+    AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY_RO
+  before_script:
+    - pip install -r requirements-test.txt
+  script:
+    - pytest -n 5 --cov-config=.coveragerc --cov-report term --cov-report html --cov-report xml --cov=./ --cov-fail-under=95
+  coverage: /Total coverage\s*:\s*([^%]+)/
+  artifacts:
+    paths:
+      - coverage/
+    reports:
+      coverage_report:
+        coverage_format: cobertura
+        path: coverage.xml
+  except:
+    - tags
+
+build-mocking:
+  stage: build
+  script:
+    - python build_mocking.py search-results.json
+  artifacts:
+    paths:
+      - search-results.json
+  only:
+    - develop
+    - tags
+
+build-validation-json:
+  stage: build
+  script:
+    - python scripts/build_validation_list.py --input-folder hestia_earth/validation/validators --output-file validation.json
+  artifacts:
+    paths:
+      - validation.json
+  only:
+    - develop
+    - tags
+
+deploy-dev:
+  stage: deploy
+  script:
+    - cp search-results.json hestia_earth/validation/.
+    - pip install twine setuptools wheel
+    - python setup.py sdist bdist_wheel
+    - twine upload -u $PYPI_USERNAME -p $PYPI_TOKEN_TEST --repository-url https://test.pypi.org/legacy/ dist/* || true
+  only:
+    - develop
+
+deploy:
+  stage: deploy
+  script:
+    - cp search-results.json hestia_earth/validation/.
+    - pip install twine setuptools wheel
+    - python setup.py sdist bdist_wheel
+    - twine upload -u $PYPI_USERNAME -p $PYPI_TOKEN --skip-existing dist/*
+  only:
+    - tags
+
+docs-python:
+  stage: deploy
+  artifacts:
+    paths:
+      - public/
+  before_script:
+    - cd docs
+    - pip install -r requirements.txt
+  script:
+    - make html
+    - cd ../
+    - mv docs/_build/html public
+  only:
+    - develop
+    - tags
+
+build-layer:
+  image: docker:stable
+  stage: build-layer
+  services:
+    - docker:dind
+  artifacts:
+    paths:
+      - layer/python
+  before_script:
+    - chmod +x layer/build.sh
+  script:
+    - ./layer/build.sh
+  only:
+    - develop
+    - master
+
+deploy-layer:
+  stage: deploy-layer
+  before_script:
+    - source envs/.${CI_COMMIT_REF_NAME}.env
+    - pip install awscli
+    - apt-get update && apt-get install -y zip
+    - chmod +x layer/deploy.sh
+  script:
+    - ./layer/deploy.sh $STAGE
+  only:
+    - develop
+    - master
+
+deploy-guide:
+  stage: deploy
+  trigger:
+    project: hestia-earth/hestia-guide
+    branch: ${CI_COMMIT_REF_NAME}
+  only:
+    - develop
+    - master
+
+.delete-guide-dev:
+  before_script:
+    - BRANCH_NAME=$(echo "hestia-data-validation${CI_COMMIT_BRANCH}" | sed 's/\//-/g')
+    - curl --request DELETE --header "PRIVATE-TOKEN:${GITLAB_TOKEN_GUIDE}" "https://gitlab.com/api/v4/projects/${GITLAB_GUIDE_PROJECT_ID}/repository/branches/${BRANCH_NAME}"
+
+deploy-guide-dev:
+  stage: deploy
+  environment:
+    name: $CI_COMMIT_BRANCH
+    url: https://www-dev.hestia.earth/guide?branch=hestia-data-validation${CI_COMMIT_BRANCH}
+    on_stop: remove-guide-dev
+  extends:
+   - .delete-guide-dev
+  script:
+    - apt-get update && apt-get install -y jq
+    - ./scripts/guide-create-branch.sh "${BRANCH_NAME}" "${GITLAB_TOKEN_GUIDE}"
+  only:
+    - branches
+  except:
+    - develop
+    - master
+
+remove-guide-dev:
+  stage: post-deploy
+  when: manual
+  variables:
+    GIT_STRATEGY: none
+  environment:
+    name: $CI_COMMIT_BRANCH
+    action: stop
+  extends:
+   - .delete-guide-dev
+  script:
+    - echo "Done"
+  only:
+    - branches
+  except:
+    - develop
+    - master
+
+publish-js:
+  image: node:22
+  stage: deploy
+  before_script:
+    - npm install
+  script:
+    - npm run build:module
+    - npm config set //registry.npmjs.org/:_authToken=${NPM_TOKEN}
+    - rm *.md
+    - npm publish --access public --verbose
+  only:
+    - tags

package/.readthedocs.yml

@@ -0,0 +1,24 @@
+# .readthedocs.yml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+
+# Build documentation with MkDocs
+#mkdocs:
+#  configuration: mkdocs.yml
+
+# Optionally build your docs in additional formats such as PDF
+#formats:
+#  - pdf
+
+# Optionally set the version of Python and requirements required to build your docs
+python:
+  version: 3.7
+  install:
+    - requirements: docs/requirements.txt

package/CODEOWNERS

@@ -0,0 +1,11 @@
+# admins
+CODEOWNERS @mebibou
+/envs/ @mebibou
+.gitlab-ci.yml @mebibou
+setup.py @mebibouu
+layer/* @hestia-earth @mebibou
+
+# hestia
+CONTRIBUTING.md @hestia-earth
+LICENSE @hestia-earth
+README.md @hestia-earth

package/Dockerfile

@@ -0,0 +1,13 @@
+FROM python:3.12
+
+WORKDIR /app
+
+RUN pip install python-dotenv
+
+COPY requirements.txt /app/requirements.txt
+COPY requirements-ci.txt /app/requirements-ci.txt
+RUN pip install -r requirements-ci.txt
+
+COPY . /app
+
+CMD run.py

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019-2025 Harmonised Environmental Storage and Tracking of the Impacts of Agriculture (HESTIA) Project
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/MANIFEST.in

@@ -0,0 +1,2 @@
+include README.md
+include hestia_earth/validation/mocking/*.json

package/bin/hestia-validate-data

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import glob
+import json
+from hestia_earth.utils.tools import flatten
+from hestia_earth.validation.preload_requests import enable_preload
+from hestia_earth.validation import validate
+import numpy as np
+
+
+# fix error "Object of type int64 is not JSON serializable"
+class NpEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, np.integer):
+            return int(obj)
+        if isinstance(obj, np.floating):
+            return float(obj)
+        if isinstance(obj, np.ndarray):
+            return obj.tolist()
+        return super(NpEncoder, self).default(obj)
+
+
+# disable GEE_API
+os.environ['VALIDATE_SPATIAL'] = 'false'
+# enable validation on all nodes
+os.environ['VALIDATE_EXISTING_NODES'] = 'true'
+EXTENSIONS = [
+  'jsonld',
+  'json',
+  'hestia'
+]
+
+
+def list_files(folder: str):
+    return flatten([
+        list(glob.iglob(f"{folder}/**/*.{ext}", recursive=True)) for ext in EXTENSIONS
+    ])
+
+
+def filter_by_level(values: list, level: str):
+    return list(filter(lambda v: isinstance(v, dict) and v.get('level', 'error') == level, values))
+
+
+def error_to_string(errors: list, warnings: list):
+    return json.dumps({'errors': errors, 'warnings': warnings}, indent=2, cls=NpEncoder)
+
+
+def validate_file(filepath: str):
+    with open(filepath) as f:
+        data = json.load(f)
+
+    print('Validating file', filepath)
+    nodes = data if isinstance(data, list) else data.get('nodes', [data])
+    results = validate(nodes)
+    errors = list(map(lambda v: filter_by_level(v, 'error'), results))
+    warnings = list(map(lambda v: filter_by_level(v, 'warning'), results))
+    has_errors = next((True for x in errors if len(x) > 0), False)
+    return has_errors, errors, warnings
+
+
+def main(args: list):
+    folder, = args
+    files = list_files(folder)
+    exit_code = ''
+
+    enable_preload(use_glossary=True)
+
+    for file in files:
+        has_errors, errors, warnings = validate_file(file)
+        if has_errors:
+            error = error_to_string(errors, warnings)
+            exit_code += f"Validation of {file} failed:\n{error}\n"
+
+    return 0 if exit_code == '' else exit_code
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))

package/build_mocking.py

@@ -0,0 +1,14 @@
+import sys
+import json
+from hestia_earth.validation.terms import RESULTS_PATH, get_all_terms
+
+
+def main(args: list):
+    filepath = args[0] if len(args) > 0 else RESULTS_PATH
+    data = get_all_terms()
+    with open(filepath, "w") as f:
+        f.write(json.dumps(data, indent=2, ensure_ascii=False))
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

package/commitlint.config.js

@@ -0,0 +1 @@
+module.exports = {extends: ['@commitlint/config-conventional']}

package/docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

package/docs/_static/styles.css

@@ -0,0 +1,4 @@
+.wy-nav-content {
+  max-width: none;
+  min-height: 100vh;
+}

package/docs/_templates/custom-class-template.rst

@@ -0,0 +1,34 @@
+{{ fullname | escape | underline}}
+
+.. currentmodule:: {{ module }}
+
+.. autoclass:: {{ objname }}
+   :members:
+   :show-inheritance:
+   :inherited-members:
+   :special-members: __call__, __add__, __mul__
+
+   {% block methods %}
+   {% if methods %}
+   .. rubric:: {{ _('Methods') }}
+
+   .. autosummary::
+      :nosignatures:
+   {% for item in methods %}
+      {%- if not item.startswith('_') %}
+      ~{{ name }}.{{ item }}
+      {%- endif -%}
+   {%- endfor %}
+   {% endif %}
+   {% endblock %}
+
+   {% block attributes %}
+   {% if attributes %}
+   .. rubric:: {{ _('Attributes') }}
+
+   .. autosummary::
+   {% for item in attributes %}
+      ~{{ name }}.{{ item }}
+   {%- endfor %}
+   {% endif %}
+   {% endblock %}

package/docs/_templates/custom-module-template.rst

@@ -0,0 +1,66 @@
+{{ fullname | escape | underline}}
+
+.. automodule:: {{ fullname }}
+
+   {% block attributes %}
+   {% if attributes %}
+   .. rubric:: Module attributes
+
+   .. autosummary::
+      :toctree:
+   {% for item in attributes %}
+      {{ item }}
+   {%- endfor %}
+   {% endif %}
+   {% endblock %}
+
+   {% block functions %}
+   {% if functions %}
+   .. rubric:: {{ _('Functions') }}
+
+   .. autosummary::
+      :toctree:
+      :nosignatures:
+   {% for item in functions %}
+      {{ item }}
+   {%- endfor %}
+   {% endif %}
+   {% endblock %}
+
+   {% block classes %}
+   {% if classes %}
+   .. rubric:: {{ _('Classes') }}
+
+   .. autosummary::
+      :toctree:
+      :template: custom-class-template.rst
+      :nosignatures:
+   {% for item in classes %}
+      {{ item }}
+   {%- endfor %}
+   {% endif %}
+   {% endblock %}
+
+   {% block exceptions %}
+   {% if exceptions %}
+   .. rubric:: {{ _('Exceptions') }}
+
+   .. autosummary::
+      :toctree:
+   {% for item in exceptions %}
+      {{ item }}
+   {%- endfor %}
+   {% endif %}
+   {% endblock %}
+
+{% block modules %}
+{% if modules %}
+.. autosummary::
+   :toctree:
+   :template: custom-module-template.rst
+   :recursive:
+{% for item in modules %}
+   {{ item }}
+{%- endfor %}
+{% endif %}
+{% endblock %}

package/docs/_templates/layout.html

@@ -0,0 +1,4 @@
+{% extends "!layout.html" %}
+{% block extrahead %}
+    <link href="{{ pathto("_static/styles.css", True) }}" rel="stylesheet" type="text/css">
+{% endblock %}