@hestia-earth/data-validation 0.37.7

package/release.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+FROM_BRANCH="develop"
+TO_BRANCH="master"
+
+git checkout $TO_BRANCH
+git merge $FROM_BRANCH
+npm run release
+git checkout $FROM_BRANCH
+git merge $TO_BRANCH
+git push

package/requirements-ci.txt

@@ -0,0 +1,6 @@
+-r requirements.txt
+boto3
+botocore
+hestia-earth-models>=0.75.1
+hestia-earth-earth-engine>=0.6.0
+hestia-earth-distribution>=0.3.1

package/requirements-test.txt

@@ -0,0 +1,4 @@
+-r requirements-ci.txt
+pytest-cov
+pytest-xdist
+coverage

package/requirements.txt

@@ -0,0 +1,2 @@
+hestia-earth-schema>=35.0.1
+hestia-earth-utils>=0.16.9

package/run-docker-test.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+docker build -t hestia-data-validation:test -f tests/Dockerfile .
+
+docker run --rm \
+  --env-file .env \
+  -v ${PWD}:/app \
+  hestia-data-validation:test "$@"

package/run-docker.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+docker build --progress=plain -t hestia-data-validation:latest .
+
+docker run --rm \
+  --name hestia-data-validation \
+  -v ${PWD}:/app \
+  -v ${PWD}/samples:/app/samples \
+  hestia-data-validation:latest python run.py "$@"

package/run.py

@@ -0,0 +1,99 @@
+# usage:
+# 1. Install python-dotenv
+# 2. Set your env variables in a `.env` file
+# 3. Put your test nodes into the `samples` folder
+# 3. Run `python run.py samples/nodes.jsonld`
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+import os
+import json
+import argparse
+from hestia_earth.utils.pipeline import to_string
+from hestia_earth.utils.api import api_url, _safe_get_request, download_hestia
+from hestia_earth.models.preload_requests import enable_preload as models_enable_preload
+from hestia_earth.validation.preload_requests import enable_preload
+from hestia_earth.validation import validate
+
+
+parser = argparse.ArgumentParser(description="Run validation on nodes.")
+parser.add_argument("--file-id", type=str, help="The ID of the file on HESTIA.")
+parser.add_argument(
+    "--filepath", type=str, help="The path of the file containing the nodes."
+)
+parser.add_argument("--node-id", type=str, help="The ID of the node to recalculate.")
+parser.add_argument(
+    "--node-type",
+    type=str,
+    help="The Type of the node to recalculate. Can be determined by the model to run when specified.",
+)
+parser.add_argument(
+    "--data-state",
+    type=str,
+    default="original",
+    choices=["original", "recalculated"],
+    help="DataState of nodes found in file or on the platform.",
+)
+args = parser.parse_args()
+
+
+def _enable_mock_models():
+    models_enable_preload(
+        os.path.join(".", "models-search-results.json"),
+        overwrite_existing=False,
+        use_glossary=True,
+    )
+    enable_preload(
+        os.path.join(".", "validation-search-results.json"),
+        overwrite_existing=False,
+        use_glossary=True,
+    )
+
+
+def _safe_download_node(node: dict, data_state):
+    return download_hestia(
+        node["@id"], node["@type"], data_state=data_state
+    ) or download_hestia(node["@id"], node["@type"])
+
+
+def _download_node(node: dict):
+    data = _safe_download_node(node, data_state=args.data_state)
+    # include site and cycle
+    if data.get("cycle", {}).get("@id"):
+        data["cycle"] = _safe_download_node(data["cycle"], data_state="recalculated")
+    if data.get("site", {}).get("@id"):
+        data["site"] = _safe_download_node(data["site"], data_state="recalculated")
+    return data
+
+
+def main():
+    if args.filepath:
+        print(f"processing {args.filepath}")
+        filename = args.filepath.split(".")[0]
+        with open(args.filepath) as f:
+            data = json.load(f)
+    elif args.file_id:
+        file_data = _safe_get_request(f"{api_url()}/files/{args.file_id}")
+        filename = f"samples/{args.file_id}"
+        data = _safe_get_request(file_data["hestiaPath"])
+        with open(f"{filename}.json", "w") as f:
+            f.write(to_string(data, indent=2))
+    elif args.node_type and args.node_id:
+        data = [_download_node({"@type": args.node_type, "@id": args.node_id})]
+        filename = f"samples/{'-'.join([args.node_type, args.node_id])}"
+        os.environ["VALIDATE_EXISTING_NODES"] = "true"
+
+    _enable_mock_models()
+
+    data = validate(data if isinstance(data, list) else data.get("nodes", [data]))
+
+    dest_filepath = f"{filename}-validated.json"
+    print("Storing results in", dest_filepath)
+    with open(dest_filepath, "w") as f:
+        f.write(to_string(data, indent=2))
+
+
+if __name__ == "__main__":
+    main()

package/scripts/build_docs.py

@@ -0,0 +1,283 @@
+#!/usr/bin/env python3
+
+"""
+Script to extract docstrings from a Python file and all its local imports
+and compile them into a single Markdown file.
+"""
+
+import ast
+import argparse
+import re
+from pathlib import Path
+from typing import List, Set, Optional
+
+# Regex to detect common markdown list items
+LIST_MARKER_REGEX = re.compile(r"^\s*(-|\*|\+|\d+\.)\s")
+
+
+def format_docstring(
+    docstring: str, is_module: bool = False, node_name: Optional[str] = None
+) -> str:
+    """
+    Formats a single docstring into Markdown.
+
+    - First line becomes a header (# for module, ## for func/class).
+    - Body is cleaned.
+    - Sections like "Parameters", "Returns" are skipped.
+    - Backticks are removed from the header.
+    """
+    if not docstring:
+        return ""
+
+    lines = [line.strip() for line in docstring.strip().split("\n")]
+    if not lines:
+        return ""
+
+    header_level = "#" if is_module else "##"
+    # Remove backticks from the header line
+    title = lines[0].strip().replace("`", "")
+
+    if not is_module and node_name:
+        # For functions/classes, add the anchor link
+        header = f"{header_level} [{title}](#{node_name})"
+    else:
+        # For modules, just the title
+        header = f"{header_level} {title}"
+
+    body_lines = []
+    # Stop keywords for sections we want to skip
+    stop_keywords = (
+        "Parameters",
+        "Returns",
+        "Yields",
+        "Raises",
+        "Attributes",
+        "Parameters:",
+        "Returns:",
+        "Yields:",
+        "Raises:",
+        "Attributes:",
+    )
+
+    skipping_section = False
+    for line in lines[1:]:
+        if any(line.strip().startswith(keyword) for keyword in stop_keywords):
+            skipping_section = True
+
+        if not skipping_section:
+            body_lines.append(line)
+
+    # New logic to reflow/unwrap paragraphs
+    reflowed_body = []
+    current_line = ""
+
+    for line in body_lines:
+        if not line:  # Line is empty
+            if current_line:
+                reflowed_body.append(current_line)
+                current_line = ""
+            reflowed_body.append("")  # Keep the empty line
+
+        elif LIST_MARKER_REGEX.match(line):  # Line is a list item
+            if current_line:
+                reflowed_body.append(current_line)
+                current_line = ""
+            reflowed_body.append(line)  # Add list item as is
+
+        elif line.startswith((" ", "\t")):  # Line is indented (code block, etc.)
+            if current_line:
+                reflowed_body.append(current_line)
+                current_line = ""
+            reflowed_body.append(line)  # Add indented line as is
+
+        else:  # Normal text line
+            if not current_line:
+                current_line = line
+            else:
+                current_line += " " + line
+
+    # Add any trailing paragraph
+    if current_line:
+        reflowed_body.append(current_line)
+
+    body = "\n".join(reflowed_body).strip()
+
+    if body:
+        return f"{header}\n\n{body}\n\n"
+    else:
+        return f"{header}\n\n"
+
+
+def extract_documentation(
+    entry_file: str, output_file: str, include_empty: bool = False
+) -> None:
+    """
+    Main function to extract docs and write to the output file.
+    """
+    entry_path = Path(entry_file).resolve()
+    base_dir = entry_path.parent
+    files_to_process: List[Path] = [entry_path]
+    processed_files: Set[Path] = set()
+    all_markdown: List[str] = []
+
+    print(f"Starting documentation extraction from: {entry_file}")
+    print(f"Base directory: {base_dir}")
+
+    while files_to_process:
+        current_file = files_to_process.pop(0)
+        if current_file in processed_files:
+            continue
+
+        print(f"Processing: {current_file.relative_to(base_dir)}")
+        processed_files.add(current_file)
+
+        try:
+            with open(current_file, "r", encoding="utf-8") as f:
+                content = f.read()
+            tree = ast.parse(content)
+        except Exception as e:
+            print(f"  [Error] Could not parse {current_file}: {e}")
+            continue
+
+        # --- 1. Extract docstrings from this file's top-level nodes ---
+
+        # Handle module docstring first
+        module_docstring = ast.get_docstring(tree)
+        if module_docstring:
+            formatted_doc = format_docstring(module_docstring, is_module=True)
+            all_markdown.append(formatted_doc)
+
+        # Handle top-level functions and classes
+        for node in tree.body:
+            if isinstance(node, (ast.FunctionDef, ast.ClassDef, ast.AsyncFunctionDef)):
+                docstring = ast.get_docstring(node)
+                node_name = node.name
+
+                if docstring:
+                    # Found a real docstring, format and add it
+                    formatted_doc = format_docstring(
+                        docstring, is_module=False, node_name=node_name
+                    )
+                    all_markdown.append(formatted_doc)
+
+                elif include_empty and not node_name.startswith("_"):
+                    # No docstring, not private, and flag is enabled:
+                    # Generate a placeholder
+
+                    # 1. Generate title from function name
+                    title = node_name.replace("_", " ").capitalize()
+
+                    # 2. Format as H2 with anchor
+                    header = f"## [{title}](#{node_name})"
+
+                    # 3. Add "Coming soon..." body
+                    body = "Coming soon..."
+
+                    all_markdown.append(f"{header}\n\n{body}\n\n")
+
+        # --- 2. Find new local imports to process ---
+        for node in ast.walk(tree):
+            import_path: Optional[Path] = None
+
+            if isinstance(node, ast.Import):
+                # e.g., import utils, other.module
+                for alias in node.names:
+                    name_parts = alias.name.split(".")
+                    potential_path = base_dir / name_parts[0]
+                    if potential_path.is_dir():
+                        import_path = potential_path / "__init__.py"
+                    else:
+                        import_path = potential_path.with_suffix(".py")
+
+            elif isinstance(node, ast.ImportFrom):
+                # e.g., from . import utils
+                # e.g., from .utils import helper
+                if node.level > 0:  # Relative import
+                    level_dir = current_file.parent
+                    for _ in range(node.level - 1):
+                        level_dir = level_dir.parent
+
+                    if node.module:
+                        module_parts = node.module.split(".")
+                        potential_path = level_dir.joinpath(*module_parts)
+                    else:
+                        potential_path = level_dir
+
+                    if potential_path.is_dir():
+                        import_path = potential_path / "__init__.py"
+                    else:
+                        import_path = potential_path.with_suffix(".py")
+
+                elif node.module:  # Absolute import (check if it's local)
+                    name_parts = node.module.split(".")
+                    potential_path = base_dir / name_parts[0]
+                    if potential_path.is_dir():
+                        import_path = potential_path / "__init__.py"
+                    else:
+                        import_path = potential_path.with_suffix(".py")
+
+            if (
+                import_path
+                and import_path.exists()
+                and import_path.is_file()
+                and import_path.parent.is_relative_to(base_dir)
+            ):
+                if (
+                    import_path not in processed_files
+                    and import_path not in files_to_process
+                ):
+                    print(f"  Found local import: {import_path.relative_to(base_dir)}")
+                    files_to_process.append(import_path)
+
+    # --- 3. Write all collected markdown to the output file ---
+    try:
+        with open(output_file, "w", encoding="utf-8") as f:
+            f.writelines(all_markdown)
+        print(f"\nSuccessfully generated documentation at: {output_file}")
+    except Exception as e:
+        print(f"\n[Error] Could not write to output file {output_file}: {e}")
+
+
+def main():
+    """
+    Parses command-line arguments and runs the extractor.
+    """
+    parser = argparse.ArgumentParser(
+        description="Extracts docstrings from a Python file and its local imports into a Markdown file.",
+        formatter_class=argparse.RawTextHelpFormatter,
+        epilog="""
+Example Usage:
+  python doc_extractor.py --input-file my_app/main.py --output-file DOCS.md
+""",
+    )
+
+    parser.add_argument(
+        "--input-file",
+        required=True,
+        help="The main Python file to start processing from.",
+    )
+    parser.add_argument(
+        "--output-file",
+        required=True,
+        help="The path to the final output Markdown file.",
+    )
+
+    parser.add_argument(
+        "--include-empty-functions",
+        action="store_true",
+        help="Include placeholders for functions without docstrings.",
+    )
+
+    args = parser.parse_args()
+
+    if not Path(args.input_file).exists():
+        print(f"[Error] Input file not found: {args.input_file}")
+        return
+
+    extract_documentation(
+        args.input_file, args.output_file, args.include_empty_functions
+    )
+
+
+if __name__ == "__main__":
+    main()

package/scripts/build_validation_list.py

@@ -0,0 +1,160 @@
+import ast
+import json
+import sys
+import os
+import argparse
+
+
+def get_str_value(node):
+    """
+    Safely extract a simple string value from an AST node.
+    Handles ast.Constant (Python 3.8+) and ast.Str (Python < 3.8).
+    Returns None if the node is not a simple string.
+    """
+    if isinstance(node, ast.Constant) and isinstance(node.value, str):
+        return node.value
+    # For compatibility with Python < 3.8
+    if isinstance(node, ast.Str):
+        return node.s
+    return None
+
+
+def extract_validations_from_code(source_code: str) -> list:
+    """
+    Parses Python source code and extracts validation dicts
+    containing 'level' and 'message' keys.
+
+    Attributes all found dicts to the top-level function they are
+    defined within.
+    """
+    validations = []
+
+    try:
+        # Parse the source code into an Abstract Syntax Tree
+        tree = ast.parse(source_code)
+    except SyntaxError as e:
+        # We'll print the syntax error but continue processing other files
+        print(f"Warning: Skipping file due to SyntaxError: {e}", file=sys.stderr)
+        return []
+
+    # --- MODIFIED LOGIC ---
+
+    # Iterate through only the top-level nodes in the file's body
+    # This will find top-level functions and classes.
+    for top_node in tree.body:
+        top_level_name = None
+
+        # Check if the node is a top-level function
+        if isinstance(top_node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            top_level_name = top_node.name
+
+        # You could add logic for classes here if needed, e.g.:
+        # elif isinstance(top_node, ast.ClassDef):
+        #     # Logic to handle methods within classes
+        #     pass
+
+        # If we didn't find a top-level function, skip this node
+        if not top_level_name:
+            continue
+
+        # Now, walk through ALL nodes *within* this top-level function
+        for child_node in ast.walk(top_node):
+
+            # We are only looking for dictionaries. We don't care
+            # if we are inside a nested function or not.
+            if isinstance(child_node, ast.Dict):
+                details = {}
+
+                # Iterate over the (key, value) pairs in the dict
+                for key_node, value_node in zip(child_node.keys, child_node.values):
+                    key_str = get_str_value(key_node)
+                    value_str = get_str_value(value_node)
+
+                    # We only care about 'level' and 'message' keys
+                    # with simple string values
+                    if key_str in ["level", "message"] and value_str is not None:
+                        details[key_str] = value_str
+
+                # If we found both keys, add it to our list
+                # using the TOP-LEVEL function name.
+                if "level" in details and "message" in details:
+                    validations.append(
+                        {
+                            "function": top_level_name,  # <-- Uses the top-level name
+                            "level": details["level"],
+                            "message": details["message"],
+                        }
+                    )
+
+    # --- END MODIFIED LOGIC ---
+
+    return validations
+
+
+def main():
+    """
+    Main function to parse arguments, scan files, extract validations,
+    and write the result to a JSON file.
+    """
+    # --- 1. Set up Argument Parser ---
+    parser = argparse.ArgumentParser(
+        description="Extracts validation 'level' and 'message' dicts from Python functions."
+    )
+    parser.add_argument(
+        "-i",
+        "--input-folder",
+        required=True,
+        help="The directory to scan for .py files (scans recursively).",
+    )
+    parser.add_argument(
+        "-o", "--output-file", required=True, help="The path to the output JSON file."
+    )
+    args = parser.parse_args()
+
+    # --- 2. Scan Directory and Process Files ---
+    all_validations = []
+    print(f"Scanning for .py files in: {args.input_folder}")
+
+    for root, dirs, files in os.walk(args.input_folder):
+        for filename in files:
+            if filename.endswith(".py"):
+                filepath = os.path.join(root, filename)
+                print(f"Processing: {filepath}")
+
+                try:
+                    with open(filepath, "r", encoding="utf-8") as f:
+                        source_code = f.read()
+
+                    # Extract validations from this file's code
+                    validations_from_file = extract_validations_from_code(source_code)
+                    all_validations.extend(validations_from_file)
+
+                except FileNotFoundError:
+                    print(f"Error: File not found at '{filepath}'", file=sys.stderr)
+                except Exception as e:
+                    print(f"Error reading file {filepath}: {e}", file=sys.stderr)
+
+    # --- 3. Format and Write Output JSON ---
+    output_data = {"validations": all_validations}
+
+    try:
+        with open(args.output_file, "w", encoding="utf-8") as out_f:
+            json.dump(output_data, out_f, indent=2)
+
+        print(f"\n✅ Success! Extracted {len(all_validations)} validation(s).")
+        print(f"Results saved to: {args.output_file}")
+
+    except IOError as e:
+        print(
+            f"\n❌ Error: Could not write to output file {args.output_file}",
+            file=sys.stderr,
+        )
+        print(f"{e}", file=sys.stderr)
+        sys.exit(1)
+    except Exception as e:
+        print(f"\n❌ An unexpected error occurred during writing: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

package/scripts/guide-create-branch.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+BRANCH="$1"
+PRIVATE_TOKEN="$2"
+
+GITLAB_URL="https://gitlab.com/api/v4"
+SOURCE_BRANCH="develop"
+
+PAYLOAD=$(jq -n --arg branch "$BRANCH" --arg ref "$SOURCE_BRANCH" '{branch: $branch, ref: $ref}')
+
+curl --request POST \
+     --header "PRIVATE-TOKEN: $PRIVATE_TOKEN" \
+     --header "Content-Type: application/json" \
+     --data "$PAYLOAD" \
+     "$GITLAB_URL/projects/${GITLAB_GUIDE_PROJECT_ID}/repository/branches"

package/scripts/update-package-version.js

@@ -0,0 +1,28 @@
+const { readFileSync, writeFileSync } = require("fs");
+const { resolve, join } = require("path");
+
+const ROOT = resolve(join(__dirname, "../"));
+const encoding = "UTF-8";
+const version = require(join(ROOT, "package.json")).version;
+
+const PYTHON_VERSION_PATH = resolve(
+  join(ROOT, "hestia_earth", "validation", "version.py")
+);
+writeFileSync(
+  PYTHON_VERSION_PATH,
+  readFileSync(PYTHON_VERSION_PATH, encoding).replace(
+    /VERSION\s=\s\"[\d\-a-z\.]+\"/,
+    `VERSION = "${version}"`
+  ),
+  encoding
+);
+
+const JS_VERSION_PATH = resolve(join(ROOT, "src", "version.ts"));
+writeFileSync(
+  JS_VERSION_PATH,
+  readFileSync(JS_VERSION_PATH, encoding).replace(
+    /\'[\d\-a-z\.]+\'/,
+    `'${version}'`
+  ),
+  encoding
+);