ecological-agent-skills 3.1.0

package/skills/ecological-data-foundation/scripts/clean_occurrences.py

@@ -0,0 +1,268 @@
+#!/usr/bin/env python3
+# ecological-agent-skills / Copyright (C) 2026 Francisco Diego Barros Barata
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+"""
+clean_occurrences.py
+Standard occurrence cleaning pipeline using Python.
+Usage: python clean_occurrences.py <input_csv> <output_dir>
+Requires: pandas, geopandas, shapely
+"""
+
+import logging
+import sys
+from datetime import datetime
+from pathlib import Path
+
+SKILL_NAME = "ecological-data-foundation"
+_LOG_DIR   = Path("logs")
+_LOG_DIR.mkdir(parents=True, exist_ok=True)
+_log_file  = _LOG_DIR / f"skill_{SKILL_NAME}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log"
+logging.basicConfig(
+    level=logging.INFO,
+    format="[%(asctime)s] [%(levelname)s] [" + SKILL_NAME + "] %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+    handlers=[
+        logging.StreamHandler(sys.stdout),
+        logging.FileHandler(_log_file, encoding="utf-8"),
+    ],
+)
+logger = logging.getLogger(SKILL_NAME)
+
+def log_step(n: int, desc: str) -> None:
+    logger.info("-- STEP %d: %s", n, desc)
+
+def log_decision(var: str, val, why: str) -> None:
+    logger.info("DECISION | %s = %s | %s", var, val, why)
+
+
+import os
+import hashlib
+
+import pandas as pd
+import numpy as np
+
+
+def load(path: str) -> pd.DataFrame:
+    try:
+        df = pd.read_csv(path, low_memory=False)
+        logger.info("Carregadas %d linhas, %d colunas de %s", len(df), len(df.columns), path)
+        return df
+    except FileNotFoundError:
+        logger.error(
+            "Input nao encontrado: %s\n  Causa provavel: arquivo nao gerado pelo passo anterior.\n  Verifique a saida de: ecological-data-foundation (download_from_gbif)\n  Skill anterior: ecological-data-foundation",
+            path,
+        )
+        sys.exit(1)
+    except Exception as e:
+        logger.error(
+            "Falha ao ler CSV de entrada '%s': %s\n  Causa provavel: arquivo corrompido ou formato invalido.\n  Skill anterior: ecological-data-foundation",
+            path, e,
+        )
+        sys.exit(1)
+
+
+def check_required_cols(df: pd.DataFrame, required: list) -> None:
+    missing = [c for c in required if c not in df.columns]
+    if missing:
+        logger.error(
+            "Colunas obrigatorias ausentes: %s\n  Causa provavel: CSV gerado por fonte diferente ou schema alterado.\n  Skill anterior: ecological-data-foundation",
+            missing,
+        )
+        raise ValueError(f"Missing required columns: {missing}")
+    logger.info("Todas as colunas obrigatorias presentes: %s", required)
+
+
+def flag_coordinate_issues(df: pd.DataFrame,
+                           lat_col="decimalLatitude",
+                           lon_col="decimalLongitude") -> pd.DataFrame:
+    df = df.copy()
+    df["QA_status"] = "OK"
+    lat = pd.to_numeric(df[lat_col], errors="coerce")
+    lon = pd.to_numeric(df[lon_col], errors="coerce")
+
+    # Invalid range
+    mask_range = (lat.abs() > 90) | (lon.abs() > 180)
+    df.loc[mask_range, "QA_status"] = "COORD_OUT_OF_RANGE"
+
+    # Zero coordinates
+    mask_zero = (lat == 0) & (lon == 0)
+    df.loc[mask_zero, "QA_status"] = "COORD_ZERO"
+
+    # Missing coords
+    mask_na = lat.isna() | lon.isna()
+    df.loc[mask_na, "QA_status"] = "MISSING_COORDS"
+
+    n_range = int(mask_range.sum())
+    n_zero  = int(mask_zero.sum())
+    n_na    = int(mask_na.sum())
+
+    logger.info(
+        "Problemas de coordenadas — Fora do intervalo: %d | Zero: %d | Ausentes: %d",
+        n_range, n_zero, n_na,
+    )
+    if n_range > 0:
+        logger.warning("Registros com coordenadas fora do intervalo valido: %d", n_range)
+    if n_zero > 0:
+        logger.warning("Registros com coordenadas zero (0,0): %d — possivelmente erros de digitacao", n_zero)
+    if n_na > 0:
+        logger.warning("Registros sem coordenadas (NA): %d", n_na)
+    return df
+
+
+def remove_exact_duplicates(df: pd.DataFrame,
+                             cols=("scientificName","decimalLatitude","decimalLongitude","eventDate")
+                             ) -> pd.DataFrame:
+    cols_present = [c for c in cols if c in df.columns]
+    log_decision(
+        "dedup_cols",
+        cols_present,
+        "combinacao padrao para identificar duplicatas espaciotemporais",
+    )
+    n_before = len(df)
+    df = df.drop_duplicates(subset=cols_present, keep="first")
+    n_removed = n_before - len(df)
+    if n_removed > 0:
+        logger.warning("Duplicatas exatas removidas: %d", n_removed)
+    else:
+        logger.info("Nenhuma duplicata exata encontrada.")
+    return df
+
+
+def check_temporal(df: pd.DataFrame, date_col="eventDate") -> pd.DataFrame:
+    if date_col not in df.columns:
+        logger.warning("Coluna '%s' ausente — verificacao temporal pulada.", date_col)
+        return df
+    dates = pd.to_datetime(df[date_col], errors="coerce")
+    future = dates > pd.Timestamp.now()
+    n_future = int(future.sum())
+    df.loc[future & (df["QA_status"] == "OK"), "QA_status"] = "DATE_FUTURE"
+    if n_future > 0:
+        logger.warning("Registros com datas futuras sinalizados: %d", n_future)
+    else:
+        logger.info("Nenhum registro com data futura encontrado.")
+    return df
+
+
+def write_outputs(df: pd.DataFrame, output_dir: Path) -> None:
+    output_dir.mkdir(parents=True, exist_ok=True)
+    clean   = df[df["QA_status"] == "OK"]
+    flagged = df[df["QA_status"] != "OK"]
+
+    try:
+        clean.to_csv(output_dir / "data_clean.csv", index=False)
+        logger.info("Gravado: %s", output_dir / "data_clean.csv")
+        flagged.to_csv(output_dir / "flagged_records.csv", index=False)
+        logger.info("Gravado: %s", output_dir / "flagged_records.csv")
+    except OSError as e:
+        logger.error(
+            "Falha ao gravar arquivos de saida em '%s': %s\n  Causa provavel: sem permissao de escrita no diretorio.\n  Skill anterior: ecological-data-foundation",
+            output_dir, e,
+        )
+        raise
+
+    # QA report
+    flag_counts = flagged["QA_status"].value_counts().to_dict()
+    lines = [
+        "# QA Report — Occurrence Cleaning",
+        f"- Raw records: {len(df):,}",
+        f"- Clean records: {len(clean):,}",
+        f"- Flagged records: {len(flagged):,}",
+        "",
+        "## Flag Counts",
+    ] + [f"- `{k}`: {v}" for k, v in flag_counts.items()]
+
+    try:
+        (output_dir / "qa_report.md").write_text("\n".join(lines))
+        logger.info("Gravado: %s", output_dir / "qa_report.md")
+    except OSError as e:
+        logger.error(
+            "Falha ao gravar relatorio QA: %s\n  Causa provavel: sem permissao de escrita.\n  Skill anterior: ecological-data-foundation",
+            e,
+        )
+        raise
+
+    if len(clean) < 30:
+        logger.warning(
+            "Apenas %d registros limpos apos todas as filtragens. SDMs requerem >= 30 registros confiaveis.",
+            len(clean),
+        )
+
+    logger.info("Concluido. Limpos: %d | Sinalizados: %d", len(clean), len(flagged))
+    logger.info("Saidas gravadas em: %s", output_dir)
+
+
+def main():
+    logger.info("Script: clean_occurrences.py | Skill: %s", SKILL_NAME)
+
+    input_file = sys.argv[1] if len(sys.argv) > 1 else "data/raw/occurrences.csv"
+    output_dir = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("data/processed")
+
+    logger.info("Input file : %s", input_file)
+    logger.info("Output dir : %s", output_dir)
+
+    # Input precondition check
+    if not Path(input_file).exists():
+        logger.error(
+            "Input nao encontrado: %s\n  Causa provavel: arquivo nao gerado pelo passo anterior.\n  Verifique a saida de: ecological-data-foundation (download_from_gbif)\n  Skill anterior: ecological-data-foundation",
+            input_file,
+        )
+        sys.exit(1)
+
+    log_decision("input_file", input_file, "caminho passado como argv[1] ou padrao")
+    log_decision("output_dir", str(output_dir), "caminho passado como argv[2] ou padrao")
+
+    log_step(1, "Carregar dados brutos de ocorrencias")
+    df = load(input_file)
+
+    if len(df) == 0:
+        logger.warning("Arquivo de entrada nao contem registros: %s", input_file)
+
+    log_step(2, "Verificar colunas obrigatorias")
+    try:
+        check_required_cols(df, ["decimalLatitude", "decimalLongitude"])
+    except ValueError as e:
+        logger.error("Verificacao de colunas falhou: %s", e)
+        sys.exit(1)
+
+    log_step(3, "Sinalizar problemas de coordenadas")
+    log_decision(
+        "cc_tests",
+        "out_of_range,zero,missing",
+        "verificacoes basicas de qualidade de coordenadas sem dependencias externas",
+    )
+    try:
+        df = flag_coordinate_issues(df)
+    except Exception as e:
+        logger.error(
+            "Falha na sinalizacao de coordenadas: %s\n  Causa provavel: colunas de coordenadas com tipos inesperados.\n  Skill anterior: ecological-data-foundation",
+            e,
+        )
+        raise
+
+    log_step(4, "Remover duplicatas exatas")
+    try:
+        df = remove_exact_duplicates(df)
+    except Exception as e:
+        logger.error(
+            "Falha ao remover duplicatas: %s\n  Causa provavel: colunas de deduplicacao ausentes ou mal tipadas.\n  Skill anterior: ecological-data-foundation",
+            e,
+        )
+        raise
+
+    log_step(5, "Verificar datas futuras")
+    try:
+        df = check_temporal(df)
+    except Exception as e:
+        logger.error(
+            "Falha na verificacao temporal: %s\n  Causa provavel: coluna eventDate com formato inesperado.\n  Skill anterior: ecological-data-foundation",
+            e,
+        )
+        raise
+
+    log_step(6, "Escrever arquivos de saida e relatorio QA")
+    write_outputs(df, output_dir)
+
+
+if __name__ == "__main__":
+    main()

package/skills/ecological-data-foundation/scripts/download_from_ebird.R

@@ -0,0 +1,251 @@
+# ecological-agent-skills / Copyright (C) 2026 Francisco Diego Barros Barata
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# Usage: Rscript download_from_ebird.R <ebd_file> <species_name_or_list_csv> <output_dir> [year_from] [year_to] [country_code]
+
+# ── Inline logger ─────────────────────────────────────────────────────────────
+SKILL_NAME <- "ecological-data-foundation"
+.log_ts  <- function() format(Sys.time(), "[%Y-%m-%d %H:%M:%S]")
+log_info <- function(...) message(.log_ts(), " [INFO]  ", sprintf(...))
+log_warn <- function(...) message(.log_ts(), " [WARN]  ", sprintf(...))
+log_error<- function(...) message(.log_ts(), " [ERROR] ", sprintf(...))
+log_step <- function(n, d) log_info("-- STEP %d: %s", n, d)
+log_decision <- function(v, val, why) log_info("DECISION | %s = %s | %s", v, val, why)
+dir.create("logs", recursive=TRUE, showWarnings=FALSE)
+
+#
+# Arguments:
+#   ebd_file                 : Path to the eBird Basic Dataset (EBD) text file
+#                              (pre-downloaded from https://ebird.org/data/download)
+#   species_name_or_list_csv : Common or scientific name, or CSV with column "scientificName"
+#   output_dir               : Directory to write outputs (created if absent)
+#   year_from                : Minimum year of observation (optional, default: 2000)
+#   year_to                  : Maximum year of observation (optional, default: current year)
+#   country_code             : ISO 3166-1 alpha-2 country code to filter (optional)
+#
+# Note: eBird data requires a pre-downloaded EBD file; this script parses it locally.
+#       Apply for access at: https://ebird.org/data/download
+#
+# Outputs (per species):
+#   occurrences_raw_eBird_{species}_{date}.csv  — standardised occurrence records
+#   download_metadata_eBird_{species}.txt        — download provenance and citation
+#
+# Standard output schema:
+#   species, decimalLatitude, decimalLongitude, eventDate, countryCode,
+#   basisOfRecord, coordinateUncertaintyInMeters, datasetName, occurrenceID,
+#   source, download_doi
+# Extra eBird columns:
+#   effort_distance_km, duration_minutes, observer_id
+
+suppressPackageStartupMessages(library(auk))
+suppressPackageStartupMessages(library(dplyr))
+suppressPackageStartupMessages(library(readr))
+
+# ── 1. Parse arguments ───────────────────────────────────────────────────────
+log_step(1, "Analisar argumentos da linha de comando")
+args <- commandArgs(trailingOnly = TRUE)
+
+if (length(args) < 3) {
+  ebd_file      <- "data/ebird/ebd_sample.txt"
+  species_input <- "Jabiru mycteria"
+  output_dir    <- "output/ebird"
+  year_from     <- 2000
+  year_to       <- as.integer(format(Sys.Date(), "%Y"))
+  country_code  <- NULL
+  log_warn("Menos de 3 argumentos fornecidos. Usando valores padrao para teste.")
+} else {
+  ebd_file      <- args[1]
+  species_input <- args[2]
+  output_dir    <- args[3]
+  year_from     <- if (length(args) >= 4) as.integer(args[4]) else 2000
+  year_to       <- if (length(args) >= 5) as.integer(args[5]) else as.integer(format(Sys.Date(), "%Y"))
+  country_code  <- if (length(args) >= 6 && args[6] != "") args[6] else NULL
+}
+
+log_info("Script: download_from_ebird.R | Skill: %s", SKILL_NAME)
+log_info("EBD file       : %s", ebd_file)
+log_info("Species input  : %s", species_input)
+log_info("Output dir     : %s", output_dir)
+log_info("Year range     : %d - %d", year_from, year_to)
+log_info("Country code   : %s", ifelse(is.null(country_code), "nenhum", country_code))
+
+log_decision("protocol", "STATIONARY,TRAVELING",
+             "apenas protocolos quantificaveis para modelagem de avistamentos")
+log_decision("approved", "TRUE",
+             "apenas listas aprovadas pelo eBird (revisao de qualidade aplicada)")
+log_decision("year_from", year_from, "filtro temporal; 2000 equilibra tamanho e qualidade")
+
+# ── 2. Check EBD file exists ─────────────────────────────────────────────────
+log_step(2, "Verificar existencia do arquivo EBD")
+if (!file.exists(ebd_file)) {
+  log_error(
+    "Input nao encontrado: %s\nCausa provavel: arquivo EBD nao baixado do eBird.\nVerifique: https://ebird.org/data/download — solicite acesso e baixe o EBD.\nSkill anterior: ecological-data-foundation",
+    ebd_file
+  )
+  stop("EBD file not found: ", ebd_file)
+}
+log_info("Arquivo EBD encontrado: %s", ebd_file)
+
+# ── 3. Create output directory ───────────────────────────────────────────────
+log_step(3, "Criar diretorio de saida")
+dir.create(output_dir, recursive = TRUE, showWarnings = FALSE)
+
+# ── 4. Build species list ────────────────────────────────────────────────────
+log_step(4, "Construir lista de especies")
+if (grepl("\\.csv$", species_input, ignore.case = TRUE) && file.exists(species_input)) {
+  tryCatch({
+    species_df <- read_csv(species_input, show_col_types = FALSE)
+    if (!"scientificName" %in% names(species_df)) {
+      log_error(
+        "Coluna 'scientificName' nao encontrada em: %s\nCausa provavel: CSV mal formatado.\nSkill anterior: ecological-data-foundation",
+        species_input
+      )
+      stop("Missing column 'scientificName'")
+    }
+    species_list <- unique(trimws(species_df$scientificName))
+    log_info("Modo batch: %d especies carregadas", length(species_list))
+    log_decision("mode", "batch", "CSV valido com coluna scientificName")
+  }, error = function(e) {
+    log_error(
+      "Falha ao ler lista de especies: %s\nCausa provavel: CSV invalido.\nSkill anterior: ecological-data-foundation",
+      conditionMessage(e)
+    )
+    stop(e)
+  })
+} else {
+  species_list <- trimws(species_input)
+  log_info("Modo especie unica: %s", species_list)
+  log_decision("mode", "single_species", "argumento nao e arquivo CSV")
+}
+
+# ── 5. Filter and parse EBD ──────────────────────────────────────────────────
+log_step(5, "Filtrar e carregar EBD com auk")
+
+# Create a temporary filtered file for all species at once
+tmp_filtered <- tempfile(fileext = ".txt")
+
+auk_filter_obj <- tryCatch({
+  flt <- auk_ebd(ebd_file) |>
+    auk_species(species_list) |>
+    auk_date(date = c(paste0(year_from, "-01-01"), paste0(year_to, "-12-31"))) |>
+    auk_protocol(c("Stationary", "Traveling")) |>
+    auk_complete()
+
+  if (!is.null(country_code)) {
+    flt <- auk_country(flt, country_code)
+  }
+  flt
+}, error = function(e) {
+  log_error(
+    "Falha ao configurar filtros auk: %s\nCausa provavel: nome de especie nao encontrado no EBD ou parametros invalidos.\nVerifique nomes usando auk_species_codes().\nSkill anterior: ecological-data-foundation",
+    conditionMessage(e)
+  )
+  stop(e)
+})
+
+ebd_filtered <- tryCatch({
+  auk_filter(auk_filter_obj, file = tmp_filtered, overwrite = TRUE)
+  read_ebd(tmp_filtered)
+}, error = function(e) {
+  log_error(
+    "Falha ao filtrar ou ler EBD: %s\nCausa provavel: arquivo EBD corrompido ou formato incompativel com versao do auk.\nVerifique: auk::auk_version_requirements().\nSkill anterior: ecological-data-foundation",
+    conditionMessage(e)
+  )
+  stop(e)
+})
+
+n_filtered <- nrow(ebd_filtered)
+log_info("Registros apos filtragem EBD: %d", n_filtered)
+
+if (n_filtered == 0) {
+  log_warn("Nenhum registro encontrado apos filtragem. Verifique nomes das especies e periodo.")
+}
+
+# ── 6. Standardise and save per-species ─────────────────────────────────────
+log_step(6, "Padronizar e gravar CSVs por especie")
+today_str <- format(Sys.Date(), "%Y%m%d")
+
+for (sp in species_list) {
+  sp_data <- ebd_filtered[ebd_filtered$scientific_name == sp |
+                             ebd_filtered$common_name == sp, ]
+  n_sp <- nrow(sp_data)
+  log_info("Especie '%s': %d registros", sp, n_sp)
+
+  if (n_sp == 0) {
+    log_warn("Nenhum registro para '%s'. Pulando.", sp)
+    next
+  }
+
+  if (n_sp < 30) {
+    log_warn(
+      "Registros insuficientes para SDM confiavel para '%s' (n = %d). Considere ampliar periodo ou area geografica.",
+      sp, n_sp
+    )
+  }
+
+  safe_name <- gsub(" ", "_", sp)
+
+  # Standardised schema + extra eBird columns
+  std <- data.frame(
+    species                       = sp,
+    decimalLatitude               = as.numeric(sp_data$latitude),
+    decimalLongitude              = as.numeric(sp_data$longitude),
+    eventDate                     = as.character(sp_data$observation_date),
+    countryCode                   = as.character(sp_data$country_code),
+    basisOfRecord                 = "HUMAN_OBSERVATION",
+    coordinateUncertaintyInMeters = NA_real_,
+    datasetName                   = "eBird Basic Dataset",
+    occurrenceID                  = as.character(sp_data$sampling_event_identifier),
+    source                        = "eBird",
+    download_doi                  = NA_character_,
+    effort_distance_km            = as.numeric(sp_data$effort_distance_km),
+    duration_minutes              = as.numeric(sp_data$duration_minutes),
+    observer_id                   = as.character(sp_data$observer_id),
+    stringsAsFactors              = FALSE
+  )
+
+  std <- std[!is.na(std$decimalLatitude) & !is.na(std$decimalLongitude), ]
+
+  csv_path <- file.path(output_dir,
+                         paste0("occurrences_raw_eBird_", safe_name, "_", today_str, ".csv"))
+  tryCatch({
+    write_csv(std, csv_path)
+    log_info("Gravado: %s (%d registros)", csv_path, nrow(std))
+  }, error = function(e) {
+    log_error(
+      "Falha ao gravar CSV para '%s': %s\nCausa provavel: sem permissao de escrita.\nSkill anterior: ecological-data-foundation",
+      sp, conditionMessage(e)
+    )
+    stop(e)
+  })
+
+  # Metadata
+  meta_lines <- c(
+    paste("Species:", sp),
+    paste("Source: eBird Basic Dataset (https://ebird.org/data/download)"),
+    paste("Protocols: Stationary, Traveling"),
+    paste("Approved only: TRUE"),
+    paste("Year range:", year_from, "-", year_to),
+    paste("Country filter:", ifelse(is.null(country_code), "none", country_code)),
+    paste("n_records:", nrow(std)),
+    paste("Download date:", Sys.Date()),
+    paste("Citation: eBird Basic Dataset. Version:", format(Sys.Date(), "%Y-%m"),
+          ". Cornell Lab of Ornithology, Ithaca, New York.", Sys.Date()),
+    paste("Note: eBird data requires a signed Data Use Agreement. Cite the dataset version used.")
+  )
+  meta_path <- file.path(output_dir, paste0("download_metadata_eBird_", safe_name, ".txt"))
+  tryCatch({
+    writeLines(meta_lines, meta_path)
+    log_info("Metadados gravados: %s", meta_path)
+  }, error = function(e) {
+    log_error(
+      "Falha ao gravar metadados para '%s': %s\nSkill anterior: ecological-data-foundation",
+      sp, conditionMessage(e)
+    )
+  })
+}
+
+# Clean up temp file
+if (file.exists(tmp_filtered)) file.remove(tmp_filtered)
+
+log_info("Todos os downloads eBird concluidos. Verifique: %s", output_dir)