@dataif/cli 0.1.0

package/templates/dataif/scripts/sync_metabase_dashboard_field_filters.py

@@ -0,0 +1,241 @@
+from __future__ import annotations
+
+import json
+import os
+import re
+import urllib.error
+import urllib.request
+
+
+API_BASE = os.getenv("METABASE_API_URL", "http://localhost:3001/api").rstrip("/")
+API_KEY = os.getenv("METABASE_API_KEY")
+DATABASE_ID = int(os.getenv("METABASE_DATABASE_ID", "2"))
+DASHBOARD_ID = int(os.getenv("METABASE_DASHBOARD_ID", "3"))
+
+if not API_KEY:
+    raise SystemExit("METABASE_API_KEY is required")
+
+
+def api(method: str, path: str, payload: dict | list | None = None) -> dict | list:
+    data = None
+    headers = {"x-api-key": API_KEY, "Accept": "application/json"}
+    if payload is not None:
+        data = json.dumps(payload).encode("utf-8")
+        headers["Content-Type"] = "application/json"
+
+    request = urllib.request.Request(f"{API_BASE}{path}", data=data, headers=headers, method=method)
+    try:
+        with urllib.request.urlopen(request) as response:
+            body = response.read().decode("utf-8")
+            return json.loads(body) if body else {}
+    except urllib.error.HTTPError as exc:
+        body = exc.read().decode("utf-8", errors="replace")
+        raise RuntimeError(f"{method} {path} failed: {exc.code} {body}") from exc
+
+
+def find_source_name(query: str) -> str | None:
+    match = re.search(r"\bFROM\s+([a-zA-Z0-9_.]+)", query, flags=re.IGNORECASE)
+    return match.group(1).split(".")[-1] if match else None
+
+
+def normalize_filter_clause(query: str, slugs: list[str]) -> str:
+    normalized = []
+    for line in query.splitlines():
+        stripped = line.strip()
+        replaced = False
+        if stripped.startswith("[[ AND"):
+            for slug in slugs:
+                if f"{{{{{slug}}}}}" in line:
+                    normalized.append("[[ AND {{" + slug + "}} ]]")
+                    replaced = True
+                    break
+        if not replaced:
+            normalized.append(line)
+    return "\n".join(normalized)
+
+
+def widget_type(slug: str) -> str:
+    if slug == "ano":
+        return "number/="
+    if slug == "municipio":
+        return "string/contains"
+    return "string/="
+
+
+def fetch_field_map() -> dict[str, dict[str, int]]:
+    metadata = api("GET", f"/database/{DATABASE_ID}/metadata")
+    result: dict[str, dict[str, int]] = {}
+    for table in metadata.get("tables", []):
+        if table.get("schema") != "curated":
+            continue
+        result[table["name"]] = {field["name"]: int(field["id"]) for field in table.get("fields", [])}
+    return result
+
+
+def build_template_tag(slug: str, field_id: int, current_tag: dict) -> dict:
+    tag = {
+        "id": current_tag.get("id"),
+        "name": slug,
+        "display-name": current_tag.get("display-name") or slug.replace("_", " ").title(),
+        "type": "dimension",
+        "required": False,
+        "widget-type": widget_type(slug),
+        "dimension": ["field", field_id, None],
+    }
+    if tag["widget-type"] == "string/contains":
+        tag["options"] = {"case-sensitive": False}
+    return tag
+
+
+def build_parameter(slug: str, tag: dict, current_parameter: dict | None) -> dict:
+    parameter = {
+        "id": tag["id"],
+        "type": tag["widget-type"],
+        "target": ["dimension", ["template-tag", slug]],
+        "name": tag["display-name"],
+        "slug": slug,
+        "required": False,
+        "isMultiSelect": True,
+    }
+    if current_parameter:
+        for key in ("values_query_type", "default", "options"):
+            if key in current_parameter:
+                parameter[key] = current_parameter[key]
+    if "options" in tag:
+        parameter["options"] = tag["options"]
+    return parameter
+
+
+def minimal_dashcard_payload(dashcard: dict, parameter_mappings: list[dict]) -> dict:
+    payload = {
+        "id": dashcard["id"],
+        "card_id": dashcard.get("card_id"),
+        "row": dashcard["row"],
+        "col": dashcard["col"],
+        "size_x": dashcard["size_x"],
+        "size_y": dashcard["size_y"],
+        "dashboard_tab_id": dashcard.get("dashboard_tab_id"),
+        "parameter_mappings": parameter_mappings,
+        "visualization_settings": dashcard.get("visualization_settings") or {},
+    }
+    return payload
+
+
+def sync() -> dict[str, int]:
+    field_map = fetch_field_map()
+    dashboard = api("GET", f"/dashboard/{DASHBOARD_ID}")
+    dashcards = dashboard.get("dashcards", [])
+
+    card_cache: dict[int, dict] = {}
+    updated_cards = 0
+    for dashcard in dashcards:
+        card_id = dashcard.get("card_id")
+        if not card_id:
+            continue
+
+        card = api("GET", f"/card/{card_id}")
+        card_cache[int(card_id)] = card
+        dataset_query = card.get("dataset_query") or {}
+        native = dataset_query.get("native") or {}
+        template_tags = native.get("template-tags") or {}
+        if not template_tags:
+            continue
+
+        source_name = find_source_name(native.get("query") or "")
+        if not source_name or source_name not in field_map:
+            continue
+
+        current_parameters = {item.get("slug"): item for item in (card.get("parameters") or []) if item.get("slug")}
+        normalized_tags = {}
+        ordered_slugs = [slug for slug in template_tags if slug in field_map[source_name]]
+        for slug in ordered_slugs:
+            normalized_tags[slug] = build_template_tag(slug, field_map[source_name][slug], template_tags[slug])
+
+        normalized_query = normalize_filter_clause(native["query"], ordered_slugs)
+        parameters = [build_parameter(slug, normalized_tags[slug], current_parameters.get(slug)) for slug in ordered_slugs]
+
+        payload = {
+            "name": card["name"],
+            "description": card.get("description"),
+            "display": card["display"],
+            "dataset_query": {
+                **dataset_query,
+                "native": {
+                    **native,
+                    "query": normalized_query,
+                    "template-tags": normalized_tags,
+                },
+            },
+            "visualization_settings": card.get("visualization_settings") or {},
+            "parameters": parameters,
+        }
+        api("PUT", f"/card/{card_id}", payload)
+        updated_cards += 1
+        card_cache[int(card_id)] = api("GET", f"/card/{card_id}")
+
+    used_slugs = []
+    for card in card_cache.values():
+        tags = (card.get("dataset_query") or {}).get("native", {}).get("template-tags", {})
+        for slug in tags:
+            if slug not in used_slugs:
+                used_slugs.append(slug)
+
+    dashboard_params = []
+    seen_slugs = set()
+    for parameter in dashboard.get("parameters", []):
+        slug = parameter.get("slug")
+        if slug not in used_slugs or slug in seen_slugs:
+            continue
+        dashboard_params.append(parameter)
+        seen_slugs.add(slug)
+
+    dashcards_payload = []
+    mapped_dashcards = 0
+    for dashcard in dashcards:
+        card_id = dashcard.get("card_id")
+        if not card_id:
+            dashcards_payload.append(minimal_dashcard_payload(dashcard, []))
+            continue
+
+        card = card_cache[int(card_id)]
+        tags = (card.get("dataset_query") or {}).get("native", {}).get("template-tags", {})
+        parameter_mappings = []
+        for parameter in dashboard_params:
+            slug = parameter["slug"]
+            if slug not in tags:
+                continue
+            parameter_mappings.append(
+                {
+                    "parameter_id": parameter["id"],
+                    "card_id": card_id,
+                    "target": ["dimension", ["template-tag", slug], {"stage-number": 0}],
+                }
+            )
+        if parameter_mappings:
+            mapped_dashcards += 1
+        dashcards_payload.append(minimal_dashcard_payload(dashcard, parameter_mappings))
+
+    api(
+        "PUT",
+        f"/dashboard/{DASHBOARD_ID}",
+        {
+            "name": dashboard["name"],
+            "description": dashboard.get("description"),
+            "width": dashboard.get("width", "fixed"),
+            "auto_apply_filters": dashboard.get("auto_apply_filters", True),
+            "parameters": dashboard_params,
+            "tabs": dashboard.get("tabs", []),
+            "dashcards": dashcards_payload,
+        },
+    )
+
+    return {
+        "updated_cards": updated_cards,
+        "dashboard_params": len(dashboard_params),
+        "mapped_dashcards": mapped_dashcards,
+        "dashcards_total": len([dc for dc in dashcards if dc.get("card_id")]),
+    }
+
+
+if __name__ == "__main__":
+    print(json.dumps(sync(), ensure_ascii=False))

package/templates/dataif/scripts/use-vanna-ollama.sh

@@ -0,0 +1,139 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+INFRA_DIR="${ROOT_DIR}/infra"
+ENV_FILE="${INFRA_DIR}/.env"
+
+run_bootstrap=true
+
+usage() {
+  cat <<'EOF'
+Uso: ./scripts/use-vanna-ollama.sh [--no-bootstrap]
+
+Alterna o Vanna para Ollama, preservando a chave Maritaca no infra/.env.
+
+Opcoes:
+  --no-bootstrap  Nao executa ollama-model-bootstrap. Use quando o modelo ja existir.
+  -h, --help      Mostra esta ajuda.
+EOF
+}
+
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    --no-bootstrap)
+      run_bootstrap=false
+      ;;
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    *)
+      echo "Opcao desconhecida: $1" >&2
+      usage >&2
+      exit 2
+      ;;
+  esac
+  shift
+done
+
+if [ ! -f "${ENV_FILE}" ]; then
+  echo "Arquivo ${ENV_FILE} nao encontrado. Crie-o a partir de infra/.env.example antes de continuar." >&2
+  exit 1
+fi
+
+get_env_value() {
+  local key="$1"
+
+  awk -v key="${key}" '
+    $0 ~ "^[[:space:]]*(export[[:space:]]+)?" key "[[:space:]]*=" {
+      value = $0
+      sub("^[[:space:]]*(export[[:space:]]+)?" key "[[:space:]]*=[[:space:]]*", "", value)
+      sub(/[[:space:]]*$/, "", value)
+      if ((value ~ /^".*"$/) || (value ~ /^'\''.*'\''$/)) {
+        value = substr(value, 2, length(value) - 2)
+      }
+      found = 1
+    }
+    END {
+      if (found) {
+        print value
+      }
+    }
+  ' "${ENV_FILE}"
+}
+
+set_env_value() {
+  local key="$1"
+  local value="$2"
+  local tmp
+
+  tmp="$(mktemp "${ENV_FILE}.tmp.XXXXXX")"
+  awk -v key="${key}" -v value="${value}" '
+    $0 ~ "^[[:space:]]*(export[[:space:]]+)?" key "[[:space:]]*=" {
+      if (!written) {
+        print key "=" value
+        written = 1
+      }
+      next
+    }
+    { print }
+    END {
+      if (!written) {
+        print key "=" value
+      }
+    }
+  ' "${ENV_FILE}" > "${tmp}"
+
+  chmod --reference="${ENV_FILE}" "${tmp}" 2>/dev/null || true
+  mv "${tmp}" "${ENV_FILE}"
+}
+
+ollama_model_name="$(get_env_value OLLAMA_MODEL_NAME)"
+if [ -z "${ollama_model_name}" ]; then
+  ollama_model_name="sabia-7b"
+fi
+
+vanna_port="$(get_env_value VANNA_PORT)"
+if [ -z "${vanna_port}" ]; then
+  vanna_port="9000"
+fi
+
+echo "Atualizando ${ENV_FILE}: VANNA_LLM_PROVIDER=ollama"
+set_env_value "VANNA_LLM_PROVIDER" "ollama"
+set_env_value "VANNA_OLLAMA_BASE_URL" "http://ollama:11434"
+set_env_value "VANNA_OLLAMA_MODEL" "${ollama_model_name}"
+
+cd "${INFRA_DIR}"
+
+echo "Subindo Ollama..."
+docker compose --profile llm up -d ollama
+
+if [ "${run_bootstrap}" = true ]; then
+  echo "Carregando modelo Ollama (${ollama_model_name})..."
+  docker compose --profile llm run --rm ollama-model-bootstrap
+else
+  echo "Bootstrap do modelo pulado (--no-bootstrap)."
+fi
+
+echo "Reiniciando Vanna..."
+docker compose up -d --build vanna
+
+health_url="http://localhost:${vanna_port}/health"
+echo "Verificando healthcheck: curl ${health_url}"
+
+for attempt in $(seq 1 30); do
+  if curl -fsS "${health_url}"; then
+    echo
+    exit 0
+  fi
+
+  if [ "${attempt}" -eq 30 ]; then
+    echo
+    echo "Vanna nao respondeu em ${health_url}. Tente novamente com:" >&2
+    echo "curl ${health_url}" >&2
+    exit 1
+  fi
+
+  sleep 2
+done

package/templates/dataif/services/api/.dockerignore

@@ -0,0 +1,18 @@
+.env
+.venv
+venv/
+
+__pycache__/
+**/__pycache__/
+*.pyc
+*.pyo
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+
+*.db
+*.sqlite
+*.sqlite3
+*.log
+tmp/
+temp/

package/templates/dataif/services/api/Dockerfile

@@ -0,0 +1,12 @@
+FROM python:3.11-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+
+WORKDIR /app
+COPY requirements.txt /app/requirements.txt
+RUN pip install --no-cache-dir -r /app/requirements.txt
+
+COPY app /app/app
+
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]

package/templates/dataif/services/api/app/__init__.py

@@ -0,0 +1 @@
+

package/templates/dataif/services/api/app/auth.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+from functools import lru_cache
+from typing import Any
+
+import jwt
+from fastapi import HTTPException, Security, status
+from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
+from jwt import PyJWKClient
+
+from .config import settings
+
+security = HTTPBearer(auto_error=False)
+
+
+@lru_cache(maxsize=1)
+def _jwks_client() -> PyJWKClient:
+    jwks_url = f"{settings.keycloak_url}/realms/{settings.keycloak_realm}/protocol/openid-connect/certs"
+    return PyJWKClient(jwks_url)
+
+
+def verify_optional_bearer(credentials: HTTPAuthorizationCredentials | None = Security(security)) -> dict[str, Any] | None:
+    if credentials is None:
+        return None
+
+    token = credentials.credentials
+    try:
+        signing_key = _jwks_client().get_signing_key_from_jwt(token)
+        payload = jwt.decode(
+            token,
+            signing_key.key,
+            algorithms=["RS256"],
+            audience=settings.keycloak_audience,
+            options={"verify_exp": True},
+        )
+        return payload
+    except Exception as exc:
+        raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail=f"Invalid token: {exc}") from exc
+
+
+def require_admin(payload: dict[str, Any] | None) -> None:
+    if payload is None:
+        raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Authentication required")
+
+    realm_access = payload.get("realm_access") or {}
+    roles = realm_access.get("roles") or []
+    if "admin" not in roles:
+        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin role required")

package/templates/dataif/services/api/app/config.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class Settings(BaseSettings):
+    api_host: str = Field(default="0.0.0.0", alias="API_HOST")
+    api_port: int = Field(default=8000, alias="API_PORT")
+
+    metabase_site_url: str = Field(default="http://localhost:3000", alias="METABASE_SITE_URL")
+    metabase_api_url: str = Field(default="http://metabase:3000", alias="METABASE_API_URL")
+    metabase_embed_secret: str = Field(default="replace_with_secure_secret", alias="METABASE_EMBED_SECRET")
+    metabase_allowed_dashboard_ids: str = Field(default="1,2,3", alias="METABASE_ALLOWED_DASHBOARD_IDS")
+    metabase_default_dashboard_id: str = Field(default="", alias="METABASE_DEFAULT_DASHBOARD_ID")
+    metabase_admin_email: str = Field(default="admin@dataif.local", alias="METABASE_ADMIN_EMAIL")
+    metabase_admin_password: str = Field(default="admin", alias="METABASE_ADMIN_PASSWORD")
+    metabase_admin_first_name: str = Field(default="DataIF", alias="METABASE_ADMIN_FIRST_NAME")
+    metabase_admin_last_name: str = Field(default="Metabase", alias="METABASE_ADMIN_LAST_NAME")
+    metabase_site_name: str = Field(default="dataif", alias="METABASE_SITE_NAME")
+    metabase_allow_tracking: bool = Field(default=False, alias="METABASE_ALLOW_TRACKING")
+
+    vanna_service_url: str = Field(default="http://localhost:9000", alias="VANNA_SERVICE_URL")
+    vanna_llm_provider: str = Field(default="ollama", alias="VANNA_LLM_PROVIDER")
+    vanna_ollama_base_url: str = Field(default="http://ollama:11434", alias="VANNA_OLLAMA_BASE_URL")
+    vanna_ollama_model: str = Field(default="sabia-7b", alias="VANNA_OLLAMA_MODEL")
+    vanna_maritaca_api_url: str = Field(
+        default="https://chat.maritaca.ai/api/chat/completions",
+        alias="VANNA_MARITACA_API_URL",
+    )
+    vanna_maritaca_api_key: str = Field(default="", alias="VANNA_MARITACA_API_KEY")
+    vanna_maritaca_model: str = Field(default="sabia-4", alias="VANNA_MARITACA_MODEL")
+    vanna_maritaca_timeout_seconds: int = Field(default=60, alias="VANNA_MARITACA_TIMEOUT_SECONDS")
+
+    keycloak_url: str = Field(default="http://localhost:8081", alias="KEYCLOAK_URL")
+    keycloak_realm: str = Field(default="dataif", alias="KEYCLOAK_REALM")
+    keycloak_audience: str = Field(default="dataif-api", alias="KEYCLOAK_AUDIENCE")
+    keycloak_client_id: str = Field(default="dataif-web", alias="KEYCLOAK_CLIENT_ID")
+    keycloak_client_secret: str = Field(default="", alias="KEYCLOAK_CLIENT_SECRET")
+    keycloak_admin_realm: str = Field(default="master", alias="KEYCLOAK_ADMIN_REALM")
+    keycloak_admin_client_id: str = Field(default="admin-cli", alias="KEYCLOAK_ADMIN_CLIENT_ID")
+    keycloak_admin_username: str = Field(default="admin", alias="KEYCLOAK_ADMIN")
+    keycloak_admin_password: str = Field(default="admin", alias="KEYCLOAK_ADMIN_PASSWORD")
+
+    warehouse_dsn: str = Field(default="", alias="WAREHOUSE_DSN")
+    airflow_api_url: str = Field(default="http://airflow-webserver:8080/airflow", alias="AIRFLOW_API_URL")
+    airflow_admin_user: str = Field(default="admin", alias="AIRFLOW_ADMIN_USER")
+    airflow_admin_password: str = Field(default="admin", alias="AIRFLOW_ADMIN_PASSWORD")
+
+    cors_allow_origins: str = Field(default="http://localhost:5173", alias="CORS_ALLOW_ORIGINS")
+    nilo_timeout_seconds: int = Field(default=60, alias="NILO_TIMEOUT_SECONDS")
+    pnp_catalog_cache_ttl_seconds: int = Field(default=900, alias="PNP_CATALOG_CACHE_TTL_SECONDS")
+
+    class Config:
+        env_file = ".env"
+        case_sensitive = False
+
+
+settings = Settings()