@dataif/cli 0.1.0

package/templates/dataif/services/api/app/metabase_admin.py

@@ -0,0 +1,191 @@
+from __future__ import annotations
+
+from typing import Any
+
+import httpx
+from fastapi import HTTPException
+
+
+class MetabaseAdminClient:
+    def __init__(
+        self,
+        *,
+        base_url: str,
+        admin_email: str,
+        admin_password: str,
+        timeout_seconds: float,
+    ) -> None:
+        self.base_url = base_url.rstrip("/")
+        self.admin_email = admin_email
+        self.admin_password = admin_password
+        self.timeout_seconds = timeout_seconds
+
+    def session_properties(self) -> dict[str, Any]:
+        return self._raw_request("GET", "/api/session/properties", expected_status={200}, auth_mode="none")
+
+    def list_admin_users(self) -> list[dict[str, Any]]:
+        items = self._request("GET", "/api/user", expected_status={200})
+        if not isinstance(items, list):
+            return []
+        return [self._normalize_user(item) for item in items if isinstance(item, dict) and bool(item.get("is_superuser"))]
+
+    def find_user_by_email(self, email: str) -> dict[str, Any] | None:
+        target = email.strip().lower()
+        if not target:
+            return None
+        for item in self.list_admin_users():
+            if str(item.get("email") or "").strip().lower() == target:
+                return item
+        return None
+
+    def create_admin_user(
+        self,
+        *,
+        email: str,
+        password: str,
+        first_name: str,
+        last_name: str,
+    ) -> dict[str, Any]:
+        existing = self.find_user_by_email(email)
+        if existing:
+            raise HTTPException(status_code=409, detail=f"Metabase admin already exists for email: {email}")
+
+        payload = {
+            "email": email,
+            "first_name": first_name,
+            "last_name": last_name,
+            "password": password,
+            "is_superuser": True,
+        }
+        created = self._request("POST", "/api/user", json=payload, expected_status={200})
+        if not isinstance(created, dict):
+            raise HTTPException(status_code=502, detail="Metabase did not return the created user")
+        return self._normalize_user(created)
+
+    def delete_user(self, user_id: int | str) -> None:
+        self._request("DELETE", f"/api/user/{user_id}", expected_status={200, 204})
+
+    def ensure_initial_admin(
+        self,
+        *,
+        first_name: str,
+        last_name: str,
+        site_name: str,
+        allow_tracking: bool,
+    ) -> dict[str, Any]:
+        properties = self.session_properties()
+        if bool(properties.get("has-user-setup", True)):
+            existing = self.find_user_by_email(self.admin_email)
+            if existing:
+                return {"bootstrapped": False, "user": existing}
+            return {"bootstrapped": False, "user": None}
+
+        setup_token = str(properties.get("setup-token") or "").strip()
+        if not setup_token:
+            raise HTTPException(status_code=502, detail="Metabase setup token was not returned")
+
+        payload = {
+            "token": setup_token,
+            "user": {
+                "email": self.admin_email,
+                "first_name": first_name,
+                "last_name": last_name,
+                "password": self.admin_password,
+            },
+            "prefs": {
+                "site_name": site_name,
+                "allow_tracking": allow_tracking,
+            },
+        }
+        self._raw_request("POST", "/api/setup", json=payload, expected_status={200}, auth_mode="none")
+        user = self.find_user_by_email(self.admin_email)
+        return {"bootstrapped": True, "user": user}
+
+    def _normalize_user(self, payload: dict[str, Any]) -> dict[str, Any]:
+        return {
+            "id": payload.get("id"),
+            "email": payload.get("email") or "",
+            "first_name": payload.get("first_name") or "",
+            "last_name": payload.get("last_name") or "",
+            "is_superuser": bool(payload.get("is_superuser")),
+            "is_active": bool(payload.get("is_active", True)),
+            "common_name": payload.get("common_name") or "",
+        }
+
+    def _request(
+        self,
+        method: str,
+        path: str,
+        *,
+        json: object | None = None,
+        expected_status: set[int],
+    ) -> Any:
+        return self._raw_request(method, path, json=json, expected_status=expected_status, auth_mode="session")
+
+    def _raw_request(
+        self,
+        method: str,
+        path: str,
+        *,
+        json: object | None = None,
+        expected_status: set[int],
+        auth_mode: str,
+    ) -> Any:
+        url = f"{self.base_url}{path}"
+        headers: dict[str, str] = {}
+        if auth_mode == "session":
+            headers["X-Metabase-Session"] = self._session_id()
+
+        try:
+            with httpx.Client(timeout=self.timeout_seconds, follow_redirects=True) as client:
+                response = client.request(method, url, json=json, headers=headers)
+        except httpx.RequestError as exc:
+            raise HTTPException(status_code=502, detail=f"Metabase request failed: {exc}") from exc
+
+        if response.status_code not in expected_status:
+            raise HTTPException(status_code=response.status_code, detail=_metabase_error_detail(response, "Metabase request failed"))
+
+        if response.status_code == 204 or not response.text.strip():
+            return None
+        try:
+            return response.json()
+        except ValueError:
+            return response.text
+
+    def _session_id(self) -> str:
+        try:
+            with httpx.Client(timeout=self.timeout_seconds, follow_redirects=True) as client:
+                response = client.post(
+                    f"{self.base_url}/api/session",
+                    json={"username": self.admin_email, "password": self.admin_password},
+                )
+        except httpx.RequestError as exc:
+            raise HTTPException(status_code=502, detail=f"Metabase session request failed: {exc}") from exc
+
+        if response.status_code >= 400:
+            raise HTTPException(status_code=response.status_code, detail=_metabase_error_detail(response, "Metabase session request failed"))
+
+        try:
+            payload = response.json()
+        except ValueError as exc:
+            raise HTTPException(status_code=502, detail="Metabase session response was not valid JSON") from exc
+
+        session_id = payload.get("id")
+        if not isinstance(session_id, str) or not session_id.strip():
+            raise HTTPException(status_code=502, detail="Metabase session response did not include an id")
+        return session_id
+
+
+def _metabase_error_detail(response: httpx.Response, fallback: str) -> str:
+    try:
+        payload = response.json()
+    except ValueError:
+        payload = None
+    if isinstance(payload, dict):
+        detail = payload.get("message") or payload.get("errors") or payload.get("error")
+        if isinstance(detail, str) and detail.strip():
+            return detail
+        if isinstance(detail, dict):
+            return str(detail)
+    text = response.text.strip()
+    return text or fallback

package/templates/dataif/services/api/app/metabase_bootstrap.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+import sys
+import time
+
+from fastapi import HTTPException
+
+from .config import settings
+from .metabase_admin import MetabaseAdminClient
+
+
+def main() -> int:
+    client = MetabaseAdminClient(
+        base_url=settings.metabase_api_url,
+        admin_email=settings.metabase_admin_email,
+        admin_password=settings.metabase_admin_password,
+        timeout_seconds=max(settings.nilo_timeout_seconds, 30.0),
+    )
+
+    deadline = time.monotonic() + 180
+    last_error = "Metabase did not become ready"
+    while time.monotonic() < deadline:
+        try:
+            outcome = client.ensure_initial_admin(
+                first_name=settings.metabase_admin_first_name,
+                last_name=settings.metabase_admin_last_name,
+                site_name=settings.metabase_site_name,
+                allow_tracking=settings.metabase_allow_tracking,
+            )
+            state = "bootstrapped" if outcome.get("bootstrapped") else "already_configured"
+            print(f"metabase_bootstrap={state}")
+            return 0
+        except HTTPException as exc:
+            last_error = str(exc.detail)
+        except Exception as exc:  # pragma: no cover - defensive bootstrap path
+            last_error = str(exc)
+        time.sleep(3)
+
+    print(f"metabase_bootstrap_failed={last_error}", file=sys.stderr)
+    return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/templates/dataif/services/api/app/metabase_embed.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+import time
+
+import jwt
+
+
+def build_signed_dashboard_url(site_url: str, embed_secret: str, dashboard_id: int, params: dict[str, object] | None = None) -> str:
+    payload = {
+        "resource": {"dashboard": int(dashboard_id)},
+        "params": params or {},
+        "exp": int(time.time()) + (10 * 60),
+    }
+    token = jwt.encode(payload, embed_secret, algorithm="HS256")
+    return f"{site_url}/embed/dashboard/{token}#bordered=true&titled=true"

package/templates/dataif/services/api/app/pnp_dag_provisioner.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from croniter import croniter
+
+
+def _pipeline_slug(value: str) -> str:
+    normalized = "".join(char.lower() if char.isalnum() else "_" for char in value.strip())
+    collapsed = "_".join(part for part in normalized.split("_") if part)
+    for prefix in ("pnp_pipe_", "pipe_", "pnp_"):
+        if collapsed.startswith(prefix):
+            collapsed = collapsed[len(prefix):]
+            break
+    if collapsed.startswith("pnp_"):
+        collapsed = collapsed[len("pnp_"):]
+    return collapsed or "pipeline"
+
+
+def build_pipeline_dag_id(instance_key: str, pipeline_id: str | None = None) -> str:
+    token = ""
+    if pipeline_id:
+        token = pipeline_id.replace("-", "").lower()[:8]
+    slug = _pipeline_slug(instance_key)
+    return f"{slug}_{token}_sync" if token else f"{slug}_sync"
+
+
+def _legacy_pipeline_dag_id(instance_key: str) -> str:
+    normalized = "".join(char.lower() if char.isalnum() else "_" for char in instance_key.strip())
+    collapsed = "_".join(part for part in normalized.split("_") if part)
+    return f"pnp_pipeline__{collapsed or 'instance'}"
+
+
+def _resolve_generated_dags_dir() -> Path:
+    resolved = Path(__file__).resolve()
+    repo_root_candidate = next(
+        (parent for parent in resolved.parents if (parent / "pipelines" / "airflow" / "dags").exists()),
+        None,
+    )
+    candidates = [Path("/app/pipelines/airflow/dags/generated")]
+    if repo_root_candidate is not None:
+        candidates.append(repo_root_candidate / "pipelines" / "airflow" / "dags" / "generated")
+    for candidate in candidates:
+        try:
+            candidate.mkdir(parents=True, exist_ok=True)
+            return candidate
+        except Exception:
+            continue
+    raise RuntimeError("PNP generated DAG directory is not writable")
+
+
+def _dag_file_path(instance_key: str, pipeline_id: str | None = None) -> Path:
+    return _resolve_generated_dags_dir() / f"{build_pipeline_dag_id(instance_key, pipeline_id)}.py"
+
+
+def _legacy_dag_file_path(instance_key: str) -> Path:
+    return _resolve_generated_dags_dir() / f"{_legacy_pipeline_dag_id(instance_key)}.py"
+
+
+def _normalize_airflow_schedule(schedule: str | None, *, is_active: bool) -> str | None:
+    if not is_active or not schedule or not schedule.strip():
+        return None
+    normalized = schedule.strip()
+    if not croniter.is_valid(normalized):
+        return None
+    return normalized
+
+
+def render_pipeline_dag_file(*, dag_id: str, instance_key: str, pipeline_id: str, schedule: str | None, is_active: bool) -> str:
+    normalized_schedule = _normalize_airflow_schedule(schedule, is_active=is_active)
+    schedule_literal = repr(normalized_schedule) if normalized_schedule else "None"
+    return f"""from dataif_pipelines.airflow.pnp_pipeline_factory import build_pipeline_dag
+
+
+dag = build_pipeline_dag(
+    dag_id={dag_id!r},
+    pipeline_id={pipeline_id!r},
+    instance_key={instance_key!r},
+    schedule={schedule_literal},
+)
+"""
+
+
+def provision_pipeline_dag(*, pipeline_id: str, instance_key: str, schedule: str | None, is_active: bool) -> str:
+    dag_id = build_pipeline_dag_id(instance_key, pipeline_id)
+    legacy_target = _legacy_dag_file_path(instance_key)
+    if legacy_target.exists():
+        legacy_target.unlink()
+    target = _dag_file_path(instance_key, pipeline_id)
+    target.write_text(
+        render_pipeline_dag_file(
+            dag_id=dag_id,
+            pipeline_id=pipeline_id,
+            instance_key=instance_key,
+            schedule=schedule,
+            is_active=is_active,
+        ),
+        encoding="utf-8",
+    )
+    if not target.exists():
+        raise RuntimeError(f"Failed to provision DAG file for pipeline {instance_key}")
+    return dag_id
+
+
+def remove_pipeline_dag(*, instance_key: str, pipeline_id: str | None = None) -> str:
+    dag_id = build_pipeline_dag_id(instance_key, pipeline_id)
+    target = _dag_file_path(instance_key, pipeline_id)
+    if target.exists():
+        target.unlink()
+    legacy_target = _legacy_dag_file_path(instance_key)
+    if legacy_target.exists():
+        legacy_target.unlink()
+    return dag_id