@pleri/olam-cli 0.1.160 → 0.1.161

package/host-cp/observability/grafana-port-forward.sh

@@ -0,0 +1,273 @@
+#!/usr/bin/env bash
+# grafana-port-forward.sh — e2e smoke test: Grafana installs via Helm,
+#                           port-forward is accessible, Loki datasource
+#                           is pre-wired and reachable.
+#
+# Usage: scripts/e2e/grafana-port-forward.sh
+#
+# Pre-conditions:
+#   - kubectl context is set to a live k8s cluster (does NOT spin up k3d)
+#   - helm binary available
+#   - jq binary available
+#   - grafana Helm repo added (helm repo add grafana https://grafana.github.io/helm-charts)
+#   - Loki is already installed (scripts/e2e/loki-ingest.sh ran successfully
+#     OR `helm status olam-loki -n monitoring` is healthy)
+#
+# Idempotency: `helm upgrade --install` is idempotent; re-runs succeed on an
+#              existing cluster. The Secret is applied via --dry-run | kubectl apply
+#              so re-runs update the password (useful for rotation testing).
+#              The olam-dashboards ConfigMap is applied before helm install so
+#              Grafana's volume mount finds the ConfigMap on first boot.
+#
+# Cleanup: port-forward is killed on exit; Helm release is left in place so
+#          downstream tasks can reuse the same cluster.
+#
+# Refs: docs/plans/k3s-ingress-observability/phase-b-tasks.md — Task B2, B3
+#       Chart: grafana/grafana 8.5.2 (pinned; latest stable 2026-05-20)
+
+set -euo pipefail
+
+NAMESPACE="monitoring"
+GRAFANA_RELEASE="olam-grafana"
+GRAFANA_CHART_VERSION="8.5.2"
+LOCAL_PORT="3000"
+GRAFANA_SVC_PORT="80"
+PF_BIND_SECONDS=5
+
+log()  { printf '[grafana-port-forward] %s\n' "$*" >&2; }
+fail() { printf '[grafana-port-forward] FAIL: %s\n' "$*" >&2; exit 1; }
+
+# -------------------------------------------------------------------------
+# Cleanup trap — kill port-forward on exit; leave Helm release in place
+# -------------------------------------------------------------------------
+PF_PID=""
+cleanup() {
+  if [[ -n "$PF_PID" ]] && kill -0 "$PF_PID" 2>/dev/null; then
+    kill "$PF_PID" 2>/dev/null || true
+  fi
+}
+trap cleanup EXIT
+
+# -------------------------------------------------------------------------
+# Pre-flight
+# -------------------------------------------------------------------------
+command -v helm    >/dev/null 2>&1 || fail "helm not installed"
+command -v kubectl >/dev/null 2>&1 || fail "kubectl not installed"
+command -v curl    >/dev/null 2>&1 || fail "curl not installed"
+command -v openssl >/dev/null 2>&1 || fail "openssl not installed"
+command -v jq      >/dev/null 2>&1 || fail "jq not installed (required for B3 dashboard assertion)"
+kubectl cluster-info >/dev/null 2>&1 || fail "kubectl: no reachable cluster; set KUBECONFIG"
+
+log "pre-flight checks passed"
+
+# -------------------------------------------------------------------------
+# Ensure grafana Helm repo is present (idempotent — safe to re-run)
+# -------------------------------------------------------------------------
+helm repo add grafana https://grafana.github.io/helm-charts 2>/dev/null || true
+helm repo update grafana
+
+# Verify Loki is already installed (B2 depends on B1)
+if ! helm status "olam-loki" -n "$NAMESPACE" >/dev/null 2>&1; then
+  fail "olam-loki Helm release not found in namespace $NAMESPACE — run scripts/e2e/loki-ingest.sh first"
+fi
+log "Loki pre-condition satisfied (olam-loki release found)"
+
+# -------------------------------------------------------------------------
+# Step 1: Resolve admin password (preserve existing on idempotent re-run)
+# -------------------------------------------------------------------------
+# Grafana persists the admin password in its internal SQLite on first
+# deploy. Subsequent helm upgrades do NOT re-read GF_SECURITY_ADMIN_PASSWORD
+# from the env (env value is set once at pod-start and not refreshed). So
+# on a re-run, rotating the Secret leaves the in-Grafana password stale
+# and breaks API auth.
+#
+# Idempotency contract: if the Secret already exists, reuse its current
+# password. The Secret's value matches Grafana's stored value (set in
+# concert on first install). Only generate a new password when the
+# Secret doesn't exist yet — i.e. true first deploy.
+if kubectl get secret olam-grafana-admin -n "$NAMESPACE" >/dev/null 2>&1; then
+  log "reusing existing admin password from Secret olam-grafana-admin"
+  GRAFANA_ADMIN_PW=$(kubectl get secret olam-grafana-admin -n "$NAMESPACE" \
+    -o jsonpath='{.data.admin-password}' | base64 -d)
+else
+  log "generating fresh admin password (first deploy)"
+  GRAFANA_ADMIN_PW=$(openssl rand -base64 24)
+fi
+export GRAFANA_ADMIN_PW
+
+# -------------------------------------------------------------------------
+# Step 2: Create / update the admin Secret idempotently
+# -------------------------------------------------------------------------
+log "applying Secret olam-grafana-admin in namespace $NAMESPACE"
+kubectl create secret generic olam-grafana-admin \
+  --from-literal=admin-user=admin \
+  --from-literal=admin-password="$GRAFANA_ADMIN_PW" \
+  -n "$NAMESPACE" \
+  --dry-run=client -o yaml \
+  | kubectl apply -f -
+
+log "Secret applied"
+
+# -------------------------------------------------------------------------
+# Step 3a: Apply olam-dashboards ConfigMap BEFORE helm install
+#          so Grafana's volume mount finds it on first boot (B3).
+#          The ConfigMap is generated from grafana-dashboards/*.json by
+#          packages/peripheral-services/scripts/sync-grafana-dashboards.sh.
+# -------------------------------------------------------------------------
+REPO_ROOT="$(git -C "$(dirname "$0")" rev-parse --show-toplevel 2>/dev/null || pwd)"
+CONFIGMAP_MANIFEST="$REPO_ROOT/packages/peripheral-services/manifests/80-grafana-dashboard-configmap.yaml"
+
+if [[ -f "$CONFIGMAP_MANIFEST" ]]; then
+  log "applying olam-dashboards ConfigMap from $CONFIGMAP_MANIFEST"
+  kubectl apply -f "$CONFIGMAP_MANIFEST"
+  log "ConfigMap applied"
+else
+  log "WARN: $CONFIGMAP_MANIFEST not found — Grafana will warn 'ConfigMap not found' until B3 is deployed"
+fi
+
+# -------------------------------------------------------------------------
+# Step 3: Helm upgrade --install
+# -------------------------------------------------------------------------
+log "installing grafana/grafana ($GRAFANA_RELEASE) in namespace $NAMESPACE"
+helm upgrade --install "$GRAFANA_RELEASE" grafana/grafana \
+  --version "$GRAFANA_CHART_VERSION" \
+  --namespace "$NAMESPACE" \
+  --create-namespace \
+  -f "$REPO_ROOT/packages/peripheral-services/helm-values/grafana-values.yaml" \
+  --wait \
+  --timeout 300s
+
+log "Grafana Helm install complete"
+
+# -------------------------------------------------------------------------
+# Step 4: Wait for Grafana pod Ready
+# -------------------------------------------------------------------------
+log "waiting for Grafana pod Ready (120s)"
+kubectl wait \
+  --for=condition=ready pod \
+  -l "app.kubernetes.io/name=grafana" \
+  -n "$NAMESPACE" \
+  --timeout=120s
+
+log "Grafana pod Ready"
+
+# -------------------------------------------------------------------------
+# Step 5: Start port-forward in background
+# -------------------------------------------------------------------------
+log "port-forwarding svc/$GRAFANA_RELEASE $LOCAL_PORT:$GRAFANA_SVC_PORT in namespace $NAMESPACE"
+kubectl port-forward \
+  -n "$NAMESPACE" \
+  "svc/$GRAFANA_RELEASE" \
+  "${LOCAL_PORT}:${GRAFANA_SVC_PORT}" &
+PF_PID=$!
+
+log "port-forward PID $PF_PID; waiting ${PF_BIND_SECONDS}s for bind"
+sleep "$PF_BIND_SECONDS"
+
+# Verify the port-forward process is still alive after sleep
+kill -0 "$PF_PID" 2>/dev/null || fail "port-forward process exited prematurely"
+
+# -------------------------------------------------------------------------
+# Diagnostic helper — called on assertion failure
+# -------------------------------------------------------------------------
+dump_diagnostics() {
+  log "DIAGNOSTIC: last 50 lines of Grafana pod logs:"
+  kubectl logs -n "$NAMESPACE" \
+    -l "app.kubernetes.io/name=grafana" \
+    --tail=50 2>&1 >&2 || true
+}
+
+# -------------------------------------------------------------------------
+# Step 6: Assertion 1 — /api/health returns 200 with database: ok
+# -------------------------------------------------------------------------
+log "asserting Grafana health (GET /api/health)"
+HEALTH_RESPONSE=$(
+  curl -sf \
+    -u "admin:${GRAFANA_ADMIN_PW}" \
+    "http://localhost:${LOCAL_PORT}/api/health" \
+  || { dump_diagnostics; fail "GET /api/health failed — Grafana not reachable on port $LOCAL_PORT"; }
+)
+
+if ! echo "$HEALTH_RESPONSE" | jq -e '.database == "ok"' >/dev/null 2>&1; then
+  log "DIAGNOSTIC: /api/health response:"
+  echo "$HEALTH_RESPONSE" >&2
+  dump_diagnostics
+  fail '/api/health returned database != "ok" — Grafana DB layer not healthy'
+fi
+
+log "PASS: /api/health → database: ok"
+
+# -------------------------------------------------------------------------
+# Step 7: Assertion 2 — /api/datasources includes Loki entry with cluster URL
+# -------------------------------------------------------------------------
+log "asserting Loki datasource pre-wired (GET /api/datasources)"
+DS_RESPONSE=$(
+  curl -sf \
+    -u "admin:${GRAFANA_ADMIN_PW}" \
+    "http://localhost:${LOCAL_PORT}/api/datasources" \
+  || { dump_diagnostics; fail "GET /api/datasources failed"; }
+)
+
+EXPECTED_URL="olam-loki.monitoring.svc.cluster.local:3100"
+
+if ! echo "$DS_RESPONSE" | jq -e 'map(select(.type == "loki")) | length >= 1' >/dev/null 2>&1; then
+  log "DIAGNOSTIC: /api/datasources response:"
+  echo "$DS_RESPONSE" >&2
+  dump_diagnostics
+  fail "datasources response contains no 'loki' type entry — datasource not provisioned"
+fi
+
+if ! echo "$DS_RESPONSE" | jq -e --arg url "$EXPECTED_URL" 'map(select(.type == "loki" and (.url | contains($url)))) | length >= 1' >/dev/null 2>&1; then
+  log "DIAGNOSTIC: /api/datasources response:"
+  echo "$DS_RESPONSE" >&2
+  dump_diagnostics
+  fail "Loki datasource URL does not contain '$EXPECTED_URL' — check grafana-values.yaml datasources block"
+fi
+
+log "PASS: Loki datasource found with cluster-local URL $EXPECTED_URL"
+
+# -------------------------------------------------------------------------
+# Step 7b: Assertion 2b — dashboard provider loaded olam-home (catches mount-path bugs)
+# -------------------------------------------------------------------------
+log "asserting olam-home dashboard visible in /api/search (catches ConfigMap mount failures)"
+DASHBOARDS=$(
+  curl -sf \
+    -u "admin:${GRAFANA_ADMIN_PW}" \
+    "http://localhost:${LOCAL_PORT}/api/search?type=dash-db&query=olam" \
+  || true
+)
+
+if ! echo "$DASHBOARDS" | jq -e 'map(select(.uid == "olam-home")) | length == 1' >/dev/null 2>&1; then
+  log "DIAGNOSTIC: /api/search response:"
+  echo "$DASHBOARDS" >&2
+  dump_diagnostics
+  fail "olam-home dashboard not found in /api/search — check ConfigMap mount path and dashboard provider config"
+fi
+
+log "PASS: olam-home dashboard found via /api/search"
+
+# -------------------------------------------------------------------------
+# Step 8: Assertion 3 — olam-home dashboard present (B3)
+# -------------------------------------------------------------------------
+log "asserting olam-home dashboard present (GET /api/dashboards/uid/olam-home)"
+DASHBOARD_RESPONSE=$(
+  curl -sf \
+    -u "admin:${GRAFANA_ADMIN_PW}" \
+    "http://localhost:${LOCAL_PORT}/api/dashboards/uid/olam-home" \
+  || { dump_diagnostics; fail "GET /api/dashboards/uid/olam-home failed — dashboard not found or Grafana unreachable"; }
+)
+
+if ! echo "$DASHBOARD_RESPONSE" | jq -e '.dashboard.uid == "olam-home"' >/dev/null 2>&1; then
+  log "DIAGNOSTIC: /api/dashboards/uid/olam-home response:"
+  echo "$DASHBOARD_RESPONSE" >&2
+  dump_diagnostics
+  fail "olam-home dashboard uid mismatch or missing — check ConfigMap provisioning and Grafana provider config"
+fi
+
+log "PASS: olam-home dashboard present with uid=olam-home"
+
+# -------------------------------------------------------------------------
+# Final
+# -------------------------------------------------------------------------
+log "PASS: Grafana port-forward accessible; Loki datasource pre-wired; olam-home dashboard provisioned — Tasks B2+B3 verified"
+exit 0