@pleri/olam-cli 0.1.159 → 0.1.161

package/host-cp/observability/kyverno-cardinality-mutate.sh

@@ -0,0 +1,452 @@
+#!/usr/bin/env bash
+# kyverno-cardinality-mutate.sh — Phase C C8 follow-up e2e smoke test.
+#
+# Verifies that the Kyverno ClusterPolicy
+# `enforce-cardinality-labeldrop` mutates incoming ServiceMonitor and
+# PodMonitor objects at admission time, regardless of authorship,
+# closing codex's "policy by convention" gap on PR #783.
+#
+# Test approach:
+#   1. helm-install Kyverno (pinned 3.8.1) into the `kyverno` namespace.
+#   2. Apply the ClusterPolicy.
+#   3. POSITIVE test: apply ServiceMonitor `kyverno-mutate-positive-test`
+#      with selector `app: kyverno-mutate-positive-test` (no backing Service)
+#      and NO metricRelabelings; assert Kyverno mutated it; delete immediately.
+#   4. IDEMPOTENCY test: apply ServiceMonitor `kyverno-mutate-idempotency-test`
+#      with selector `app: kyverno-mutate-idempotency-test` (different non-existent
+#      label) and the labeldrop already present; assert count stays at 1; delete.
+#   5. SCRAPE-VERIFICATION test: deploy synthetic `kyverno-emitter` (Service +
+#      Deployment + ConfigMap) + dedicated ServiceMonitor `kyverno-emitter-sm`
+#      applied WITHOUT metricRelabelings; assert Kyverno mutates the SM at admission;
+#      wait for pod Ready; poll Prometheus for http_requests_total; assert
+#      world_id label is ABSENT.
+#
+# Key design decision: POSITIVE and IDEMPOTENCY tests use selectors that match
+# no real Service, so they are isolated from each other and from the SCRAPE test.
+# A single dedicated SM (`kyverno-emitter-sm`) owns the emitter endpoint, so
+# prometheus-operator can reliably reconcile exactly one scrape config for it.
+# Root cause of the prior failure (PR #828 CI run 26239574154): two SMs
+# (naive-violator + pre-armoured-violator) competed for the same
+# `app: kyverno-emitter` Endpoints; operator never reconciled either.
+#
+# Pre-conditions:
+#   - kube-prometheus-stack installed (cardinality-drop.sh ran).
+#   - kubectl context set to a live cluster; helm + jq + curl available.
+#
+# Idempotency: kubectl apply is idempotent; helm upgrade --install is
+# idempotent. Cleanup trap removes synthetic resources on exit. The
+# ClusterPolicy + Kyverno install are LEFT in the cluster (permanent
+# C8 fixtures).
+#
+# Refs: docs/plans/k3s-ingress-observability/phase-c-tasks.md — C8
+#       codex review on PR #783 ("policy by convention" finding)
+#       PR #828 CI run 26239574154 (competing-SM root cause)
+
+set -euo pipefail
+
+KYVERNO_VERSION="3.8.1"
+KYVERNO_NAMESPACE="kyverno"
+TEST_NAMESPACE="monitoring"
+PROM_LOCAL_PORT="9092"   # 9090, 9091 may be in use by sibling Phase C scripts
+PF_BIND_SECONDS=5
+TARGET_DISCOVERY_TIMEOUT=180
+SCRAPE_POLL_INTERVAL=10
+
+log()  { printf '[kyverno-mutate] %s\n' "$*" >&2; }
+fail() { printf '[kyverno-mutate] FAIL: %s\n' "$*" >&2; exit 1; }
+
+REPO_ROOT="$(git -C "$(dirname "$0")" rev-parse --show-toplevel 2>/dev/null || pwd)"
+
+# -------------------------------------------------------------------------
+# Cleanup trap — kill port-forwards; remove synthetic resources on exit.
+# Kyverno chart + ClusterPolicy stay (permanent C8 fixtures).
+# -------------------------------------------------------------------------
+PROM_PF_PID=""
+cleanup() {
+  [[ -n "$PROM_PF_PID" ]] && kill "$PROM_PF_PID" 2>/dev/null || true
+  log "removing synthetic resources (idempotent)"
+  # Mutation-test SMs (already deleted inline, but --ignore-not-found makes this safe)
+  kubectl delete servicemonitor kyverno-mutate-positive-test    -n "$TEST_NAMESPACE" --ignore-not-found=true 2>/dev/null || true
+  kubectl delete servicemonitor kyverno-mutate-idempotency-test -n "$TEST_NAMESPACE" --ignore-not-found=true 2>/dev/null || true
+  # Scrape-verification resources
+  kubectl delete servicemonitor kyverno-emitter-sm     -n "$TEST_NAMESPACE" --ignore-not-found=true 2>/dev/null || true
+  kubectl delete deployment     kyverno-emitter        -n "$TEST_NAMESPACE" --ignore-not-found=true 2>/dev/null || true
+  kubectl delete service        kyverno-emitter-svc    -n "$TEST_NAMESPACE" --ignore-not-found=true 2>/dev/null || true
+  kubectl delete configmap      kyverno-emitter-config -n "$TEST_NAMESPACE" --ignore-not-found=true 2>/dev/null || true
+}
+trap cleanup EXIT
+
+# -------------------------------------------------------------------------
+# Pre-flight
+# -------------------------------------------------------------------------
+command -v helm    >/dev/null 2>&1 || fail "helm not installed"
+command -v kubectl >/dev/null 2>&1 || fail "kubectl not installed"
+command -v curl    >/dev/null 2>&1 || fail "curl not installed"
+command -v jq      >/dev/null 2>&1 || fail "jq not installed"
+kubectl cluster-info >/dev/null 2>&1 || fail "kubectl: no reachable cluster; set KUBECONFIG"
+
+# kube-prom-stack must already be up — we rely on Prometheus + the
+# ServiceMonitor CRD existing.
+kubectl get crd servicemonitors.monitoring.coreos.com >/dev/null 2>&1 \
+  || fail "ServiceMonitor CRD not present — run prom-no-double-grafana.sh first"
+kubectl get deployment -n "$TEST_NAMESPACE" -l "app.kubernetes.io/name=prometheus-operator" \
+  >/dev/null 2>&1 \
+  || fail "prometheus-operator not found in $TEST_NAMESPACE — run prom-no-double-grafana.sh first"
+
+log "pre-flight checks passed"
+
+# -------------------------------------------------------------------------
+# Step 1: helm-install Kyverno
+#
+# Repo add is idempotent; helm upgrade --install handles fresh install + upgrade.
+# `--wait` blocks until pods are Ready; admission webhook needs to be live
+# before we apply the ClusterPolicy or our test ServiceMonitors.
+# -------------------------------------------------------------------------
+log "ensuring kyverno helm repo is configured"
+helm repo add kyverno https://kyverno.github.io/kyverno/ >/dev/null 2>&1 || true
+helm repo update kyverno >/dev/null 2>&1 || true
+
+log "installing kyverno chart $KYVERNO_VERSION (waits for admission webhook Ready)"
+helm upgrade --install olam-kyverno kyverno/kyverno \
+  --version "$KYVERNO_VERSION" \
+  --namespace "$KYVERNO_NAMESPACE" \
+  --create-namespace \
+  -f "$REPO_ROOT/packages/peripheral-services/helm-values/kyverno-values.yaml" \
+  --wait --timeout 300s 2>&1 | tail -8
+
+# Sanity: kyverno-admission-controller Deployment Ready.
+kubectl get deployment -n "$KYVERNO_NAMESPACE" -l "app.kubernetes.io/component=admission-controller" \
+  >/dev/null 2>&1 \
+  || fail "kyverno admission controller not found in $KYVERNO_NAMESPACE"
+
+log "waiting for kyverno admission webhook to be registered with apiserver"
+# The webhook registration is the LAST thing kyverno does after pod-Ready;
+# poll until our ClusterPolicy can be admitted.
+elapsed=0
+while [ "$elapsed" -lt 120 ]; do
+  if kubectl get validatingwebhookconfiguration kyverno-policy-validating-webhook-cfg \
+       >/dev/null 2>&1; then
+    log "kyverno webhooks registered after ${elapsed}s"
+    break
+  fi
+  sleep 5
+  elapsed=$((elapsed + 5))
+done
+if [ "$elapsed" -ge 120 ]; then
+  fail "kyverno webhook registration timed out after 120s"
+fi
+
+# -------------------------------------------------------------------------
+# Step 2: Apply the ClusterPolicy
+# -------------------------------------------------------------------------
+log "applying ClusterPolicy enforce-cardinality-labeldrop"
+kubectl apply -f "$REPO_ROOT/packages/peripheral-services/manifests/96-kyverno-cardinality-mutate.yaml"
+
+# Wait for policy to be Ready (Kyverno controller picks it up and reports
+# readiness in status.ready / .conditions).
+log "waiting up to 60s for ClusterPolicy to be Ready"
+elapsed=0
+while [ "$elapsed" -lt 60 ]; do
+  READY=$(kubectl get clusterpolicy enforce-cardinality-labeldrop \
+    -o jsonpath='{.status.ready}' 2>/dev/null || echo "")
+  if [ "$READY" = "true" ]; then
+    log "ClusterPolicy Ready after ${elapsed}s"
+    break
+  fi
+  sleep 3
+  elapsed=$((elapsed + 3))
+done
+if [ "$elapsed" -ge 60 ]; then
+  log "WARN: ClusterPolicy status.ready not observed within 60s; proceeding (status field can lag)"
+fi
+
+# -------------------------------------------------------------------------
+# Step 3: POSITIVE test — mutation only, no backing Service
+#
+# Uses selector `app: kyverno-mutate-positive-test` — a label that no
+# real Service carries, so this SM never competes with anything for
+# Endpoints. Its sole job is to exercise the Kyverno admission webhook.
+#
+# Deleted immediately after assertion so the SM space is clean when
+# the scrape test runs.
+# -------------------------------------------------------------------------
+log "POSITIVE test: applying naive ServiceMonitor (no metricRelabelings, non-Service-backed selector)"
+kubectl apply -f - <<'EOF'
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: kyverno-mutate-positive-test
+  namespace: monitoring
+  labels:
+    release: olam-prom
+spec:
+  namespaceSelector:
+    matchNames:
+      - monitoring
+  selector:
+    matchLabels:
+      app: kyverno-mutate-positive-test
+  endpoints:
+    - port: metrics
+      interval: 15s
+      # NOTE: deliberately NO metricRelabelings — Kyverno must inject it.
+EOF
+
+# Read back and assert.
+ACTUAL=$(kubectl get servicemonitor kyverno-mutate-positive-test -n "$TEST_NAMESPACE" -o json \
+  | jq -r '.spec.endpoints[0].metricRelabelings // [] | tojson')
+log "kyverno-mutate-positive-test metricRelabelings after admission: $ACTUAL"
+
+INJECTED_COUNT=$(echo "$ACTUAL" | jq '[ .[] | select(.action == "labeldrop" and (.regex | contains("world_id"))) ] | length')
+if [ "$INJECTED_COUNT" -lt 1 ]; then
+  log "actual policy state:"
+  kubectl get clusterpolicy enforce-cardinality-labeldrop -o yaml >&2 || true
+  fail "POSITIVE test FAILED: Kyverno did not inject labeldrop into naive ServiceMonitor — third-party bypass gap NOT closed"
+fi
+log "PASS: naive ServiceMonitor was mutated at admission (labeldrop injected)"
+
+log "deleting kyverno-mutate-positive-test (mutation-only test; SM space clean for scrape test)"
+kubectl delete servicemonitor kyverno-mutate-positive-test -n "$TEST_NAMESPACE" --ignore-not-found=true
+
+# -------------------------------------------------------------------------
+# Step 4: IDEMPOTENCY test — mutation only, no backing Service
+#
+# Uses selector `app: kyverno-mutate-idempotency-test` — different from
+# the positive test and from the scrape test label. No real Service.
+# Deleted immediately after assertion.
+# -------------------------------------------------------------------------
+log "IDEMPOTENCY test: applying pre-armoured ServiceMonitor (labeldrop already present)"
+kubectl apply -f - <<'EOF'
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: kyverno-mutate-idempotency-test
+  namespace: monitoring
+  labels:
+    release: olam-prom
+spec:
+  namespaceSelector:
+    matchNames:
+      - monitoring
+  selector:
+    matchLabels:
+      app: kyverno-mutate-idempotency-test
+  endpoints:
+    - port: metrics
+      interval: 15s
+      metricRelabelings:
+        - action: labeldrop
+          regex: 'world_id|trace_id|user_id|request_id|operator_id'
+EOF
+
+DUP_COUNT=$(kubectl get servicemonitor kyverno-mutate-idempotency-test -n "$TEST_NAMESPACE" -o json \
+  | jq '[ .spec.endpoints[0].metricRelabelings[] | select(.action == "labeldrop" and (.regex | contains("world_id"))) ] | length')
+log "kyverno-mutate-idempotency-test labeldrop count: $DUP_COUNT"
+if [ "$DUP_COUNT" -ne 1 ]; then
+  kubectl get servicemonitor kyverno-mutate-idempotency-test -n "$TEST_NAMESPACE" -o yaml >&2
+  fail "IDEMPOTENCY test FAILED: expected 1 labeldrop entry, got $DUP_COUNT — policy double-adds"
+fi
+log "PASS: pre-armoured ServiceMonitor has exactly 1 labeldrop (no double-add)"
+
+log "deleting kyverno-mutate-idempotency-test (mutation-only test; SM space clean for scrape test)"
+kubectl delete servicemonitor kyverno-mutate-idempotency-test -n "$TEST_NAMESPACE" --ignore-not-found=true
+
+# -------------------------------------------------------------------------
+# Step 5: SCRAPE-VERIFICATION test — dedicated SM + Service + Pod
+#
+# One SM (`kyverno-emitter-sm`) selects exactly one Service (`kyverno-emitter-svc`).
+# No other SM in the cluster selects `app: kyverno-emitter`, so prometheus-operator
+# reconciles a single clean scrape config.
+#
+# The SM is applied WITHOUT metricRelabelings so Kyverno's admission webhook
+# fires — this is the load-bearing check that the policy applies during real
+# scrape setup, not just on test fixtures.
+#
+# After admission we verify the spec has the labeldrop, then wait for the pod
+# to be Ready and poll Prometheus for http_requests_total. We assert
+# world_id is absent from all returned series.
+#
+# Mirrors the working pattern from dashboards-have-data.sh (single dedicated
+# SM + co-located Service in `monitoring` namespace).
+# -------------------------------------------------------------------------
+log "SCRAPE-VERIFICATION test: deploying synthetic kyverno-emitter (emits http_requests_total{world_id})"
+kubectl apply -f - <<'EOF'
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kyverno-emitter-config
+  namespace: monitoring
+data:
+  metrics: |
+    # HELP http_requests_total Synthetic counter; world_id is the cardinality bomb
+    # TYPE http_requests_total counter
+    http_requests_total{world_id="kyverno-world",route="/api",method="GET",status_code="200"} 1
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kyverno-emitter
+  namespace: monitoring
+  labels:
+    app: kyverno-emitter
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: kyverno-emitter
+  template:
+    metadata:
+      labels:
+        app: kyverno-emitter
+    spec:
+      containers:
+        - name: emitter
+          image: python:3.11-alpine
+          ports:
+            - containerPort: 8080
+          command: ["python3", "-c"]
+          args:
+            - |
+              import http.server
+              with open('/config/metrics') as f: METRICS = f.read().encode()
+              class H(http.server.BaseHTTPRequestHandler):
+                  def do_GET(self):
+                      if self.path != '/metrics':
+                          self.send_response(404); self.end_headers(); return
+                      self.send_response(200)
+                      self.send_header('Content-Type', 'text/plain; version=0.0.4; charset=utf-8')
+                      self.end_headers()
+                      self.wfile.write(METRICS)
+                  def log_message(self, *a): pass
+              http.server.HTTPServer(('0.0.0.0', 8080), H).serve_forever()
+          volumeMounts:
+            - name: config
+              mountPath: /config
+      volumes:
+        - name: config
+          configMap:
+            name: kyverno-emitter-config
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: kyverno-emitter-svc
+  namespace: monitoring
+  labels:
+    app: kyverno-emitter
+spec:
+  selector:
+    app: kyverno-emitter
+  ports:
+    - name: metrics
+      port: 8080
+      targetPort: 8080
+EOF
+
+log "waiting for kyverno-emitter deployment Ready"
+kubectl rollout status deployment/kyverno-emitter -n "$TEST_NAMESPACE" --timeout=120s
+
+# Apply the dedicated ServiceMonitor WITHOUT metricRelabelings so Kyverno
+# mutates it at admission — this proves the policy fires on real SM objects,
+# not just on the POSITIVE test fixture.
+log "applying kyverno-emitter-sm (no metricRelabelings — Kyverno must inject)"
+kubectl apply -f - <<'EOF'
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: kyverno-emitter-sm
+  namespace: monitoring
+  labels:
+    release: olam-prom
+spec:
+  namespaceSelector:
+    matchNames:
+      - monitoring
+  selector:
+    matchLabels:
+      app: kyverno-emitter
+  endpoints:
+    - port: metrics
+      interval: 15s
+      # NOTE: NO metricRelabelings — Kyverno must inject the labeldrop at admission.
+EOF
+
+# Verify Kyverno mutated this SM too (belt-and-suspenders: proves the policy
+# applies to the SM that actually drives the scrape, not just the test fixtures).
+SCRAPE_SM_ACTUAL=$(kubectl get servicemonitor kyverno-emitter-sm -n "$TEST_NAMESPACE" -o json \
+  | jq -r '.spec.endpoints[0].metricRelabelings // [] | tojson')
+log "kyverno-emitter-sm metricRelabelings after admission: $SCRAPE_SM_ACTUAL"
+
+SCRAPE_SM_INJECTED=$(echo "$SCRAPE_SM_ACTUAL" | jq '[ .[] | select(.action == "labeldrop" and (.regex | contains("world_id"))) ] | length')
+if [ "$SCRAPE_SM_INJECTED" -lt 1 ]; then
+  log "actual policy state:"
+  kubectl get clusterpolicy enforce-cardinality-labeldrop -o yaml >&2 || true
+  fail "SCRAPE-VERIFICATION test FAILED: Kyverno did not mutate kyverno-emitter-sm at admission"
+fi
+log "PASS: kyverno-emitter-sm was mutated at admission (labeldrop injected)"
+
+# Port-forward Prometheus and poll for metric samples.
+log "port-forwarding svc/prometheus-operated $PROM_LOCAL_PORT:9090"
+kubectl port-forward \
+  -n "$TEST_NAMESPACE" \
+  "svc/prometheus-operated" \
+  "${PROM_LOCAL_PORT}:9090" &
+PROM_PF_PID=$!
+sleep "$PF_BIND_SECONDS"
+kill -0 "$PROM_PF_PID" 2>/dev/null \
+  || fail "Prometheus port-forward exited prematurely"
+
+PROM_URL="http://localhost:${PROM_LOCAL_PORT}"
+
+# Direct-metric polling rather than target-discovery polling.
+#
+# Rationale: kube-prometheus-stack's default relabel sets the `job` label
+# from the k8s Service name. Polling by job-name is brittle — operator
+# reconciliation races, dropped-target filtering, and rare CRD revision
+# lag have all surfaced as "target not in activeTargets" flakes during
+# earlier ingress-integration runs. What we ACTUALLY care about is
+# whether the mutated relabel was applied to a real scrape sample. So
+# poll for the metric directly. With a single SM selecting on
+# `app=kyverno-emitter`, any http_requests_total series returned
+# necessarily came through kyverno-emitter-sm.
+log "polling Prometheus for http_requests_total samples (up to ${TARGET_DISCOVERY_TIMEOUT}s)"
+elapsed=0
+RESULT=""
+while [ "$elapsed" -lt "$TARGET_DISCOVERY_TIMEOUT" ]; do
+  RESULT=$(curl -sf "${PROM_URL}/api/v1/query?query=http_requests_total" 2>/dev/null || echo "")
+  if [ -n "$RESULT" ]; then
+    SERIES_COUNT=$(echo "$RESULT" | jq '.data.result | length' 2>/dev/null || echo "0")
+    if [ "$SERIES_COUNT" -ge 1 ]; then
+      log "http_requests_total returned $SERIES_COUNT series after ${elapsed}s"
+      break
+    fi
+  fi
+  sleep "$SCRAPE_POLL_INTERVAL"
+  elapsed=$((elapsed + SCRAPE_POLL_INTERVAL))
+done
+
+if [ "$elapsed" -ge "$TARGET_DISCOVERY_TIMEOUT" ]; then
+  log "Active targets snapshot for diagnosis:"
+  curl -sf "${PROM_URL}/api/v1/targets" | jq '.data.activeTargets[] | {job: .labels.job, service: .labels.service, namespace: .labels.namespace, health: .health, lastError: .lastError}' >&2 || true
+  log "ServiceMonitor kyverno-emitter-sm status:"
+  kubectl get servicemonitor kyverno-emitter-sm -n "$TEST_NAMESPACE" -o yaml >&2 || true
+  log "prometheus-operator log tail (last 50 lines):"
+  kubectl logs -n "$TEST_NAMESPACE" -l "app.kubernetes.io/name=prometheus-operator" --tail=50 >&2 || true
+  fail "Prometheus did not scrape kyverno-emitter within ${TARGET_DISCOVERY_TIMEOUT}s"
+fi
+
+SERIES_COUNT=$(echo "$RESULT" | jq '.data.result | length')
+
+LEAKED=$(echo "$RESULT" | jq '[.data.result[] | .metric | has("world_id")] | any')
+if [ "$LEAKED" = "true" ]; then
+  echo "$RESULT" | jq '.data.result[] | .metric' >&2
+  fail "world_id label leaked into Prometheus — Kyverno-mutated relabel did NOT take effect at scrape time"
+fi
+
+log "PASS: kyverno-emitter scraped via kyverno-emitter-sm; world_id absent at scrape time"
+log "PASS: C8 verified — Kyverno mutates third-party-shaped ServiceMonitors at admission and the mutation takes effect at scrape time"
+exit 0