@pleri/olam-cli 0.1.159 → 0.1.161

package/host-cp/observability/loki-ingest.sh

@@ -0,0 +1,243 @@
+#!/usr/bin/env bash
+# loki-ingest.sh — e2e smoke test: Loki single-binary installs, Promtail tails,
+#                  OAuth query-param scrubbing verified (code=REDACTED, no raw token).
+#
+# Usage: scripts/e2e/loki-ingest.sh
+#
+# Pre-conditions:
+#   - kubectl context is set to a live k8s cluster (does NOT spin up k3d)
+#   - helm binary available
+#   - grafana Helm repo added (helm repo add grafana https://grafana.github.io/helm-charts)
+#
+# This script is invoked by the A12 harness (scripts/test-ingress-integration/)
+# after cluster-up.sh. It can also be run manually against any live cluster.
+#
+# Idempotency: `helm upgrade --install` is idempotent; re-runs succeed on an
+#              existing cluster. The synthetic pod is cleaned up regardless of
+#              pass/fail via a trap.
+#
+# Refs: docs/plans/k3s-ingress-observability/phase-b-tasks.md — Task B1
+#       Chart: grafana/loki 6.7.4 (pinned; latest stable 2026-05-20)
+#       Chart: grafana/promtail 6.16.6 (latest stable 2026-05-20)
+
+set -euo pipefail
+
+NAMESPACE="monitoring"
+LOKI_RELEASE="olam-loki"
+PROMTAIL_RELEASE="olam-promtail"
+SYNTHETIC_POD="loki-e2e-synthetic"
+LOKI_PORT="3100"
+LOCAL_PORT="13100"   # avoid conflict with any host-level Loki
+
+# Magic-number commentary: Promtail's tail → ingest cycle involves:
+#   - inotify event (near-instant)
+#   - Promtail pipeline processing (~1s)
+#   - Loki write path (ingester chunk idle period: default 30m, but flush on
+#     query pressure; typically <5s in practice)
+# 10s is conservative for a single log line in a lightly loaded cluster.
+INGEST_LAG_SECONDS=10
+
+log()  { printf '[loki-ingest] %s\n' "$*" >&2; }
+fail() { printf '[loki-ingest] FAIL: %s\n' "$*" >&2; exit 1; }
+
+# -------------------------------------------------------------------------
+# Cleanup trap — remove synthetic pod and port-forward on exit
+# -------------------------------------------------------------------------
+PF_PID=""
+cleanup() {
+  if [[ -n "$PF_PID" ]] && kill -0 "$PF_PID" 2>/dev/null; then
+    kill "$PF_PID" 2>/dev/null || true
+  fi
+  kubectl delete pod "$SYNTHETIC_POD" -n default --ignore-not-found=true 2>/dev/null || true
+}
+trap cleanup EXIT
+
+# -------------------------------------------------------------------------
+# Pre-flight
+# -------------------------------------------------------------------------
+command -v helm    >/dev/null 2>&1 || fail "helm not installed"
+command -v kubectl >/dev/null 2>&1 || fail "kubectl not installed"
+command -v curl    >/dev/null 2>&1 || fail "curl not installed"
+kubectl cluster-info >/dev/null 2>&1 || fail "kubectl: no reachable cluster; set KUBECONFIG"
+
+log "pre-flight checks passed"
+
+# -------------------------------------------------------------------------
+# Resolve repo root so helm -f paths work regardless of invocation cwd
+# -------------------------------------------------------------------------
+REPO_ROOT="$(git -C "$(dirname "$0")" rev-parse --show-toplevel 2>/dev/null || pwd)"
+
+# -------------------------------------------------------------------------
+# Ensure grafana Helm repo is present (idempotent — safe to re-run)
+# -------------------------------------------------------------------------
+helm repo add grafana https://grafana.github.io/helm-charts 2>/dev/null || true
+helm repo update grafana
+
+# -------------------------------------------------------------------------
+# Step 1: Install / upgrade Loki (single-binary mode)
+# -------------------------------------------------------------------------
+log "installing grafana/loki ($LOKI_RELEASE) in namespace $NAMESPACE"
+helm upgrade --install "$LOKI_RELEASE" grafana/loki \
+  --version 6.7.4 \
+  --namespace "$NAMESPACE" \
+  --create-namespace \
+  -f "$REPO_ROOT/packages/peripheral-services/helm-values/loki-values.yaml" \
+  --wait \
+  --timeout 300s
+
+log "loki helm install complete"
+
+# -------------------------------------------------------------------------
+# Step 2: Install / upgrade Promtail
+# -------------------------------------------------------------------------
+log "installing grafana/promtail ($PROMTAIL_RELEASE) in namespace $NAMESPACE"
+helm upgrade --install "$PROMTAIL_RELEASE" grafana/promtail \
+  --version 6.16.6 \
+  --namespace "$NAMESPACE" \
+  -f "$REPO_ROOT/packages/peripheral-services/helm-values/promtail-values.yaml" \
+  --wait \
+  --timeout 120s
+
+log "promtail helm install complete"
+
+# -------------------------------------------------------------------------
+# Step 3: Wait for Loki pod Ready
+# -------------------------------------------------------------------------
+log "waiting for Loki pod Ready (120s)"
+kubectl wait \
+  --for=condition=ready pod \
+  -l app.kubernetes.io/name=loki \
+  -n "$NAMESPACE" \
+  --timeout=120s
+
+log "loki pod Ready"
+
+# -------------------------------------------------------------------------
+# Step 4: Generate synthetic log line with raw OAuth tokens in URL and headers.
+#
+# The pod prints a single log line containing all 4 scrub patterns:
+#   ?code=SECRETTOKEN123        → code=REDACTED
+#   &access_token=SECRETTOKEN456 → access_token=REDACTED
+#   &state=SESSION789            → state=REDACTED
+#   Authorization: Bearer SECRETBEARER000 → Authorization: Bearer REDACTED
+#
+# Promtail tails it, runs the scrubbing pipeline, and pushes to Loki with all
+# 4 raw tokens absent and all 4 REDACTED markers present.
+# -------------------------------------------------------------------------
+log "launching synthetic pod (prints all 4 raw token patterns)"
+kubectl run "$SYNTHETIC_POD" \
+  --image=busybox \
+  --restart=Never \
+  -n default \
+  -- sh -c 'echo "GET http://example.com/callback?code=SECRETTOKEN123&access_token=SECRETTOKEN456&state=SESSION789 HTTP/1.1 Authorization: Bearer SECRETBEARER000"'
+
+# -------------------------------------------------------------------------
+# Step 5: Wait for Promtail tail + ingest lag
+# -------------------------------------------------------------------------
+log "waiting ${INGEST_LAG_SECONDS}s for Promtail to tail and ingest synthetic log"
+sleep "$INGEST_LAG_SECONDS"
+
+# -------------------------------------------------------------------------
+# Step 6: Port-forward Loki and query
+# -------------------------------------------------------------------------
+log "port-forwarding Loki svc to localhost:${LOCAL_PORT}"
+kubectl port-forward \
+  "svc/${LOKI_RELEASE}" \
+  "${LOCAL_PORT}:${LOKI_PORT}" \
+  -n "$NAMESPACE" &
+PF_PID=$!
+
+# Give port-forward a moment to establish
+sleep 2
+
+# Query Loki for log lines from the default namespace within the last 5 minutes.
+# We search broadly for "SECRETTOKEN" to catch any raw token that leaked through,
+# and separately verify all 4 REDACTED markers are present.
+log "querying Loki for scrubbed entries"
+QUERY_RESPONSE=$(
+  curl -s -G \
+    "http://localhost:${LOCAL_PORT}/loki/api/v1/query_range" \
+    --data-urlencode 'query={namespace="default"} |= "REDACTED"' \
+    --data-urlencode "start=$(date -u -v-5M +%s 2>/dev/null || date -u -d '5 minutes ago' +%s)000000000" \
+    --data-urlencode "end=$(date -u +%s)000000000" \
+    --data-urlencode 'limit=50'
+)
+
+# -------------------------------------------------------------------------
+# Step 7: Assertions — verify all 4 scrub patterns
+#
+# Contract (matches Phase B spec + promtail-values.yaml):
+#   ?code=SECRETTOKEN123        → code=REDACTED       (absent: SECRETTOKEN123)
+#   &access_token=SECRETTOKEN456 → access_token=REDACTED (absent: SECRETTOKEN456)
+#   &state=SESSION789            → state=REDACTED      (absent: SESSION789)
+#   Authorization: Bearer SECRETBEARER000 → Bearer REDACTED (absent: SECRETBEARER000)
+# -------------------------------------------------------------------------
+log "asserting scrubbing correctness (all 4 patterns)"
+
+diag() {
+  log "DIAGNOSTIC: Loki query response:"
+  echo "$QUERY_RESPONSE" >&2
+  log "DIAGNOSTIC: last 50 lines of Promtail logs:"
+  kubectl logs -n "$NAMESPACE" -l app.kubernetes.io/name=promtail --tail=50 2>&1 >&2 || true
+}
+
+# Assertion 1: query response is non-empty (Loki returned results)
+if ! echo "$QUERY_RESPONSE" | grep -q '"result"'; then
+  diag
+  fail "Loki returned no result block — Promtail may not have ingested the synthetic log yet"
+fi
+
+# --- Scrubbed markers present ---
+
+# Assertion 2a: code= is scrubbed
+if ! echo "$QUERY_RESPONSE" | grep -q 'code=REDACTED'; then
+  diag
+  fail "'code=REDACTED' not found in Loki response — code= scrub stage not working"
+fi
+
+# Assertion 2b: access_token= is scrubbed
+if ! echo "$QUERY_RESPONSE" | grep -q 'access_token=REDACTED'; then
+  diag
+  fail "'access_token=REDACTED' not found in Loki response — access_token= scrub stage not working"
+fi
+
+# Assertion 2c: state= is scrubbed
+if ! echo "$QUERY_RESPONSE" | grep -q 'state=REDACTED'; then
+  diag
+  fail "'state=REDACTED' not found in Loki response — state= scrub stage not working"
+fi
+
+# Assertion 2d: Authorization Bearer is scrubbed
+if ! echo "$QUERY_RESPONSE" | grep -q 'Bearer REDACTED'; then
+  diag
+  fail "'Bearer REDACTED' not found in Loki response — Authorization Bearer scrub stage not working"
+fi
+
+# --- Raw tokens absent ---
+
+# Assertion 3a: raw code= token is absent
+if echo "$QUERY_RESPONSE" | grep -q 'SECRETTOKEN123'; then
+  diag
+  fail "raw token 'SECRETTOKEN123' (code=) found in Loki response — scrubbing pipeline is NOT working"
+fi
+
+# Assertion 3b: raw access_token= token is absent
+if echo "$QUERY_RESPONSE" | grep -q 'SECRETTOKEN456'; then
+  diag
+  fail "raw token 'SECRETTOKEN456' (access_token=) found in Loki response — scrubbing pipeline is NOT working"
+fi
+
+# Assertion 3c: raw state= token is absent
+if echo "$QUERY_RESPONSE" | grep -q 'SESSION789'; then
+  diag
+  fail "raw token 'SESSION789' (state=) found in Loki response — scrubbing pipeline is NOT working"
+fi
+
+# Assertion 3d: raw Bearer token is absent
+if echo "$QUERY_RESPONSE" | grep -q 'SECRETBEARER000'; then
+  diag
+  fail "raw token 'SECRETBEARER000' (Authorization Bearer) found in Loki response — scrubbing pipeline is NOT working"
+fi
+
+log "PASS: all 4 scrub patterns verified — code=REDACTED, access_token=REDACTED, state=REDACTED, Bearer REDACTED present; all raw tokens absent"
+exit 0

package/host-cp/observability/prom-no-double-grafana.sh

@@ -0,0 +1,301 @@
+#!/usr/bin/env bash
+# prom-no-double-grafana.sh — Phase C Task C1 e2e smoke test.
+#
+# Verifies:
+#   1. kube-prometheus-stack installs (Prometheus pod becomes Ready).
+#   2. ServiceMonitor CRD is Established before Phase B charts are upgraded.
+#   3. Phase B charts (Loki + Promtail + Grafana) are helm-upgraded to pick up
+#      serviceMonitor.enabled: true now that the CRD exists.
+#   4. Exactly one Grafana Deployment is running in the cluster (no double-Grafana).
+#   5. Phase B's Grafana (olam-grafana) has exactly one Prometheus datasource
+#      provisioned (from grafana-values.yaml datasources block added in C1).
+#   6. Prometheus is scraping at least one active target.
+#
+# Pre-conditions:
+#   - kubectl context is set to a live k8s cluster.
+#   - Phase B e2e (loki-ingest.sh + grafana-port-forward.sh + grafana-dashboard-persistence.sh)
+#     has already run: olam-loki, olam-promtail, and olam-grafana releases are installed.
+#   - The olam-grafana-admin Secret exists (created by grafana-port-forward.sh).
+#   - helm, kubectl, curl, jq binaries available.
+#
+# Chart: prometheus-community/kube-prometheus-stack 85.2.0 (pinned; latest stable 2026-05-21).
+#
+# Idempotency: helm upgrade --install is idempotent; re-runs on an existing
+# cluster succeed. Port-forwards are killed on exit via trap.
+#
+# Refs: docs/plans/k3s-ingress-observability/phase-c-tasks.md — Task C1
+
+set -euo pipefail
+
+NAMESPACE="monitoring"
+PROM_RELEASE="olam-prom"
+PROM_CHART_VERSION="85.2.0"
+GRAFANA_RELEASE="olam-grafana"
+GRAFANA_LOCAL_PORT="3001"   # avoid collision if phase-b-e2e left a port-forward on 3000
+GRAFANA_SVC_PORT="80"
+PROM_LOCAL_PORT="9090"
+PF_BIND_SECONDS=5
+
+log()  { printf '[prom-no-double-grafana] %s\n' "$*" >&2; }
+fail() { printf '[prom-no-double-grafana] FAIL: %s\n' "$*" >&2; exit 1; }
+
+# -------------------------------------------------------------------------
+# Resolve repo root so helm -f paths work regardless of invocation cwd
+# -------------------------------------------------------------------------
+REPO_ROOT="$(git -C "$(dirname "$0")" rev-parse --show-toplevel 2>/dev/null || pwd)"
+
+# -------------------------------------------------------------------------
+# Cleanup trap — kill port-forwards on exit; leave Helm releases in place
+# -------------------------------------------------------------------------
+GRAFANA_PF_PID=""
+PROM_PF_PID=""
+cleanup() {
+  [[ -n "$GRAFANA_PF_PID" ]] && kill "$GRAFANA_PF_PID" 2>/dev/null || true
+  [[ -n "$PROM_PF_PID"   ]] && kill "$PROM_PF_PID"   2>/dev/null || true
+}
+trap cleanup EXIT
+
+# -------------------------------------------------------------------------
+# Pre-flight
+# -------------------------------------------------------------------------
+command -v helm    >/dev/null 2>&1 || fail "helm not installed"
+command -v kubectl >/dev/null 2>&1 || fail "kubectl not installed"
+command -v curl    >/dev/null 2>&1 || fail "curl not installed"
+command -v jq      >/dev/null 2>&1 || fail "jq not installed"
+kubectl cluster-info >/dev/null 2>&1 || fail "kubectl: no reachable cluster; set KUBECONFIG"
+
+log "pre-flight checks passed"
+
+# Verify Phase B pre-conditions
+for release in olam-loki olam-promtail "$GRAFANA_RELEASE"; do
+  helm status "$release" -n "$NAMESPACE" >/dev/null 2>&1 \
+    || fail "Phase B release '$release' not found in namespace $NAMESPACE — run phase-b-e2e first"
+done
+log "Phase B pre-conditions satisfied (olam-loki, olam-promtail, olam-grafana releases found)"
+
+# -------------------------------------------------------------------------
+# Step 1: Add prometheus-community repo and install kube-prometheus-stack
+# -------------------------------------------------------------------------
+helm repo add prometheus-community https://prometheus-community.github.io/helm-charts 2>/dev/null || true
+helm repo update prometheus-community
+
+log "installing prometheus-community/kube-prometheus-stack ($PROM_RELEASE) version $PROM_CHART_VERSION"
+helm upgrade --install "$PROM_RELEASE" prometheus-community/kube-prometheus-stack \
+  --version "$PROM_CHART_VERSION" \
+  --namespace "$NAMESPACE" \
+  --create-namespace \
+  -f "$REPO_ROOT/packages/peripheral-services/helm-values/kube-prom-stack-values.yaml" \
+  --wait \
+  --timeout 600s
+
+log "kube-prometheus-stack helm install complete"
+
+# -------------------------------------------------------------------------
+# Step 2: Wait for ServiceMonitor CRD to be Established
+# This is the gate before upgrading Phase B charts — the CRD must exist
+# for serviceMonitor.enabled: true to produce a valid ServiceMonitor object.
+# -------------------------------------------------------------------------
+log "waiting for ServiceMonitor CRD to be Established (60s)"
+kubectl wait \
+  --for=condition=established \
+  crd/servicemonitors.monitoring.coreos.com \
+  --timeout=60s
+
+log "ServiceMonitor CRD Established"
+
+# -------------------------------------------------------------------------
+# Step 3: Helm-upgrade Phase B charts to enable ServiceMonitor at RUNTIME
+#
+# The source-of-truth values files keep serviceMonitor.enabled: false so a
+# standalone Phase B install (without kube-prometheus-stack) does not
+# hard-fail with "no matches for kind ServiceMonitor". We flip the toggle
+# at runtime here, AFTER the CRD is Established, via --set overrides. This
+# preserves Phase B's standalone-installability invariant while wiring
+# Prometheus discovery when kube-prom-stack is present.
+#
+# NOTE: Loki 6.7.4 uses monitoring.serviceMonitor (not top-level serviceMonitor)
+# — chart-version-specific path.
+# -------------------------------------------------------------------------
+# Chart version pins MUST match the ones in phase-b-e2e's loki-ingest.sh +
+# grafana-port-forward.sh. Without --version, helm pulls latest from the repo;
+# the latest charts may reference new template values not present in our
+# values files (e.g., Loki 6.8.x references .Values.loki.ui.enabled which is
+# nil in our 6.7.4-shaped values, producing a nil-pointer template error
+# during upgrade).
+LOKI_CHART_VERSION="6.7.4"
+PROMTAIL_CHART_VERSION="6.16.6"
+GRAFANA_CHART_VERSION="8.5.2"
+
+log "upgrading Phase B charts with runtime --set serviceMonitor.enabled=true (pinned versions)"
+
+helm upgrade olam-loki grafana/loki \
+  --version "$LOKI_CHART_VERSION" \
+  --namespace "$NAMESPACE" \
+  -f "$REPO_ROOT/packages/peripheral-services/helm-values/loki-values.yaml" \
+  --wait \
+  --timeout 300s \
+  --reuse-values \
+  --set monitoring.serviceMonitor.enabled=true
+
+log "olam-loki upgraded (ServiceMonitor enabled)"
+
+helm upgrade olam-promtail grafana/promtail \
+  --version "$PROMTAIL_CHART_VERSION" \
+  --namespace "$NAMESPACE" \
+  -f "$REPO_ROOT/packages/peripheral-services/helm-values/promtail-values.yaml" \
+  --wait \
+  --timeout 300s \
+  --reuse-values \
+  --set serviceMonitor.enabled=true
+
+log "olam-promtail upgraded (ServiceMonitor enabled)"
+
+helm upgrade "$GRAFANA_RELEASE" grafana/grafana \
+  --version "$GRAFANA_CHART_VERSION" \
+  --namespace "$NAMESPACE" \
+  -f "$REPO_ROOT/packages/peripheral-services/helm-values/grafana-values.yaml" \
+  --wait \
+  --timeout 300s \
+  --reuse-values \
+  --set serviceMonitor.enabled=true
+
+log "$GRAFANA_RELEASE upgraded (ServiceMonitor enabled; Prometheus datasource provisioned)"
+
+# -------------------------------------------------------------------------
+# Step 4: Wait for Prometheus pod Ready
+# -------------------------------------------------------------------------
+log "waiting for Prometheus pod Ready (300s)"
+kubectl wait \
+  --for=condition=ready pod \
+  -l "app.kubernetes.io/name=prometheus" \
+  -n "$NAMESPACE" \
+  --timeout=300s
+
+log "Prometheus pod Ready"
+
+# -------------------------------------------------------------------------
+# Step 5: Assertion — exactly one Grafana Deployment in the cluster
+# This catches any regression where kube-prometheus-stack's bundled Grafana
+# sub-chart accidentally gets enabled.
+# -------------------------------------------------------------------------
+log "asserting exactly 1 Grafana Deployment in namespace $NAMESPACE"
+GRAFANA_DEPS=$(kubectl get deployment \
+  -n "$NAMESPACE" \
+  -l "app.kubernetes.io/name=grafana" \
+  -o name \
+  | wc -l \
+  | tr -d ' ')
+
+if [ "$GRAFANA_DEPS" != "1" ]; then
+  log "FAIL: expected exactly 1 Grafana Deployment, found $GRAFANA_DEPS"
+  kubectl get deployment -n "$NAMESPACE" -l "app.kubernetes.io/name=grafana" >&2
+  fail "double-Grafana detected — kube-prometheus-stack's grafana.enabled must be false"
+fi
+
+log "PASS: exactly 1 Grafana Deployment found"
+
+# -------------------------------------------------------------------------
+# Step 6: Assertion — Grafana has exactly one Prometheus datasource
+# Re-read the admin password from the Secret (grafana-port-forward.sh created it).
+# Use port 3001 to avoid colliding with any live phase-b-e2e port-forward on 3000.
+# -------------------------------------------------------------------------
+log "reading admin password from Secret olam-grafana-admin"
+GRAFANA_ADMIN_PW=$(kubectl get secret olam-grafana-admin \
+  -n "$NAMESPACE" \
+  -o jsonpath='{.data.admin-password}' \
+  | base64 -d)
+
+log "port-forwarding svc/$GRAFANA_RELEASE $GRAFANA_LOCAL_PORT:$GRAFANA_SVC_PORT"
+kubectl port-forward \
+  -n "$NAMESPACE" \
+  "svc/$GRAFANA_RELEASE" \
+  "${GRAFANA_LOCAL_PORT}:${GRAFANA_SVC_PORT}" &
+GRAFANA_PF_PID=$!
+
+log "waiting ${PF_BIND_SECONDS}s for Grafana port-forward to bind"
+sleep "$PF_BIND_SECONDS"
+kill -0 "$GRAFANA_PF_PID" 2>/dev/null \
+  || fail "Grafana port-forward process exited prematurely"
+
+log "asserting exactly 1 Prometheus datasource in Grafana (GET /api/datasources)"
+DATASOURCES=$(curl -sf \
+  -u "admin:${GRAFANA_ADMIN_PW}" \
+  "http://localhost:${GRAFANA_LOCAL_PORT}/api/datasources" \
+  || { kubectl logs -n "$NAMESPACE" -l "app.kubernetes.io/name=grafana" --tail=30 >&2 || true
+       fail "GET /api/datasources failed — Grafana not reachable on port $GRAFANA_LOCAL_PORT"; })
+
+if ! echo "$DATASOURCES" | jq -e 'map(select(.type == "prometheus")) | length == 1' >/dev/null 2>&1; then
+  log "FAIL: Grafana does not have exactly 1 Prometheus datasource"
+  echo "$DATASOURCES" | jq . >&2
+  fail "Prometheus datasource not provisioned — check datasources block in grafana-values.yaml"
+fi
+
+PROM_URL=$(echo "$DATASOURCES" | jq -r 'map(select(.type == "prometheus")) | .[0].url')
+log "PASS: Grafana has exactly 1 Prometheus datasource (url=$PROM_URL)"
+
+# -------------------------------------------------------------------------
+# Step 7: Assertion — Prometheus is scraping at least one active target
+# -------------------------------------------------------------------------
+log "port-forwarding svc/prometheus-operated $PROM_LOCAL_PORT:9090"
+kubectl port-forward \
+  -n "$NAMESPACE" \
+  "svc/prometheus-operated" \
+  "${PROM_LOCAL_PORT}:9090" &
+PROM_PF_PID=$!
+
+log "waiting ${PF_BIND_SECONDS}s for Prometheus port-forward to bind"
+sleep "$PF_BIND_SECONDS"
+kill -0 "$PROM_PF_PID" 2>/dev/null \
+  || fail "Prometheus port-forward process exited prematurely"
+
+log "querying Prometheus /api/v1/targets for active targets"
+TARGETS=$(curl -sf "http://localhost:${PROM_LOCAL_PORT}/api/v1/targets" \
+  || fail "GET /api/v1/targets failed — Prometheus not reachable on port $PROM_LOCAL_PORT")
+
+ACTIVE=$(echo "$TARGETS" | jq '.data.activeTargets | length')
+if [ "$ACTIVE" -lt 1 ]; then
+  log "FAIL: Prometheus has 0 active scrape targets"
+  echo "$TARGETS" | jq '.data.activeTargets' >&2
+  fail "Prometheus has no active targets — check ServiceMonitor CRD and scrapeConfig"
+fi
+
+log "PASS: $ACTIVE active scrape target(s) found in Prometheus"
+
+# -------------------------------------------------------------------------
+# Assertion C4: Recording rules from 95-prom-recording-rules.yaml are loaded
+#
+# The 9[0-9]-prom-* glob in apply-manifests.sh skips this file (requires
+# kube-prom-stack CRDs to exist). We kubectl apply it here, then poll
+# /api/v1/rules until the olam-http-aggregations group appears.
+# The port-forward on PROM_LOCAL_PORT is already open from Step 7 above.
+# -------------------------------------------------------------------------
+PROM_URL="http://localhost:${PROM_LOCAL_PORT}"
+
+log "applying 95-prom-recording-rules.yaml (skipped by apply-manifests due to 9[0-9]-prom-* filter)"
+kubectl apply -f "$REPO_ROOT/packages/peripheral-services/manifests/95-prom-recording-rules.yaml"
+
+# Prometheus operator reconcile + config reload can take ~60-90s (C2 lesson).
+# Poll /api/v1/rules until our group appears (up to 180s).
+RECORDING_RULES_TIMEOUT=180
+log "polling ${PROM_URL}/api/v1/rules for olam-http-aggregations group (up to ${RECORDING_RULES_TIMEOUT}s)"
+elapsed=0
+while [ "$elapsed" -lt "$RECORDING_RULES_TIMEOUT" ]; do
+  if curl -sf "${PROM_URL}/api/v1/rules" 2>/dev/null \
+       | jq -e '.data.groups[] | select(.name == "olam-http-aggregations") | .rules[] | select(.name == "olam:http_requests:rate5m_by_service")' >/dev/null 2>&1; then
+    log "PASS: olam-http-aggregations rule group loaded after ${elapsed}s"
+    break
+  fi
+  sleep 10
+  elapsed=$((elapsed + 10))
+done
+if [ "$elapsed" -ge "$RECORDING_RULES_TIMEOUT" ]; then
+  log "FAIL: olam-http-aggregations rule group not found in /api/v1/rules within ${RECORDING_RULES_TIMEOUT}s"
+  curl -sf "${PROM_URL}/api/v1/rules" | jq '.data.groups[] | .name' >&2 || true
+  fail "PrometheusRule not loaded by operator"
+fi
+
+# -------------------------------------------------------------------------
+# Final
+# -------------------------------------------------------------------------
+log "PASS: kube-prometheus-stack installed; single Grafana confirmed; Prometheus datasource provisioned; $ACTIVE active target(s); recording rules loaded — Tasks C1+C4 verified"
+exit 0