ecip-observability-stack 1.0.0

package/ci/ruff-shared.toml

@@ -0,0 +1,41 @@
+# ===========================================================================
+# ECIP Shared Ruff Configuration
+# ===========================================================================
+# Enforces M08 observability contract for Python modules (M02).
+#
+# Install in consuming module's pyproject.toml:
+#   [tool.ruff]
+#   extend = "../ecip-observability-stack/ci/ruff-shared.toml"
+# ===========================================================================
+
+target-version = "py311"
+line-length = 120
+
+[lint]
+select = [
+    "E",    # pycodestyle errors
+    "F",    # pyflakes
+    "W",    # pycodestyle warnings
+    "I",    # isort
+    "N",    # pep8-naming
+    "UP",   # pyupgrade
+    "B",    # flake8-bugbear
+    "A",    # flake8-builtins
+    "C4",   # flake8-comprehensions
+    "SIM",  # flake8-simplify
+    "T20",  # flake8-print — BANS print() in production code
+]
+
+# T20 = flake8-print: flags any use of print() or pprint()
+# This is the Python equivalent of the ESLint console.log ban.
+# Modules must use ecip_observability.get_logger() instead.
+
+[lint.per-file-ignores]
+# Allow print() in test files and scripts
+"**/tests/**" = ["T20"]
+"**/test/**" = ["T20"]
+"**/scripts/**" = ["T20"]
+"conftest.py" = ["T20"]
+
+[lint.isort]
+known-first-party = ["ecip_observability"]

package/collector/otel-collector-config.yaml

@@ -0,0 +1,226 @@
+# =============================================================================
+# ECIP M08 — OpenTelemetry Collector Configuration
+# =============================================================================
+# CRITICAL FILE: An error here silently drops ALL observability data.
+# All changes must be reviewed by the Platform team.
+#
+# Four separate pipelines:
+#   traces         → Grafana Tempo
+#   metrics        → Prometheus (remote write)
+#   logs/security  → Elasticsearch (security events ONLY — NFR-SEC-007)
+#   logs           → Grafana Loki (general application logs — OD-01 resolved)
+# =============================================================================
+
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+
+  # Collector self-metrics
+  prometheus:
+    config:
+      scrape_configs:
+        - job_name: otel-collector
+          scrape_interval: 15s
+          static_configs:
+            - targets: ["0.0.0.0:8888"]
+
+processors:
+  # Memory limiter — prevents OOM on the DaemonSet collector pod
+  memory_limiter:
+    check_interval: 5s
+    limit_mib: 512
+    spike_limit_mib: 128
+
+  # Batch processor — reduces network overhead
+  batch:
+    send_batch_size: 8192
+    send_batch_max_size: 16384
+    timeout: 5s
+
+  # Enforce required ECIP span attributes
+  attributes/enforce:
+    actions:
+      - key: ecip.module
+        action: upsert
+        from_attribute: service.name
+      - key: ecip.org_id
+        action: upsert
+        from_attribute: ecip.org_id
+      - key: ecip.repo_id
+        action: upsert
+        from_attribute: ecip.repo_id
+
+  # Resource detection for Kubernetes metadata
+  resourcedetection:
+    detectors: [env, system, docker, gcp, ecs, ec2]
+    timeout: 5s
+    override: false
+
+  k8sattributes:
+    auth_type: "serviceAccount"
+    passthrough: false
+    extract:
+      metadata:
+        - k8s.pod.name
+        - k8s.pod.uid
+        - k8s.namespace.name
+        - k8s.node.name
+        - k8s.deployment.name
+
+  # Tail-based sampling — decisions made after full trace assembled
+  # Imported from sampling-config.yaml via --config flag
+  tail_sampling:
+    decision_wait: 10s
+    num_traces: 100000
+    expected_new_traces_per_sec: 10000
+    policies:
+      # Rule 1: Always sample error traces (100%)
+      - name: errors-always-sample
+        type: status_code
+        status_code:
+          status_codes:
+            - ERROR
+
+      # Rule 2: Always sample slow traces > 1000ms (100%)
+      - name: slow-queries-sample
+        type: latency
+        latency:
+          threshold_ms: 1000
+
+      # Rule 3: Default sampling — 5% of healthy traces
+      - name: default-sample
+        type: probabilistic
+        probabilistic:
+          sampling_percentage: 5
+
+  # Filter: only security events go to the logs pipeline
+  filter/security_events:
+    logs:
+      include:
+        match_type: strict
+        record_attributes:
+          - key: event.category
+            value: authentication
+          - key: event.category
+            value: authorization
+
+exporters:
+  # Traces → Grafana Tempo
+  otlp/tempo:
+    endpoint: tempo.monitoring:4317
+    tls:
+      insecure: false
+      ca_file: /etc/ssl/certs/ca-certificates.crt
+    retry_on_failure:
+      enabled: true
+      initial_interval: 5s
+      max_interval: 30s
+      max_elapsed_time: 300s
+
+  # Metrics → Prometheus (via remote write)
+  prometheusremotewrite:
+    endpoint: http://prometheus.monitoring:9090/api/v1/write
+    tls:
+      insecure: true
+    resource_to_telemetry_conversion:
+      enabled: true
+
+  # Security logs → Elasticsearch
+  elasticsearch/security:
+    endpoints:
+      - https://elasticsearch.monitoring:9200
+    logs_index: ecip-security-events
+    tls:
+      ca_file: /etc/ssl/certs/ca-certificates.crt
+    retry:
+      enabled: true
+      initial_interval: 5s
+      max_interval: 60s
+    flush:
+      bytes: 5000000
+      interval: 10s
+
+  # General application logs → Grafana Loki (OD-01 resolution)
+  loki:
+    endpoint: http://loki-gateway.monitoring:3100/loki/api/v1/push
+    labels:
+      resource:
+        service.name: "service_name"
+        ecip.module: "module"
+        k8s.namespace.name: "namespace"
+      attributes:
+        level: ""
+    tenant_id: ecip
+    retry_on_failure:
+      enabled: true
+      initial_interval: 5s
+      max_interval: 30s
+
+  # Debug exporter — enabled only in dev/staging
+  logging:
+    loglevel: warn
+
+extensions:
+  health_check:
+    endpoint: 0.0.0.0:13133
+
+  zpages:
+    endpoint: 0.0.0.0:55679
+
+  pprof:
+    endpoint: 0.0.0.0:1777
+
+service:
+  extensions: [health_check, zpages, pprof]
+
+  pipelines:
+    # Traces pipeline: OTLP → process → Tempo
+    traces:
+      receivers: [otlp]
+      processors:
+        - memory_limiter
+        - k8sattributes
+        - resourcedetection
+        - attributes/enforce
+        - tail_sampling
+        - batch
+      exporters: [otlp/tempo]
+
+    # Metrics pipeline: OTLP → process → Prometheus
+    metrics:
+      receivers: [otlp, prometheus]
+      processors:
+        - memory_limiter
+        - k8sattributes
+        - resourcedetection
+        - batch
+      exporters: [prometheusremotewrite]
+
+    # Logs pipeline (security): OTLP → filter security events → Elasticsearch
+    logs/security:
+      receivers: [otlp]
+      processors:
+        - memory_limiter
+        - filter/security_events
+        - batch
+      exporters: [elasticsearch/security]
+
+    # Logs pipeline (general): OTLP → all application logs → Loki
+    logs:
+      receivers: [otlp]
+      processors:
+        - memory_limiter
+        - k8sattributes
+        - resourcedetection
+        - batch
+      exporters: [loki]
+
+  telemetry:
+    logs:
+      level: info
+    metrics:
+      address: 0.0.0.0:8888

package/collector/otel-collector-daemonset.yaml

@@ -0,0 +1,168 @@
+# =============================================================================
+# ECIP M08 — OTel Collector DaemonSet
+# =============================================================================
+# DaemonSet topology: one collector pod per node.
+# Pods communicate via localhost — no cross-node span transit on hot path.
+# =============================================================================
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: otel-collector
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/name: otel-collector
+    app.kubernetes.io/component: observability
+    app.kubernetes.io/part-of: ecip
+    ecip.module: M08
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: otel-collector
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: otel-collector
+        ecip.module: M08
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "8888"
+        prometheus.io/path: "/metrics"
+    spec:
+      serviceAccountName: otel-collector
+      containers:
+        - name: otel-collector
+          image: otel/opentelemetry-collector-contrib:0.96.0
+          args:
+            - --config=/etc/otel/otel-collector-config.yaml
+          ports:
+            - name: otlp-grpc
+              containerPort: 4317
+              hostPort: 4317
+              protocol: TCP
+            - name: otlp-http
+              containerPort: 4318
+              hostPort: 4318
+              protocol: TCP
+            - name: health
+              containerPort: 13133
+              protocol: TCP
+            - name: metrics
+              containerPort: 8888
+              protocol: TCP
+            - name: zpages
+              containerPort: 55679
+              protocol: TCP
+          resources:
+            requests:
+              cpu: 200m
+              memory: 256Mi
+            limits:
+              cpu: 1000m
+              memory: 512Mi
+          volumeMounts:
+            - name: collector-config
+              mountPath: /etc/otel
+              readOnly: true
+            - name: tls-certs
+              mountPath: /etc/ssl/certs
+              readOnly: true
+          livenessProbe:
+            httpGet:
+              path: /
+              port: 13133
+            initialDelaySeconds: 10
+            periodSeconds: 15
+            timeoutSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 13133
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            timeoutSeconds: 3
+          env:
+            - name: K8S_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: K8S_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: K8S_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+      volumes:
+        - name: collector-config
+          configMap:
+            name: otel-collector-config
+        - name: tls-certs
+          secret:
+            secretName: otel-collector-tls
+      tolerations:
+        - effect: NoSchedule
+          operator: Exists
+      terminationGracePeriodSeconds: 30
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: otel-collector
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/name: otel-collector
+spec:
+  type: ClusterIP
+  ports:
+    - name: otlp-grpc
+      port: 4317
+      targetPort: 4317
+      protocol: TCP
+    - name: otlp-http
+      port: 4318
+      targetPort: 4318
+      protocol: TCP
+    - name: metrics
+      port: 8888
+      targetPort: 8888
+      protocol: TCP
+  selector:
+    app.kubernetes.io/name: otel-collector
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: otel-collector
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/name: otel-collector
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: otel-collector
+rules:
+  - apiGroups: [""]
+    resources: ["pods", "namespaces", "nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["apps"]
+    resources: ["deployments", "replicasets", "daemonsets", "statefulsets"]
+    verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: otel-collector
+subjects:
+  - kind: ServiceAccount
+    name: otel-collector
+    namespace: monitoring
+roleRef:
+  kind: ClusterRole
+  name: otel-collector
+  apiGroup: rbac.authorization.k8s.io

package/collector/sampling-config.yaml

@@ -0,0 +1,83 @@
+# =============================================================================
+# ECIP M08 — Tail-Based Sampling Configuration
+# =============================================================================
+# Sampling decisions are made AFTER the full trace is assembled.
+# This ensures error traces are always captured regardless of sample rate.
+#
+# Rules are evaluated in order — first match wins.
+# Default: 5% for healthy traces, 100% for errors and slow traces.
+# =============================================================================
+
+# Decision wait time: how long to wait for all spans of a trace to arrive
+# before making a sampling decision. 10s is conservative — most ECIP
+# traces complete in < 2s even with MCP fan-out.
+decision_wait: 10s
+
+# Maximum number of traces held in memory pending a decision.
+# At 10K new traces/sec and 10s wait, this is ~100K traces in flight.
+num_traces: 100000
+
+# Expected trace arrival rate — used for memory pre-allocation.
+expected_new_traces_per_sec: 10000
+
+policies:
+  # --- Priority 1: Always sample errors ---
+  # Any trace containing a span with status ERROR is sampled at 100%.
+  # This is the most critical rule — production debugging depends on it.
+  - name: errors-always-sample
+    type: status_code
+    status_code:
+      status_codes:
+        - ERROR
+
+  # --- Priority 2: Always sample slow traces ---
+  # Any trace with end-to-end latency > 1000ms is sampled at 100%.
+  # This catches SLA-breaching queries before alerts fire.
+  - name: slow-queries-sample
+    type: latency
+    latency:
+      threshold_ms: 1000
+
+  # --- Priority 3: Always sample security events ---
+  # Traces containing auth/RBAC-related spans are always captured.
+  - name: security-events-sample
+    type: string_attribute
+    string_attribute:
+      key: event.category
+      values:
+        - authentication
+        - authorization
+
+  # --- Priority 4: Sample LSP daemon operations ---
+  # LSP operations are high-value for debugging but high-volume.
+  # Sample at 20% (higher than default) for better coverage.
+  - name: lsp-operations-sample
+    type: string_attribute
+    string_attribute:
+      key: ecip.module
+      values:
+        - M02
+    probabilistic:
+      sampling_percentage: 20
+
+  # --- Default: 5% probabilistic sampling ---
+  # All remaining traces are sampled at 5%.
+  # At 10K traces/sec, this yields ~500 traces/sec to Tempo storage.
+  - name: default-sample
+    type: probabilistic
+    probabilistic:
+      sampling_percentage: 5
+
+# =============================================================================
+# Tuning notes (to be updated after Week 8 load testing):
+#
+# If Tempo storage grows faster than budget:
+#   1. Reduce default from 5% → 2%
+#   2. Reduce lsp-operations from 20% → 10%
+#   3. NEVER reduce errors-always-sample below 100%
+#
+# If Collector memory exceeds limit_mib (512):
+#   1. Reduce num_traces from 100K → 50K
+#   2. Reduce decision_wait from 10s → 5s
+#   3. Increase DaemonSet memory limit (last resort)
+# =============================================================================

package/dashboards/_provisioning/grafana-dashboards.yaml

@@ -0,0 +1,16 @@
+# Grafana Dashboard Provisioning — ECIP Observability
+# Auto-provisions dashboards via Grafana sidecar
+apiVersion: 1
+
+providers:
+  - name: ecip-dashboards
+    orgId: 1
+    folder: ECIP
+    type: file
+    disableDeletion: false
+    editable: true
+    updateIntervalSeconds: 30
+    allowUiUpdates: true
+    options:
+      path: /var/lib/grafana/dashboards/ecip
+      foldersFromFilesStructure: false

package/dashboards/analysis-throughput.json

@@ -0,0 +1,166 @@
+{
+  "description": "ECIP Analysis Engine (M02) — Events processed, backlog, Kafka consumer lag",
+  "editable": true,
+  "panels": [
+    {
+      "title": "Analysis Throughput",
+      "type": "row",
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
+      "collapsed": false
+    },
+    {
+      "title": "Analysis Duration p50 / p95",
+      "type": "timeseries",
+      "datasource": "Prometheus",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 60000 },
+              { "color": "red", "value": 120000 }
+            ]
+          }
+        }
+      },
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.50, sum(rate(analysis_duration_ms_bucket{job=\"ecip-analysis-engine\"}[5m])) by (le))",
+          "legendFormat": "p50"
+        },
+        {
+          "expr": "histogram_quantile(0.95, sum(rate(analysis_duration_ms_bucket{job=\"ecip-analysis-engine\"}[5m])) by (le))",
+          "legendFormat": "p95"
+        }
+      ]
+    },
+    {
+      "title": "Analysis Duration by Branch Type",
+      "type": "timeseries",
+      "datasource": "Prometheus",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 },
+      "fieldConfig": {
+        "defaults": { "unit": "ms" }
+      },
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.95, sum(rate(analysis_duration_ms_bucket{job=\"ecip-analysis-engine\"}[5m])) by (le, branch_type))",
+          "legendFormat": "p95 — {{branch_type}}"
+        }
+      ]
+    },
+    {
+      "title": "Events Processed / sec",
+      "type": "timeseries",
+      "datasource": "Prometheus",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 },
+      "fieldConfig": {
+        "defaults": { "unit": "ops" }
+      },
+      "targets": [
+        {
+          "expr": "sum(rate(analysis_duration_ms_count{job=\"ecip-analysis-engine\"}[5m]))",
+          "legendFormat": "Events/s"
+        }
+      ]
+    },
+    {
+      "title": "Kafka Consumer Lag (Analysis Topics)",
+      "type": "timeseries",
+      "datasource": "Prometheus",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 9 },
+      "fieldConfig": {
+        "defaults": { "unit": "short" }
+      },
+      "targets": [
+        {
+          "expr": "sum(kafka_consumergroup_lag{group=~\"ecip-analysis.*\"}) by (topic)",
+          "legendFormat": "{{topic}}"
+        }
+      ]
+    },
+    {
+      "title": "Analysis Duration by Language (p95)",
+      "type": "bargauge",
+      "datasource": "Prometheus",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 17 },
+      "fieldConfig": {
+        "defaults": { "unit": "ms" }
+      },
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.95, sum(rate(analysis_duration_ms_bucket{job=\"ecip-analysis-engine\"}[5m])) by (le, language))",
+          "legendFormat": "{{language}}",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "title": "Event Backlog",
+      "type": "stat",
+      "datasource": "Prometheus",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 17 },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 500 },
+              { "color": "red", "value": 1000 }
+            ]
+          }
+        }
+      },
+      "targets": [
+        {
+          "expr": "sum(kafka_consumergroup_lag{group=~\"ecip-analysis.*\"})",
+          "legendFormat": "Total Backlog"
+        }
+      ]
+    },
+    {
+      "title": "Embedding Migration Progress",
+      "type": "gauge",
+      "datasource": "Prometheus",
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 25 },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "min": 0,
+          "max": 1
+        }
+      },
+      "targets": [
+        {
+          "expr": "embedding_migration_progress{phase=\"cutover\"}",
+          "legendFormat": "{{repo}} — {{phase}}"
+        }
+      ]
+    }
+  ],
+  "refresh": "30s",
+  "schemaVersion": 39,
+  "tags": ["ecip", "m02", "analysis-engine", "throughput"],
+  "templating": {
+    "list": [
+      {
+        "name": "repo",
+        "type": "query",
+        "datasource": "Prometheus",
+        "query": "label_values(analysis_duration_ms_bucket, repo)",
+        "refresh": 2,
+        "includeAll": true,
+        "multi": true
+      }
+    ]
+  },
+  "time": { "from": "now-1h", "to": "now" },
+  "title": "ECIP — Analysis Throughput",
+  "uid": "ecip-analysis-throughput",
+  "version": 1
+}