@thotischner/observability-mcp 1.2.1 → 1.3.0

package/dist/connectors/prometheus.d.ts

@@ -10,7 +10,7 @@ export declare class PrometheusConnector implements ObservabilityConnector {
     private metrics;
     private serviceLabels;
     private labelValuesCache;
-    private metricNamesCache;
+    private candidateCache;
     private userOverrides;
     connect(config: SourceConfig): Promise<void>;
     getDefaultMetrics(): MetricDefinition[];
@@ -23,8 +23,10 @@ export declare class PrometheusConnector implements ObservabilityConnector {
     listAvailableMetrics(_service: string): Promise<MetricInfo[]>;
     queryMetrics(params: MetricQuery): Promise<MetricResult>;
     private buildQuery;
+    private groupKey;
+    private getDistinctLabelValues;
     private pickMetricCandidate;
-    private getAllMetricNames;
+    private seriesExistsForService;
     private resolveServiceLabel;
     private getLabelValues;
     private getUnit;

package/dist/connectors/prometheus.js

@@ -3,11 +3,13 @@ const PROMETHEUS_METRIC_CANDIDATES = {
     cpu: [
         {
             seriesName: "process_cpu_seconds_total",
+            // rate() preserves all labels — already broken down per-instance.
             query: 'rate(process_cpu_seconds_total{ {{selector}} }[1m]) * 100',
         },
         {
             seriesName: "node_cpu_seconds_total",
             query: '100 - avg(rate(node_cpu_seconds_total{ {{selector}}, mode="idle" }[1m])) * 100',
+            groupedQuery: '100 - avg by({{groupBy}}) (rate(node_cpu_seconds_total{ {{selector}}, mode="idle" }[1m])) * 100',
         },
     ],
     memory: [
@@ -24,33 +26,40 @@ const PROMETHEUS_METRIC_CANDIDATES = {
         {
             seriesName: "http_requests_total",
             query: 'sum(rate(http_requests_total{ {{selector}} }[1m]))',
+            groupedQuery: 'sum by({{groupBy}}) (rate(http_requests_total{ {{selector}} }[1m]))',
         },
     ],
     error_rate: [
         {
             seriesName: "http_requests_total",
             query: 'sum(rate(http_requests_total{ {{selector}}, status=~"5.." }[1m]))',
+            groupedQuery: 'sum by({{groupBy}}) (rate(http_requests_total{ {{selector}}, status=~"5.." }[1m]))',
         },
     ],
     latency_p99: [
         {
             seriesName: "http_request_duration_seconds_bucket",
             query: 'histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{ {{selector}} }[1m])) by (le))',
+            groupedQuery: 'histogram_quantile(0.99, sum by(le, {{groupBy}}) (rate(http_request_duration_seconds_bucket{ {{selector}} }[1m])))',
         },
     ],
     latency_p50: [
         {
             seriesName: "http_request_duration_seconds_bucket",
             query: 'histogram_quantile(0.50, sum(rate(http_request_duration_seconds_bucket{ {{selector}} }[1m])) by (le))',
+            groupedQuery: 'histogram_quantile(0.50, sum by(le, {{groupBy}}) (rate(http_request_duration_seconds_bucket{ {{selector}} }[1m])))',
         },
     ],
     latency_avg: [
         {
             seriesName: "http_request_duration_seconds_sum",
             query: 'sum(rate(http_request_duration_seconds_sum{ {{selector}} }[1m])) / sum(rate(http_request_duration_seconds_count{ {{selector}} }[1m]))',
+            groupedQuery: 'sum by({{groupBy}}) (rate(http_request_duration_seconds_sum{ {{selector}} }[1m])) / sum by({{groupBy}}) (rate(http_request_duration_seconds_count{ {{selector}} }[1m]))',
         },
     ],
 };
+// Common breakdown labels probed for the auto-hint when no groupBy is set.
+const HINT_BREAKDOWN_LABELS = ["instance", "pod"];
 const DEFAULT_METRIC_META = {
     cpu: { unit: "percent", description: "CPU usage % (auto: prom-client process_cpu_seconds_total or node_exporter node_cpu_seconds_total)" },
     memory: { unit: "bytes", description: "Resident memory bytes (auto: prom-client process_resident_memory_bytes or node_memory used)" },
@@ -78,7 +87,7 @@ export class PrometheusConnector {
     metrics = [];
     serviceLabels = DEFAULT_SERVICE_LABELS;
     labelValuesCache = new Map();
-    metricNamesCache = null;
+    candidateCache = new Map();
     userOverrides = new Set();
     async connect(config) {
         this.name = config.name;
@@ -187,83 +196,168 @@ export class PrometheusConnector {
         return metrics;
     }
     async queryMetrics(params) {
-        const { promql, label } = await this.buildQuery(params.service, params.metric);
+        const { promql, label, candidate } = await this.buildQuery(params.service, params.metric, params.groupBy);
         const { start, end, step } = this.parseTimeRange(params.duration, params.step);
         const data = await this.apiGet(`/api/v1/query_range?query=${encodeURIComponent(promql)}&start=${start}&end=${end}&step=${step}`);
-        const values = [];
-        const rawValues = [];
-        const resultData = data?.data?.result?.[0]?.values || [];
-        for (const [ts, val] of resultData) {
-            const numVal = parseFloat(val);
-            if (!isNaN(numVal)) {
-                values.push({ timestamp: new Date(ts * 1000).toISOString(), value: numVal });
-                rawValues.push(numVal);
+        const seriesList = data?.data?.result || [];
+        // Build groups from each returned series, keyed either by the explicit
+        // groupBy label (when grouped) or by a synthesized name from any
+        // remaining labels (for naturally per-instance queries like cpu/memory
+        // prom-client). Empty when nothing came back.
+        const groups = [];
+        for (const series of seriesList) {
+            const seriesValues = [];
+            const rawValues = [];
+            for (const [ts, val] of series.values || []) {
+                const numVal = parseFloat(val);
+                if (!isNaN(numVal)) {
+                    seriesValues.push({ timestamp: new Date(ts * 1000).toISOString(), value: numVal });
+                    rawValues.push(numVal);
+                }
             }
+            groups.push({
+                key: this.groupKey(series.metric || {}, params.groupBy),
+                values: seriesValues,
+                summary: this.computeSummary(rawValues),
+            });
         }
-        return {
+        // Top-level values/summary always reflect the first series (back-compat:
+        // single-aggregated queries always return one row, so this is unchanged).
+        const top = groups[0] || { values: [], summary: this.computeSummary([]) };
+        const result = {
             source: this.name,
             service: params.service,
             metric: params.metric,
             unit: this.getUnit(params.metric),
-            values,
-            summary: this.computeSummary(rawValues),
+            values: top.values,
+            summary: top.summary,
             resolvedSeries: promql,
             resolvedLabel: label,
         };
+        if (params.groupBy && groups.length > 1) {
+            result.groupBy = params.groupBy;
+            result.groups = groups;
+        }
+        else if (!params.groupBy && candidate) {
+            // Probe common breakdown labels and hint when more than one distinct
+            // value exists for this service. Helps the model ask the right
+            // follow-up instead of silently looking at an aggregated number.
+            const escaped = params.service.replace(/\\/g, "\\\\").replace(/"/g, '\\"');
+            for (const breakdownLabel of HINT_BREAKDOWN_LABELS) {
+                const distinct = await this.getDistinctLabelValues(candidate.seriesName, label, escaped, breakdownLabel);
+                if (distinct.length > 1) {
+                    result.hint = `${distinct.length} distinct ${breakdownLabel}s exist for this service. Pass groupBy="${breakdownLabel}" to break the result down.`;
+                    break;
+                }
+            }
+        }
+        return result;
     }
     // --- Private helpers ---
-    async buildQuery(service, metric) {
-        // Pick the query template. For built-in metrics with no user override,
-        // probe candidate series in the backend and pick the first that exists
-        // (e.g. prom-client process_cpu_seconds_total → falls back to
-        // node_exporter node_cpu_seconds_total). User-overridden metrics use
-        // their query verbatim.
+    async buildQuery(service, metric, groupBy) {
+        // Resolve the service-filter label first. Candidate probing uses this
+        // label to scope existence checks per-service rather than per-source.
+        const escaped = service.replace(/\\/g, "\\\\").replace(/"/g, '\\"');
+        let label = "job";
         let template;
+        let candidate = null;
         if (!this.userOverrides.has(metric) && PROMETHEUS_METRIC_CANDIDATES[metric]) {
-            const candidate = await this.pickMetricCandidate(metric);
-            template = candidate?.query || PROMETHEUS_METRIC_CANDIDATES[metric][0].query;
+            label = await this.resolveServiceLabel(service);
+            candidate = await this.pickMetricCandidate(metric, label, escaped);
+            const fallback = PROMETHEUS_METRIC_CANDIDATES[metric][0];
+            const chosen = candidate || fallback;
+            template = (groupBy && chosen.groupedQuery) ? chosen.groupedQuery : chosen.query;
         }
         else {
             const def = this.metrics.find((m) => m.name === metric);
             template = def?.query || `${metric}{ {{selector}} }`;
         }
-        const escaped = service.replace(/\\/g, "\\\\").replace(/"/g, '\\"');
         let promql = template;
-        let label = "job";
         if (template.includes("{{selector}}")) {
-            label = await this.resolveServiceLabel(service);
+            // Resolve label here for non-candidate paths that haven't done it yet.
+            if (label === "job" && !PROMETHEUS_METRIC_CANDIDATES[metric]) {
+                label = await this.resolveServiceLabel(service);
+            }
             const selector = `${label}="${escaped}"`;
             promql = promql.replace(/\{\{selector\}\}/g, selector);
         }
+        if (groupBy && template.includes("{{groupBy}}")) {
+            // groupBy is a label name (caller-supplied). Constrain to the same
+            // safe character set we use for service names so it can't break out
+            // of the by(...) clause.
+            const safe = groupBy.replace(/[^a-zA-Z0-9_]/g, "");
+            promql = promql.replace(/\{\{groupBy\}\}/g, safe);
+        }
         promql = promql.replace(/\{\{service\}\}/g, escaped);
-        return { promql, label };
+        return { promql, label, candidate };
+    }
+    groupKey(metric, groupBy) {
+        if (groupBy && metric[groupBy] !== undefined)
+            return metric[groupBy];
+        // No explicit groupBy: synthesize a key from instance/pod/node if any,
+        // else from all labels. Useful for naturally-per-series queries like
+        // process_resident_memory_bytes (no aggregator dropping labels).
+        for (const probe of HINT_BREAKDOWN_LABELS) {
+            if (metric[probe])
+                return metric[probe];
+        }
+        const entries = Object.entries(metric);
+        if (entries.length === 0)
+            return "default";
+        return entries.map(([k, v]) => `${k}=${v}`).join(",");
+    }
+    async getDistinctLabelValues(seriesName, label, escapedService, breakdownLabel) {
+        try {
+            const matchExpr = `${seriesName}{${label}="${escapedService}"}`;
+            const url = `/api/v1/label/${encodeURIComponent(breakdownLabel)}/values?match[]=${encodeURIComponent(matchExpr)}`;
+            const data = await this.apiGet(url);
+            return data?.data || [];
+        }
+        catch {
+            return [];
+        }
     }
-    async pickMetricCandidate(metric) {
+    async pickMetricCandidate(metric, label, escapedService) {
         const candidates = PROMETHEUS_METRIC_CANDIDATES[metric];
         if (!candidates || candidates.length === 0)
             return null;
         if (candidates.length === 1)
             return candidates[0];
-        const allNames = await this.getAllMetricNames();
+        // Per-service cache: a source can have BOTH process_* and node_* series
+        // present (e.g. an apps stack alongside node_exporter), so probing has
+        // to check whether THIS service has the series, not whether the source
+        // has it anywhere.
+        const cacheKey = `${metric}|${label}|${escapedService}`;
+        const cached = this.candidateCache.get(cacheKey);
+        if (cached && cached.expiresAt > Date.now())
+            return cached.candidate;
         for (const c of candidates) {
-            if (allNames.has(c.seriesName))
+            if (await this.seriesExistsForService(c.seriesName, label, escapedService)) {
+                this.candidateCache.set(cacheKey, {
+                    candidate: c,
+                    expiresAt: Date.now() + LABEL_CACHE_TTL_MS,
+                });
                 return c;
+            }
         }
-        return candidates[0];
+        // Nothing found — return first candidate as best-effort. Cache the
+        // negative outcome so we don't probe again for 60s.
+        const fallback = candidates[0];
+        this.candidateCache.set(cacheKey, {
+            candidate: fallback,
+            expiresAt: Date.now() + LABEL_CACHE_TTL_MS,
+        });
+        return fallback;
     }
-    async getAllMetricNames() {
-        if (this.metricNamesCache && this.metricNamesCache.expiresAt > Date.now()) {
-            return this.metricNamesCache.values;
-        }
+    async seriesExistsForService(seriesName, label, escapedService) {
         try {
-            const data = await this.apiGet("/api/v1/label/__name__/values");
-            const values = new Set(data?.data || []);
-            this.metricNamesCache = { values, expiresAt: Date.now() + LABEL_CACHE_TTL_MS };
-            return values;
+            const matchExpr = `${seriesName}{${label}="${escapedService}"}`;
+            const url = `/api/v1/series?match[]=${encodeURIComponent(matchExpr)}`;
+            const data = await this.apiGet(url);
+            return Array.isArray(data?.data) && data.data.length > 0;
         }
         catch {
-            this.metricNamesCache = { values: new Set(), expiresAt: Date.now() + LABEL_CACHE_TTL_MS };
-            return new Set();
+            return false;
         }
     }
     async resolveServiceLabel(service) {

package/dist/index.js

@@ -72,6 +72,7 @@ async function main() {
         metric: z.string().describe(`Metric name. Available: ${uniqueNames.join(", ")}`),
         duration: z.string().optional().describe("Time range (e.g. '5m', '1h', '24h'). Default: '5m'"),
         source: z.string().optional().describe("Specific source name. If omitted, queries all metrics backends."),
+        groupBy: z.string().optional().describe("Label to break the result down by, e.g. 'instance', 'pod', 'node'. Returns one series per distinct value in 'groups'."),
     }, async (args) => queryMetricsHandler(registry, args));
     mcpServer.tool("query_logs", "Query logs for a service over a given timeframe. Returns log entries with a summary including error/warning counts and top error patterns.", {
         service: z.string().describe("Service name (e.g. 'payment-service')"),

package/dist/tools/query-metrics.d.ts

@@ -21,6 +21,10 @@ export declare const queryMetricsDefinition: {
                 type: string;
                 description: string;
             };
+            groupBy: {
+                type: string;
+                description: string;
+            };
         };
         required: string[];
     };
@@ -30,6 +34,7 @@ export declare function queryMetricsHandler(registry: ConnectorRegistry, args: {
     metric: string;
     duration?: string;
     source?: string;
+    groupBy?: string;
 }): Promise<{
     content: {
         type: "text";

package/dist/tools/query-metrics.js

@@ -21,6 +21,10 @@ export const queryMetricsDefinition = {
                 type: "string",
                 description: "Specific source name to query. If omitted, queries all metrics backends.",
             },
+            groupBy: {
+                type: "string",
+                description: "Optional label to break the result down by, e.g. 'instance', 'pod', 'node'. When set, the response includes a 'groups' array with one time-series per distinct value. When the service has only one matching series, the result is unchanged.",
+            },
         },
         required: ["service", "metric"],
     },
@@ -36,6 +40,9 @@ export async function queryMetricsHandler(registry, args) {
     const metricErr = validateMetricName(args.metric, registry);
     if (metricErr)
         return errorResponse(metricErr);
+    if (args.groupBy && !/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(args.groupBy)) {
+        return errorResponse(`Invalid groupBy "${args.groupBy}". Must be a valid Prometheus label name (alphanumeric + underscore, starting with letter/underscore).`);
+    }
     const connectors = args.source
         ? [registry.getByName(args.source)].filter(Boolean)
         : registry.getBySignal("metrics");
@@ -55,6 +62,7 @@ export async function queryMetricsHandler(registry, args) {
                 service: args.service,
                 metric: args.metric,
                 duration,
+                groupBy: args.groupBy,
             });
             results.push(result);
         }

package/dist/types.d.ts

@@ -95,6 +95,7 @@ export interface MetricQuery {
     metric: string;
     duration: string;
     step?: string;
+    groupBy?: string;
 }
 export interface LogQuery {
     service: string;
@@ -114,6 +115,11 @@ export interface MetricSummary {
     max: number;
     trend: Trend;
 }
+export interface MetricGroup {
+    key: string;
+    values: DataPoint[];
+    summary: MetricSummary;
+}
 export interface MetricResult {
     source: string;
     service: string;
@@ -123,6 +129,9 @@ export interface MetricResult {
     summary: MetricSummary;
     resolvedSeries?: string;
     resolvedLabel?: string;
+    groupBy?: string;
+    groups?: MetricGroup[];
+    hint?: string;
 }
 export interface LogEntry {
     timestamp: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@thotischner/observability-mcp",
-  "version": "1.2.1",
+  "version": "1.3.0",
   "description": "Unified observability gateway for AI agents — one MCP server for Prometheus, Loki, and any backend",
   "type": "module",
   "license": "MIT",