@dipseth/opensearch-logs 0.1.0

package/dist/services/search.service.js

@@ -0,0 +1,885 @@
+/**
+ * Search service — 9 modes of log search and analysis.
+ *
+ * Modes: search, count, histogram, timeline, dashboard, latency, errors, services, report
+ */
+import { resolveServiceName, SERVICE_REGISTRY } from "../utils/index.js";
+// ── Report Mode Constants ────────────────────────────────────────────────
+const REPORT_PATTERNS = {
+    "HTTP 500s": '"500 Internal Server Error"',
+    "HTTP 503s": 'log:"status=503"',
+    "HTTP 502s": 'log:"status=502"',
+    "Connection Refused": '"Connection refused"',
+    "Worker Shutdowns": '"Shutting down"',
+    "Worker Startups": '"Application startup complete"',
+    "Max Request Limit": '"Maximum request limit"',
+    "Playwright Failures": '"Failed to launch browser"',
+    "OOM / Memory": 'OOM OR "Cannot allocate memory"',
+    "API Key Missing": '"ENCORE_G_API_KEY: None" OR "ENCORE_G_API_KEY: NOT SET"',
+    "Client Timeouts (499)": 'log:"status=499"',
+};
+// Painless scripts for parsing latency from correlation middleware messages.
+// Message format: "METHOD /endpoint STATUS DURATIONms"
+const PAINLESS_LATENCY = {
+    init: "state.values = []",
+    map: String.raw `def msg = doc["message.keyword"].value; if (msg != null) { def m = /\d+(?=ms)/.matcher(msg); if (m.find()) { state.values.add(Integer.parseInt(m.group())); } }`,
+    combine: "return state.values",
+    reduce: [
+        'List all = new ArrayList();',
+        'for (s in states) { if (s instanceof List) { all.addAll(s); } }',
+        'Map result = new HashMap();',
+        'if (all.isEmpty()) { result.put("count", 0); result.put("avg", 0); result.put("p50", 0); result.put("p95", 0); result.put("p99", 0); return result; }',
+        'Collections.sort(all); long sum = 0; for (def v : all) { sum += v; }',
+        'int sz = all.size();',
+        'int p50i = sz / 2; int p95i = sz * 95 / 100; if (p95i >= sz) { p95i = sz - 1; }',
+        'int p99i = sz * 99 / 100; if (p99i >= sz) { p99i = sz - 1; }',
+        'result.put("count", sz); result.put("avg", sum / sz);',
+        'result.put("p50", all.get(p50i)); result.put("p95", all.get(p95i)); result.put("p99", all.get(p99i));',
+        'return result;',
+    ].join(" "),
+};
+function autoInterval(hours) {
+    if (hours <= 2)
+        return "15m";
+    if (hours <= 6)
+        return "30m";
+    if (hours <= 12)
+        return "1h";
+    if (hours <= 24)
+        return "2h";
+    return "4h";
+}
+export class SearchService {
+    repo;
+    constructor(repo) {
+        this.repo = repo;
+    }
+    // ── Query Builders ──────────────────────────────────────────────────
+    parseStatusFilter(status) {
+        if (!status)
+            return null;
+        const s = status.trim().toLowerCase();
+        if (s === "5xx")
+            return { range: { status_code: { gte: 500, lt: 600 } } };
+        if (s === "4xx")
+            return { range: { status_code: { gte: 400, lt: 500 } } };
+        if (s === "3xx")
+            return { range: { status_code: { gte: 300, lt: 400 } } };
+        if (s === "2xx")
+            return { range: { status_code: { gte: 200, lt: 300 } } };
+        if (s.startsWith(">="))
+            return { range: { status_code: { gte: Number(s.slice(2)) } } };
+        if (s.startsWith(">"))
+            return { range: { status_code: { gt: Number(s.slice(1)) } } };
+        if (s.startsWith("<="))
+            return { range: { status_code: { lte: Number(s.slice(2)) } } };
+        if (s.startsWith("<"))
+            return { range: { status_code: { lt: Number(s.slice(1)) } } };
+        return { term: { status_code: Number(s) } };
+    }
+    serviceFilter(svcName) {
+        return {
+            bool: {
+                should: [
+                    { term: { "service.keyword": svcName } },
+                    { match_phrase: { log: svcName } },
+                ],
+                minimum_should_match: 1,
+            },
+        };
+    }
+    /** Hybrid 5xx filter: works whether status_code field exists or not. */
+    hybridErrorFilter() {
+        return {
+            bool: {
+                should: [
+                    { range: { status_code: { gte: 500 } } },
+                    { term: { "level.keyword": "ERROR" } },
+                ],
+                minimum_should_match: 1,
+            },
+        };
+    }
+    /** Hybrid 4xx filter: works whether status_code field exists or not. */
+    hybridClientErrorFilter() {
+        return {
+            bool: {
+                should: [
+                    { range: { status_code: { gte: 400, lt: 500 } } },
+                    { term: { "level.keyword": "WARNING" } },
+                ],
+                minimum_should_match: 1,
+            },
+        };
+    }
+    /** Scripted metric aggregation that parses latency from correlation middleware messages. */
+    latencyScriptedMetric() {
+        return {
+            scripted_metric: {
+                init_script: PAINLESS_LATENCY.init,
+                map_script: PAINLESS_LATENCY.map,
+                combine_script: PAINLESS_LATENCY.combine,
+                reduce_script: PAINLESS_LATENCY.reduce,
+            },
+        };
+    }
+    buildQuery(args, fromDt, toDt) {
+        const must = [];
+        const filters = [
+            { range: { "@timestamp": { gte: fromDt.toISOString(), lte: toDt.toISOString() } } },
+        ];
+        if (args.query)
+            must.push({ query_string: { query: args.query } });
+        if (args.service) {
+            must.push(this.serviceFilter(resolveServiceName(args.service)));
+        }
+        if (args.level) {
+            const upper = args.level.toUpperCase();
+            must.push({
+                bool: {
+                    should: [{ term: { "level.keyword": upper } }, { match_phrase: { log: upper } }],
+                    minimum_should_match: 1,
+                },
+            });
+        }
+        const statusFilter = this.parseStatusFilter(args.status);
+        if (statusFilter)
+            filters.push(statusFilter);
+        if (args.correlationId) {
+            must.push({
+                bool: {
+                    should: [
+                        { term: { "correlation_id.keyword": args.correlationId } },
+                        { match_phrase: { log: args.correlationId } },
+                    ],
+                    minimum_should_match: 1,
+                },
+            });
+        }
+        return { bool: { must: must.length ? must : [{ match_all: {} }], filter: filters } };
+    }
+    // ── Mode: search ────────────────────────────────────────────────────
+    async modeSearch(index, query, args) {
+        const result = await this.repo.search(index, {
+            size: args.limit,
+            sort: [{ "@timestamp": args.asc ? "asc" : "desc" }],
+            query,
+        });
+        const { total } = result.hits;
+        console.log(`Total: ${total.value}${total.relation === "gte" ? "+" : ""} hits\n`);
+        for (const hit of result.hits.hits) {
+            const src = hit._source;
+            const ts = src["@timestamp"] ?? "?";
+            const log = String(src.log ?? "");
+            const svc = String(src.service ?? "");
+            const level = String(src.level ?? "");
+            const statusCode = src.status_code ?? "";
+            const duration = src.duration_ms ?? "";
+            if (args.json) {
+                console.log(JSON.stringify({
+                    id: hit._id, timestamp: ts, service: svc, level,
+                    status_code: statusCode, duration_ms: duration, log,
+                }));
+            }
+            else {
+                const parts = [];
+                if (svc)
+                    parts.push(`[${svc}]`);
+                if (level)
+                    parts.push(String(level));
+                if (statusCode)
+                    parts.push(`HTTP ${statusCode}`);
+                if (duration)
+                    parts.push(`${duration}ms`);
+                const prefix = parts.join(" ");
+                const display = args.full ? log : log.slice(0, 300);
+                if (prefix) {
+                    console.log(`[${ts}] ${prefix}`);
+                    console.log(`  ${display}`);
+                }
+                else {
+                    console.log(`[${ts}] ${display}`);
+                }
+                if (args.ids)
+                    console.log(`  _id: ${hit._id}`);
+                if (src.correlation_id && args.ids)
+                    console.log(`  correlation_id: ${src.correlation_id}`);
+                console.log();
+            }
+        }
+    }
+    // ── Mode: count ─────────────────────────────────────────────────────
+    async modeCount(index, query) {
+        const n = await this.repo.count(index, { query });
+        console.log(`Count: ${n}`);
+    }
+    // ── Mode: histogram ─────────────────────────────────────────────────
+    async modeHistogram(index, query, args) {
+        const result = await this.repo.search(index, {
+            size: 0, query,
+            aggs: {
+                over_time: {
+                    date_histogram: { field: "@timestamp", fixed_interval: args.interval ?? "1h" },
+                },
+            },
+        });
+        const total = result.hits.total.value;
+        const buckets = result.aggregations.over_time.buckets;
+        console.log(`Total: ${total} hits\n`);
+        const maxCount = Math.max(...buckets.map(b => b.doc_count), 1);
+        for (const b of buckets) {
+            const barLen = Math.round((b.doc_count / maxCount) * 40);
+            console.log(`  ${b.key_as_string}  ${String(b.doc_count).padStart(6)}  ${"█".repeat(barLen)}`);
+        }
+    }
+    // ── Mode: timeline ──────────────────────────────────────────────────
+    async modeTimeline(index, query, args) {
+        const result = await this.repo.search(index, {
+            size: args.limit,
+            sort: [{ "@timestamp": "asc" }],
+            query,
+            _source: ["@timestamp", "log", "service", "level", "status_code", "duration_ms", "correlation_id", "endpoint"],
+        });
+        const total = result.hits.total.value;
+        console.log(`Timeline (${total} total, showing ${result.hits.hits.length})\n`);
+        for (const hit of result.hits.hits) {
+            const src = hit._source;
+            const ts = src["@timestamp"] ?? "?";
+            const log = String(src.log ?? "");
+            const svc = String(src.service ?? "");
+            const level = String(src.level ?? "");
+            const display = args.full ? log : log.slice(0, 250);
+            let tag = "";
+            if (svc)
+                tag += `[${svc}] `;
+            if (level)
+                tag += `${level} `;
+            console.log(`  ${ts}  ${tag}${display}`);
+        }
+    }
+    // ── Mode: dashboard ─────────────────────────────────────────────────
+    async modeDashboard(index, fromDt, toDt) {
+        const timeFilter = { range: { "@timestamp": { gte: fromDt.toISOString(), lte: toDt.toISOString() } } };
+        const aggs = {};
+        // Global health checks
+        const textChecks = {
+            "HTTP 500s (text)": '"500 Internal Server Error"',
+            "HTTP 503s (text)": 'log:"status=503"',
+            "Connection Refused": '"Connection refused"',
+            "Worker Shutdowns": '"Shutting down"',
+            "Worker Startups": '"Application startup complete"',
+            "Max Request Limit": '"Maximum request limit"',
+            "Playwright Failures": '"Failed to launch browser"',
+            "OOM / Memory": 'OOM OR "Cannot allocate memory"',
+            "API Key None": '"ENCORE_G_API_KEY: None"',
+        };
+        for (const [name, q] of Object.entries(textChecks)) {
+            const safe = name.replace(/[^a-zA-Z0-9]/g, "_");
+            aggs[safe] = {
+                filter: { bool: { must: [{ query_string: { query: q } }], filter: [timeFilter] } },
+            };
+        }
+        // Per-service errors (5xx or ERROR level)
+        for (const [alias, info] of SERVICE_REGISTRY) {
+            const safe = `svc_5xx_${alias}`;
+            aggs[safe] = {
+                filter: {
+                    bool: {
+                        must: [
+                            this.serviceFilter(info.logName),
+                            {
+                                bool: {
+                                    should: [
+                                        { range: { status_code: { gte: 500 } } },
+                                        { term: { "level.keyword": "ERROR" } },
+                                        { query_string: { query: '"500 Internal Server Error" OR log:"status=503" OR log:"status=502"' } },
+                                    ],
+                                    minimum_should_match: 1,
+                                },
+                            },
+                        ],
+                        filter: [timeFilter],
+                    },
+                },
+            };
+        }
+        // Latency from correlation middleware messages
+        aggs.latency_stats = {
+            filter: { bool: { filter: [timeFilter], must: [{ term: { "name.keyword": "common.middleware.correlation" } }] } },
+            aggs: { parsed: this.latencyScriptedMetric() },
+        };
+        // Total requests
+        aggs.total_requests = { filter: { bool: { filter: [timeFilter] } } };
+        const result = await this.repo.search(index, { size: 0, aggs });
+        const aggData = result.aggregations;
+        const window = `${fmtTime(fromDt)} - ${fmtTime(toDt)} UTC`;
+        console.log(`${"=".repeat(60)}`);
+        console.log(`  Production Health Dashboard (${window})`);
+        console.log(`${"=".repeat(60)}\n`);
+        console.log(`  Total log entries: ${aggData.total_requests.doc_count.toLocaleString()}\n`);
+        console.log("  ── Global Health Checks ──\n");
+        for (const name of Object.keys(textChecks)) {
+            const safe = name.replace(/[^a-zA-Z0-9]/g, "_");
+            const count = aggData[safe]?.doc_count ?? 0;
+            console.log(`  [${count === 0 ? "✓" : "⚠"}] ${name.padEnd(25)} ${String(count).padStart(6)}`);
+        }
+        console.log(`\n  ── Per-Service 5xx Errors ──\n`);
+        const svcErrors = [];
+        for (const [alias, info] of SERVICE_REGISTRY) {
+            const safe = `svc_5xx_${alias}`;
+            svcErrors.push([alias, aggData[safe]?.doc_count ?? 0, info.description]);
+        }
+        svcErrors.sort((a, b) => b[1] - a[1]);
+        for (const [alias, count, desc] of svcErrors) {
+            console.log(`  [${count === 0 ? "✓" : "⚠"}] ${alias.padEnd(25)} ${String(count).padStart(6)}  (${desc})`);
+        }
+        console.log(`\n  ── Latency (from correlation middleware) ──\n`);
+        const lat = aggData.latency_stats;
+        if (lat.doc_count > 0) {
+            const parsed = lat.parsed.value;
+            console.log(`  Requests tracked: ${parsed.count.toLocaleString()}`);
+            console.log(`  Average:  ${String(Math.round(parsed.avg)).padStart(8)} ms`);
+            console.log(`  P50:      ${String(Math.round(parsed.p50)).padStart(8)} ms`);
+            console.log(`  P95:      ${String(Math.round(parsed.p95)).padStart(8)} ms`);
+            console.log(`  P99:      ${String(Math.round(parsed.p99)).padStart(8)} ms`);
+        }
+        else {
+            console.log("  No correlation middleware logs in this window.");
+        }
+        console.log(`\n  Index: ${index}`);
+        console.log(`  Dashboard: ${this.buildDashboardLink(undefined, fromDt, toDt)}`);
+        console.log();
+    }
+    // ── Mode: latency ───────────────────────────────────────────────────
+    async modeLatency(index, fromDt, toDt, args) {
+        const timeFilter = { range: { "@timestamp": { gte: fromDt.toISOString(), lte: toDt.toISOString() } } };
+        const must = [{ term: { "name.keyword": "common.middleware.correlation" } }];
+        if (args.service) {
+            must.push(this.serviceFilter(resolveServiceName(args.service)));
+        }
+        const result = await this.repo.search(index, {
+            size: 0,
+            query: { bool: { must, filter: [timeFilter] } },
+            aggs: {
+                overall: this.latencyScriptedMetric(),
+                by_service: {
+                    terms: { field: "service.keyword", size: 20 },
+                    aggs: { parsed: this.latencyScriptedMetric() },
+                },
+                over_time: {
+                    date_histogram: {
+                        field: "@timestamp", fixed_interval: args.interval ?? "1h", min_doc_count: 0,
+                        extended_bounds: { min: fromDt.getTime(), max: toDt.getTime() },
+                    },
+                    aggs: { parsed: this.latencyScriptedMetric() },
+                },
+            },
+        });
+        const total = result.hits.total.value;
+        const agg = result.aggregations;
+        const window = `${fmtTime(fromDt)} - ${fmtTime(toDt)} UTC`;
+        console.log(`${"=".repeat(60)}`);
+        console.log(`  Latency Analysis (${window})`);
+        console.log(`${"=".repeat(60)}\n`);
+        console.log(`  Correlation middleware requests: ${total.toLocaleString()}\n`);
+        const overall = agg.overall.value;
+        if (overall.count > 0) {
+            console.log("  ── Overall ──\n");
+            console.log(`  Average:  ${String(Math.round(overall.avg)).padStart(8)} ms`);
+            console.log(`  P50:      ${String(Math.round(overall.p50)).padStart(8)} ms`);
+            console.log(`  P95:      ${String(Math.round(overall.p95)).padStart(8)} ms`);
+            console.log(`  P99:      ${String(Math.round(overall.p99)).padStart(8)} ms`);
+        }
+        else {
+            console.log("  No correlation middleware logs with parseable duration.\n");
+        }
+        if (agg.by_service.buckets.length) {
+            console.log(`\n  ── Per-Service ──\n`);
+            console.log(`  ${"Service".padEnd(30)} ${"Count".padStart(7)} ${"Avg".padStart(8)} ${"P95".padStart(8)} ${"P99".padStart(8)}`);
+            console.log(`  ${"-".repeat(30)} ${"-".repeat(7)} ${"-".repeat(8)} ${"-".repeat(8)} ${"-".repeat(8)}`);
+            const sorted = [...agg.by_service.buckets].sort((a, b) => b.parsed.value.p95 - a.parsed.value.p95);
+            for (const b of sorted) {
+                const p = b.parsed.value;
+                if (p.count === 0)
+                    continue;
+                console.log(`  ${b.key.padEnd(30)} ${String(p.count).padStart(7)} ${String(Math.round(p.avg)).padStart(7)}ms ${String(Math.round(p.p95)).padStart(7)}ms ${String(Math.round(p.p99)).padStart(7)}ms`);
+            }
+        }
+        const nonEmptyBuckets = agg.over_time.buckets.filter(b => b.parsed.value.count > 0);
+        if (nonEmptyBuckets.length) {
+            console.log(`\n  ── P95 Latency Over Time ──\n`);
+            const maxP95 = Math.max(...nonEmptyBuckets.map(b => b.parsed.value.p95), 1);
+            for (const b of agg.over_time.buckets) {
+                const p95 = b.parsed.value.count > 0 ? b.parsed.value.p95 : 0;
+                const barLen = Math.round((p95 / maxP95) * 30);
+                console.log(`  ${b.key_as_string}  ${String(Math.round(p95)).padStart(7)}ms  ${"█".repeat(barLen)}`);
+            }
+        }
+        console.log();
+    }
+    // ── Mode: errors ────────────────────────────────────────────────────
+    async modeErrors(index, fromDt, toDt, args) {
+        const timeFilter = { range: { "@timestamp": { gte: fromDt.toISOString(), lte: toDt.toISOString() } } };
+        const must = [{
+                bool: {
+                    should: [
+                        { range: { status_code: { gte: 400 } } },
+                        { term: { "level.keyword": "ERROR" } },
+                        { query_string: { query: '"ERROR" OR "500 Internal Server Error" OR "status=503" OR "status=502"' } },
+                    ],
+                    minimum_should_match: 1,
+                },
+            }];
+        if (args.service) {
+            must.push(this.serviceFilter(resolveServiceName(args.service)));
+        }
+        const result = await this.repo.search(index, {
+            size: 0,
+            query: { bool: { must, filter: [timeFilter] } },
+            aggs: {
+                by_service: {
+                    terms: { field: "service.keyword", size: 20 },
+                    aggs: { by_status: { terms: { field: "status_code", size: 10 } } },
+                },
+                by_status_code: { terms: { field: "status_code", size: 20 } },
+                by_endpoint: { terms: { field: "endpoint.keyword", size: 15, order: { _count: "desc" } } },
+                by_error_type: { terms: { field: "error_type.keyword", size: 10 } },
+                over_time: {
+                    date_histogram: { field: "@timestamp", fixed_interval: args.interval ?? "1h" },
+                },
+                recent_errors: {
+                    top_hits: {
+                        size: 5, sort: [{ "@timestamp": "desc" }],
+                        _source: ["@timestamp", "log", "service", "level", "status_code", "endpoint", "error_type", "correlation_id"],
+                    },
+                },
+            },
+        });
+        const total = result.hits.total.value;
+        const agg = result.aggregations;
+        const window = `${fmtTime(fromDt)} - ${fmtTime(toDt)} UTC`;
+        console.log(`${"=".repeat(60)}`);
+        console.log(`  Error Analysis (${window})`);
+        console.log(`${"=".repeat(60)}\n`);
+        console.log(`  Total errors/4xx+: ${total.toLocaleString()}\n`);
+        if (agg.by_status_code.buckets.length) {
+            console.log("  ── By Status Code ──\n");
+            for (const b of agg.by_status_code.buckets)
+                console.log(`  HTTP ${String(b.key).padEnd(6)} ${String(b.doc_count).padStart(6)}`);
+        }
+        if (agg.by_error_type.buckets.length) {
+            console.log(`\n  ── By Error Type ──\n`);
+            for (const b of agg.by_error_type.buckets)
+                console.log(`  ${String(b.key).padEnd(20)} ${String(b.doc_count).padStart(6)}`);
+        }
+        if (agg.by_service.buckets.length) {
+            console.log(`\n  ── By Service ──\n`);
+            const sorted = [...agg.by_service.buckets].sort((a, b) => b.doc_count - a.doc_count);
+            for (const b of sorted) {
+                const detail = b.by_status.buckets.map(s => `${s.key}:${s.doc_count}`).join(", ");
+                console.log(`  ${String(b.key).padEnd(30)} ${String(b.doc_count).padStart(6)}  (${detail})`);
+            }
+        }
+        if (agg.by_endpoint.buckets.length) {
+            console.log(`\n  ── Top Error Endpoints ──\n`);
+            for (const b of agg.by_endpoint.buckets)
+                console.log(`  ${String(b.key).padEnd(50)} ${String(b.doc_count).padStart(6)}`);
+        }
+        if (agg.over_time.buckets.length) {
+            console.log(`\n  ── Errors Over Time ──\n`);
+            const maxCount = Math.max(...agg.over_time.buckets.map(b => b.doc_count), 1);
+            for (const b of agg.over_time.buckets) {
+                const barLen = Math.round((b.doc_count / maxCount) * 30);
+                console.log(`  ${b.key_as_string}  ${String(b.doc_count).padStart(6)}  ${"█".repeat(barLen)}`);
+            }
+        }
+        if (agg.recent_errors.hits.hits.length) {
+            console.log(`\n  ── Recent Error Samples ──\n`);
+            for (const hit of agg.recent_errors.hits.hits) {
+                const src = hit._source;
+                console.log(`  [${src["@timestamp"]}] [${src.service ?? "?"}] ${src.level ?? "?"} HTTP ${src.status_code ?? "?"} ${src.endpoint ?? "?"}`);
+                if (src.correlation_id)
+                    console.log(`    correlation_id: ${src.correlation_id}`);
+                console.log(`    ${String(src.log ?? "").slice(0, 200)}`);
+                console.log();
+            }
+        }
+        console.log();
+    }
+    // ── Mode: services ──────────────────────────────────────────────────
+    async modeServices(index, fromDt, toDt) {
+        const timeFilter = { range: { "@timestamp": { gte: fromDt.toISOString(), lte: toDt.toISOString() } } };
+        const result = await this.repo.search(index, {
+            size: 0,
+            query: { bool: { filter: [timeFilter], must: [{ exists: { field: "service" } }] } },
+            aggs: {
+                by_service: {
+                    terms: { field: "service.keyword", size: 30 },
+                    aggs: {
+                        errors: { filter: this.hybridErrorFilter() },
+                        client_errors: { filter: this.hybridClientErrorFilter() },
+                        latency: {
+                            filter: { term: { "name.keyword": "common.middleware.correlation" } },
+                            aggs: { parsed: this.latencyScriptedMetric() },
+                        },
+                    },
+                },
+            },
+        });
+        const window = `${fmtTime(fromDt)} - ${fmtTime(toDt)} UTC`;
+        console.log(`${"=".repeat(60)}`);
+        console.log(`  Service Health Summary (${window})`);
+        console.log(`${"=".repeat(60)}\n`);
+        const buckets = result.aggregations.by_service.buckets;
+        if (!buckets.length) {
+            console.log("  No service-level data found.");
+            return;
+        }
+        console.log(`  ${"Service".padEnd(30)} ${"Total".padStart(7)} ${"5xx".padStart(5)} ${"4xx".padStart(5)} ${"Avg ms".padStart(8)} ${"P95 ms".padStart(8)} ${"Err%".padStart(6)}`);
+        console.log(`  ${"-".repeat(30)} ${"-".repeat(7)} ${"-".repeat(5)} ${"-".repeat(5)} ${"-".repeat(8)} ${"-".repeat(8)} ${"-".repeat(6)}`);
+        const sorted = [...buckets].sort((a, b) => b.errors.doc_count - a.errors.doc_count);
+        for (const b of sorted) {
+            const errs = b.errors.doc_count;
+            const clientErrs = b.client_errors.doc_count;
+            const parsedLat = b.latency.doc_count > 0 ? b.latency.parsed.value : null;
+            const avgMs = parsedLat?.avg ?? 0;
+            const p95Ms = parsedLat?.p95 ?? 0;
+            const errPct = b.doc_count > 0 ? (errs / b.doc_count * 100) : 0;
+            const marker = errs > 0 ? "⚠" : " ";
+            console.log(` ${marker}${String(b.key).padEnd(30)} ${String(b.doc_count).padStart(7)} ${String(errs).padStart(5)} ${String(clientErrs).padStart(5)} ${String(Math.round(avgMs)).padStart(7)}ms ${String(Math.round(p95Ms)).padStart(7)}ms ${errPct.toFixed(1).padStart(5)}%`);
+        }
+        console.log();
+    }
+    // ── Mode: report ───────────────────────────────────────────────────
+    async modeReport(index, fromDt, toDt, args) {
+        const timeFilter = { range: { "@timestamp": { gte: fromDt.toISOString(), lte: toDt.toISOString() } } };
+        const hours = (toDt.getTime() - fromDt.getTime()) / 3_600_000;
+        const interval = args.interval ?? autoInterval(hours);
+        const svcFilterClause = args.service
+            ? this.serviceFilter(resolveServiceName(args.service))
+            : null;
+        // ── Phase 1: Discovery ──────────────────────────────────────────
+        const p1Aggs = {};
+        // Per-service breakdown with sub-aggs
+        const svcMust = [{ exists: { field: "service" } }];
+        if (svcFilterClause)
+            svcMust.push(svcFilterClause);
+        p1Aggs.by_service = {
+            filter: { bool: { must: svcMust, filter: [timeFilter] } },
+            aggs: {
+                services: {
+                    terms: { field: "service.keyword", size: 30 },
+                    aggs: {
+                        errors_5xx: { filter: this.hybridErrorFilter() },
+                        errors_4xx: { filter: this.hybridClientErrorFilter() },
+                        latency: {
+                            filter: { term: { "name.keyword": "common.middleware.correlation" } },
+                            aggs: { parsed: this.latencyScriptedMetric() },
+                        },
+                    },
+                },
+            },
+        };
+        // Incident pattern counts
+        for (const [name, q] of Object.entries(REPORT_PATTERNS)) {
+            const safe = `pat_${name.replace(/[^a-zA-Z0-9]/g, "_")}`;
+            const patMust = [{ query_string: { query: q } }];
+            if (svcFilterClause)
+                patMust.push(svcFilterClause);
+            p1Aggs[safe] = {
+                filter: { bool: { must: patMust, filter: [timeFilter] } },
+            };
+        }
+        // Error histogram
+        const errHistMust = [{
+                bool: {
+                    should: [
+                        { range: { status_code: { gte: 400 } } },
+                        { term: { "level.keyword": "ERROR" } },
+                    ],
+                    minimum_should_match: 1,
+                },
+            }];
+        if (svcFilterClause)
+            errHistMust.push(svcFilterClause);
+        p1Aggs.error_total = {
+            filter: { bool: { must: errHistMust, filter: [timeFilter] } },
+            aggs: {
+                over_time: { date_histogram: {
+                        field: "@timestamp", fixed_interval: interval, min_doc_count: 0,
+                        extended_bounds: { min: fromDt.getTime(), max: toDt.getTime() },
+                    } },
+            },
+        };
+        // Overall latency from correlation middleware messages
+        const latMust = [{ term: { "name.keyword": "common.middleware.correlation" } }];
+        if (svcFilterClause)
+            latMust.push(svcFilterClause);
+        p1Aggs.latency_overview = {
+            filter: { bool: { must: latMust, filter: [timeFilter] } },
+            aggs: { parsed: this.latencyScriptedMetric() },
+        };
+        // Total log entries
+        const totalMust = svcFilterClause ? [svcFilterClause] : [];
+        p1Aggs.total = {
+            filter: { bool: { must: totalMust.length ? totalMust : [{ match_all: {} }], filter: [timeFilter] } },
+        };
+        const p1Result = await this.repo.search(index, { size: 0, aggs: p1Aggs });
+        const p1 = p1Result.aggregations;
+        // Extract Phase 1 data
+        const totalCount = p1.total.doc_count;
+        const svcBuckets = p1.by_service.services.buckets;
+        const totalErrors = p1.error_total.doc_count;
+        const errorTimeBuckets = p1.error_total.over_time.buckets;
+        const latOverview = p1.latency_overview;
+        const hasLatency = latOverview.doc_count > 0;
+        // Collect active patterns
+        const activePatterns = [];
+        const clearPatterns = [];
+        for (const [name] of Object.entries(REPORT_PATTERNS)) {
+            const safe = `pat_${name.replace(/[^a-zA-Z0-9]/g, "_")}`;
+            const count = p1[safe].doc_count;
+            if (count > 0) {
+                const rate = (count / hours).toFixed(1);
+                activePatterns.push({ name, count, rate: `${rate}/hr` });
+            }
+            else {
+                clearPatterns.push(name);
+            }
+        }
+        // ── Phase 2: Detail (if errors or latency exist) ────────────────
+        let statusBuckets = [];
+        let errorTypeBuckets = [];
+        let errorEpBuckets = [];
+        let recentErrors = [];
+        let latP95TimeBuckets = [];
+        if (totalErrors > 0 || hasLatency) {
+            const p2Aggs = {};
+            if (totalErrors > 0) {
+                const errMust = [{
+                        bool: {
+                            should: [
+                                { range: { status_code: { gte: 400 } } },
+                                { term: { "level.keyword": "ERROR" } },
+                            ],
+                            minimum_should_match: 1,
+                        },
+                    }];
+                if (svcFilterClause)
+                    errMust.push(svcFilterClause);
+                p2Aggs.error_detail = {
+                    filter: { bool: { must: errMust, filter: [timeFilter] } },
+                    aggs: {
+                        by_status_code: { terms: { field: "status_code", size: 20 } },
+                        by_error_type: { terms: { field: "error_type.keyword", size: 10 } },
+                        by_endpoint: { terms: { field: "endpoint.keyword", size: 10, order: { _count: "desc" } } },
+                        recent: {
+                            top_hits: {
+                                size: 5, sort: [{ "@timestamp": "desc" }],
+                                _source: ["@timestamp", "log", "service", "level", "status_code", "endpoint", "correlation_id"],
+                            },
+                        },
+                    },
+                };
+            }
+            if (hasLatency) {
+                const latDetailMust = [{ term: { "name.keyword": "common.middleware.correlation" } }];
+                if (svcFilterClause)
+                    latDetailMust.push(svcFilterClause);
+                p2Aggs.latency_detail = {
+                    filter: { bool: { must: latDetailMust, filter: [timeFilter] } },
+                    aggs: {
+                        p95_over_time: {
+                            date_histogram: {
+                                field: "@timestamp", fixed_interval: interval, min_doc_count: 0,
+                                extended_bounds: { min: fromDt.getTime(), max: toDt.getTime() },
+                            },
+                            aggs: { p95_parsed: this.latencyScriptedMetric() },
+                        },
+                    },
+                };
+            }
+            const p2Result = await this.repo.search(index, { size: 0, aggs: p2Aggs });
+            const p2 = p2Result.aggregations;
+            if (totalErrors > 0 && p2.error_detail) {
+                statusBuckets = p2.error_detail.by_status_code.buckets;
+                errorTypeBuckets = p2.error_detail.by_error_type.buckets;
+                errorEpBuckets = p2.error_detail.by_endpoint.buckets;
+                recentErrors = p2.error_detail.recent.hits.hits;
+            }
+            if (hasLatency && p2.latency_detail) {
+                latP95TimeBuckets = p2.latency_detail.p95_over_time.buckets;
+            }
+        }
+        // ── Phase 3: Pattern samples (if active patterns exist) ─────────
+        const patternSamples = {};
+        if (activePatterns.length > 0) {
+            const p3Aggs = {};
+            for (const pat of activePatterns) {
+                const safe = `sample_${pat.name.replace(/[^a-zA-Z0-9]/g, "_")}`;
+                const q = REPORT_PATTERNS[pat.name];
+                const patMust = [{ query_string: { query: q } }];
+                if (svcFilterClause)
+                    patMust.push(svcFilterClause);
+                p3Aggs[safe] = {
+                    filter: { bool: { must: patMust, filter: [timeFilter] } },
+                    aggs: {
+                        samples: {
+                            top_hits: {
+                                size: 3, sort: [{ "@timestamp": "desc" }],
+                                _source: ["@timestamp", "log", "service", "level", "status_code"],
+                            },
+                        },
+                    },
+                };
+            }
+            const p3Result = await this.repo.search(index, { size: 0, aggs: p3Aggs });
+            const p3 = p3Result.aggregations;
+            for (const pat of activePatterns) {
+                const safe = `sample_${pat.name.replace(/[^a-zA-Z0-9]/g, "_")}`;
+                if (p3[safe]) {
+                    patternSamples[pat.name] = p3[safe].samples.hits.hits;
+                }
+            }
+        }
+        // ── Render ───────────────────────────────────────────────────────
+        const window = `${fromDt.toISOString().slice(0, 16)}Z → ${toDt.toISOString().slice(0, 16)}Z`;
+        const svcLabel = args.service ? ` — ${resolveServiceName(args.service)}` : "";
+        const activeSvcCount = svcBuckets.length;
+        const totalSvcCount = SERVICE_REGISTRY.size;
+        console.log(`${"=".repeat(60)}`);
+        console.log(`  PRODUCTION REPORT${svcLabel} — ${Math.round(hours)}h window`);
+        console.log(`  ${window}`);
+        console.log(`${"=".repeat(60)}\n`);
+        console.log(`  Total: ${totalCount.toLocaleString()} log entries | ${activeSvcCount} of ${totalSvcCount} services active\n`);
+        // Active Services table
+        console.log("  ── Active Services ──\n");
+        if (svcBuckets.length === 0) {
+            console.log("  No service-level data found.\n");
+        }
+        else {
+            console.log(`  ${"Service".padEnd(30)} ${"Total".padStart(7)} ${"5xx".padStart(5)} ${"4xx".padStart(5)} ${"Avg ms".padStart(8)} ${"P95 ms".padStart(8)} ${"Err%".padStart(6)}`);
+            console.log(`  ${"-".repeat(30)} ${"-".repeat(7)} ${"-".repeat(5)} ${"-".repeat(5)} ${"-".repeat(8)} ${"-".repeat(8)} ${"-".repeat(6)}`);
+            const sorted = [...svcBuckets].sort((a, b) => b.errors_5xx.doc_count - a.errors_5xx.doc_count);
+            for (const b of sorted) {
+                const errs5 = b.errors_5xx.doc_count;
+                const errs4 = b.errors_4xx.doc_count;
+                const parsedLat = b.latency.doc_count > 0 ? b.latency.parsed.value : null;
+                const avgMs = parsedLat?.avg ?? 0;
+                const p95Ms = parsedLat?.p95 ?? 0;
+                const errPct = b.doc_count > 0 ? (errs5 / b.doc_count * 100) : 0;
+                const marker = errs5 > 0 ? "⚠" : " ";
+                console.log(` ${marker}${String(b.key).padEnd(30)} ${String(b.doc_count).padStart(7)} ${String(errs5).padStart(5)} ${String(errs4).padStart(5)} ${String(Math.round(avgMs)).padStart(7)}ms ${String(Math.round(p95Ms)).padStart(7)}ms ${errPct.toFixed(1).padStart(5)}%`);
+            }
+        }
+        // Incident Patterns
+        console.log(`\n  ── Incident Patterns ──\n`);
+        if (activePatterns.length > 0) {
+            console.log("  ACTIVE:");
+            for (const pat of activePatterns.sort((a, b) => b.count - a.count)) {
+                console.log(`    ⚠ ${pat.name.padEnd(28)} ${String(pat.count).padStart(6)}  (${pat.rate})`);
+            }
+        }
+        if (clearPatterns.length > 0) {
+            console.log(`  CLEAR: ${clearPatterns.join(", ")}`);
+        }
+        // Error Analysis
+        if (totalErrors > 0) {
+            const errRate = (totalErrors / hours).toFixed(1);
+            const errPct = totalCount > 0 ? (totalErrors / totalCount * 100).toFixed(1) : "0.0";
+            console.log(`\n  ── Error Analysis (${totalErrors.toLocaleString()} errors, ${errPct}%, ${errRate}/hr) ──\n`);
+            if (statusBuckets.length) {
+                console.log("  By Status Code:");
+                for (const b of statusBuckets)
+                    console.log(`    HTTP ${String(b.key).padEnd(6)} ${String(b.doc_count).padStart(6)}`);
+            }
+            if (errorTypeBuckets.length) {
+                console.log("  By Error Type:");
+                for (const b of errorTypeBuckets)
+                    console.log(`    ${String(b.key).padEnd(25)} ${String(b.doc_count).padStart(6)}`);
+            }
+            if (errorEpBuckets.length) {
+                console.log("  Top Error Endpoints:");
+                for (const b of errorEpBuckets)
+                    console.log(`    ${String(b.key).padEnd(45)} ${String(b.doc_count).padStart(6)}`);
+            }
+            // Error Trend histogram
+            if (errorTimeBuckets.length) {
+                console.log(`\n  ── Error Trend (${interval} buckets) ──\n`);
+                const maxCount = Math.max(...errorTimeBuckets.map(b => b.doc_count), 1);
+                for (const b of errorTimeBuckets) {
+                    const barLen = Math.round((b.doc_count / maxCount) * 30);
+                    console.log(`  ${b.key_as_string}  ${String(b.doc_count).padStart(6)}  ${"█".repeat(barLen)}`);
+                }
+            }
+        }
+        // Latency Overview
+        if (hasLatency) {
+            const parsed = latOverview.parsed.value;
+            console.log(`\n  ── Latency Overview (from correlation middleware) ──\n`);
+            console.log(`  Overall: Avg ${Math.round(parsed.avg).toLocaleString()}ms  P50 ${Math.round(parsed.p50).toLocaleString()}ms  P95 ${Math.round(parsed.p95).toLocaleString()}ms  P99 ${Math.round(parsed.p99).toLocaleString()}ms`);
+            console.log(`  Tracked requests: ${parsed.count.toLocaleString()}`);
+            if (latP95TimeBuckets.length) {
+                const nonEmpty = latP95TimeBuckets.filter(b => b.p95_parsed.value.count > 0);
+                if (nonEmpty.length > 0) {
+                    console.log(`\n  P95 Over Time (${interval} buckets):`);
+                    const maxP95 = Math.max(...nonEmpty.map(b => b.p95_parsed.value.p95), 1);
+                    for (const b of latP95TimeBuckets) {
+                        const p95 = b.p95_parsed.value.count > 0 ? b.p95_parsed.value.p95 : 0;
+                        const barLen = Math.round((p95 / maxP95) * 30);
+                        console.log(`  ${b.key_as_string}  ${String(Math.round(p95)).padStart(7)}ms  ${"█".repeat(barLen)}`);
+                    }
+                }
+            }
+        }
+        // Recent Error Samples
+        if (recentErrors.length) {
+            console.log(`\n  ── Recent Error Samples (${recentErrors.length}) ──\n`);
+            for (const hit of recentErrors) {
+                const src = hit._source;
+                console.log(`  [${src["@timestamp"]}] [${src.service ?? "?"}] ${src.level ?? "?"} HTTP ${src.status_code ?? "?"} ${src.endpoint ?? "?"}`);
+                if (src.correlation_id)
+                    console.log(`    correlation_id: ${src.correlation_id}`);
+                console.log(`    ${String(src.log ?? "").slice(0, 200)}`);
+                console.log();
+            }
+        }
+        // Pattern Samples
+        if (Object.keys(patternSamples).length > 0) {
+            console.log("  ── Pattern Samples ──\n");
+            for (const [name, hits] of Object.entries(patternSamples)) {
+                console.log(`  ${name}:`);
+                for (const hit of hits) {
+                    const src = hit._source;
+                    const ts = String(src["@timestamp"] ?? "").slice(11, 19);
+                    const svc = src.service ? `[${src.service}]` : "";
+                    console.log(`    ${ts} ${svc} ${String(src.log ?? "").slice(0, 150)}`);
+                }
+                console.log();
+            }
+        }
+        // Summary line
+        const status = activePatterns.length > 0 || totalErrors > 100
+            ? (activePatterns.length > 3 || totalErrors > 1000 ? "DEGRADED" : "WARNING")
+            : "HEALTHY";
+        const errRate = hours > 0 ? (totalErrors / hours).toFixed(1) : "0";
+        console.log("  ── Summary ──\n");
+        console.log(`  ${status}: ${activePatterns.length} active patterns, ${totalErrors.toLocaleString()} errors (${errRate}/hr), ${activeSvcCount} services active`);
+        console.log(`  Dashboard: ${this.buildDashboardLink(undefined, fromDt, toDt)}`);
+        console.log(`${"=".repeat(60)}\n`);
+    }
+    // ── Dashboard Link ──────────────────────────────────────────────────
+    buildDashboardLink(queryStr, fromDt, toDt) {
+        const encoded = encodeURIComponent(queryStr ?? "");
+        const timeG = `_g=(time:(from:'${fromDt.toISOString()}',to:'${toDt.toISOString()}'))`;
+        const queryA = `_a=(query:(language:kuery,query:'${encoded}'))`;
+        const dashboardUrl = `https://${this.repo.config.host}/app/dashboards?security_tenant=global#/view/python-services-production-health-v2?${timeG}&${queryA}`;
+        const discoverUrl = `https://${this.repo.config.host}/app/discover?security_tenant=global#/?${timeG}&${queryA}`;
+        return `${dashboardUrl}\n  Discover: ${discoverUrl}`;
+    }
+}
+// ── Helpers ─────────────────────────────────────────────────────────────
+function fmtTime(d) {
+    return d.toISOString().slice(11, 16);
+}
+//# sourceMappingURL=search.service.js.map