@topy-ai/maggie 0.7.54 → 0.7.56

package/bundled-tools/clis/maggie_security_logs.py

@@ -0,0 +1,777 @@
+#!/usr/bin/env python3
+"""Logs and traffic (docs/maggie-security/06-logs-traffic-prd.md):
+``logs-traffic``, ``logs-sigma``, ``logs-ioc``, ``logs-remote``.
+
+Logs are read where they are (files, ``.gz``, stdin, or over SSH with an
+``ssh-read`` consent) and never leave the machine. Every match is an
+"attempt" unless the evidence in PRD 06 section 8 is there (a webshell or a
+secret file answered 2xx with a body, an exploit for a CVE the site is
+currently affected by answered 2xx, a successful admin login from a high-score
+IP); only then is it a "compromise indicator".
+
+Engines are the standard library: a combined/vcombined/JSON log parser, a
+traffic summary, the SigmaHQ web rules (DRL 1.1, compiled by
+``scripts/security/build_sigma_pack.py``) evaluated in Python, and maltrail's
+MIT mass-scanner and user-agent lists. Third-party IP feeds stay off until
+their terms are reviewed (``--feeds``).
+"""
+
+from __future__ import annotations
+
+import argparse
+import collections
+import glob
+import gzip
+import ipaddress
+import json
+import re
+import shlex
+import subprocess
+import sys
+import urllib.parse
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+import maggie_security_core as core
+
+COMBINED = re.compile(r'^(?:(?P<vhost>[^\s:]+)(?::\d+)?\s+)?(?P<ip>[0-9a-fA-F.:]+)\s+\S+\s+(?P<user>\S+)\s+\[(?P<time>[^\]]+)\]\s+'
+                      r'"(?P<method>[A-Z]+|-)\s*(?P<target>[^"\s]*)\s*(?P<version>[^"]*)"\s+(?P<status>\d{3})\s+(?P<bytes>\d+|-)'
+                      r'(?:\s+"(?P<referer>[^"]*)"\s+"(?P<ua>[^"]*)")?')
+CLF_TIME = "%d/%b/%Y:%H:%M:%S %z"
+LEVEL_SEVERITY = {"informational": "info", "low": "low", "medium": "medium", "high": "high", "critical": "critical"}
+WEBSHELL_HINT = re.compile(r"(?i)(webshell|web shell|regeorg|china chopper|cmd\.jsp|shell\.php)")
+SECRET_FILE = re.compile(r"(?i)/(\.env(\.[\w-]+)?|\.git/(config|HEAD)|wp-config\.php\.(bak|old|save|orig|txt|~)|\.htpasswd|id_rsa|\.aws/credentials)$")
+SENSITIVE_QUERY = core.SECRET_QUERY
+
+
+class Record:
+    __slots__ = ("ip", "time", "method", "target", "path", "query", "version", "status", "bytes", "referer", "ua", "vhost", "raw")
+
+    def field(self, name: str) -> str:
+        return {
+            "c-ip": self.ip, "cs-method": self.method, "cs-uri-stem": self.path, "c-uri-stem": self.path, "cs-uri-path": self.path,
+            "cs-uri-query": self.query, "c-uri-query": self.query, "cs-uri": self.target, "c-uri": self.target,
+            "sc-status": str(self.status), "cs-user-agent": self.ua, "c-useragent": self.ua, "cs-referer": self.referer,
+            "cs-host": self.vhost, "r-dns": self.vhost, "cs-version": self.version,
+            "cs-uri-extension": Path(self.path).suffix.lstrip("."), "c-uri-extension": Path(self.path).suffix.lstrip("."), "cs-cookie": "",
+        }.get(name, "")
+
+
+def parse_line(line: str) -> Record | None:
+    line = line.rstrip("\n")
+    if not line:
+        return None
+    if line.lstrip().startswith("{"):
+        return parse_json_line(line)
+    match = COMBINED.match(line)
+    if not match:
+        return None
+    rec = Record()
+    rec.raw = line
+    rec.ip = match.group("ip")
+    try:
+        rec.time = datetime.strptime(match.group("time"), CLF_TIME).astimezone(timezone.utc)
+    except ValueError:
+        return None
+    rec.method = match.group("method")
+    rec.target = match.group("target") or ""
+    rec.path, _, rec.query = rec.target.partition("?")
+    rec.version = match.group("version") or ""
+    rec.status = int(match.group("status"))
+    rec.bytes = int(match.group("bytes")) if match.group("bytes") not in (None, "-") else 0
+    rec.referer = match.group("referer") or ""
+    rec.ua = match.group("ua") or ""
+    rec.vhost = match.group("vhost") or ""
+    return rec
+
+
+def parse_json_line(line: str) -> Record | None:
+    try:
+        data = json.loads(line)
+    except ValueError:
+        return None
+    rec = Record()
+    rec.raw = line
+    request = data.get("request") if isinstance(data.get("request"), dict) else {}
+    if request:  # Caddy
+        headers = request.get("headers") or {}
+        rec.ip = request.get("client_ip") or request.get("remote_ip") or ""
+        stamp = data.get("ts")
+        rec.time = datetime.fromtimestamp(float(stamp), timezone.utc) if isinstance(stamp, (int, float)) else core.parse_time(str(stamp))
+        rec.method, rec.target, rec.vhost = request.get("method", ""), request.get("uri", ""), request.get("host", "")
+        rec.ua = (headers.get("User-Agent") or [""])[0]
+        rec.referer = (headers.get("Referer") or [""])[0]
+        rec.status = int(data.get("status") or 0)
+        rec.bytes = int(data.get("size") or 0)
+        rec.version = request.get("proto", "")
+    elif "ClientIP" in data:  # Cloudflare Logpush (http_requests)
+        rec.ip = data.get("ClientIP", "")
+        stamp = data.get("EdgeStartTimestamp")
+        rec.time = datetime.fromtimestamp(int(stamp) / (1e9 if int(stamp) > 1e12 else 1), timezone.utc) if str(stamp).isdigit() else core.parse_time(str(stamp))
+        rec.method, rec.target, rec.vhost = data.get("ClientRequestMethod", ""), data.get("ClientRequestURI", ""), data.get("ClientRequestHost", "")
+        rec.ua, rec.referer = data.get("ClientRequestUserAgent", ""), data.get("ClientRequestReferer", "")
+        rec.status, rec.bytes, rec.version = int(data.get("EdgeResponseStatus") or 0), int(data.get("EdgeResponseBytes") or 0), ""
+    else:
+        return None
+    rec.path, _, rec.query = rec.target.partition("?")
+    return rec
+
+
+# --------------------------------------------------------------------------- reading
+
+
+def read_lines(ctx: core.Context):
+    args = ctx.args
+    cfg = (ctx.state.config().get("logs") or {})
+    if args.stdin:
+        yield from sys.stdin
+        return
+    if args.ssh:
+        alias, _, target = args.ssh.partition("=")
+        if not target:
+            raise core.UsageError("--ssh takes ALIAS=user@host[:path-glob]")
+        core.require_consent(ctx.state, alias, "ssh-read")
+        host, _, remote_glob = target.partition(":")
+        remote_glob = remote_glob or cfg.get("access_path") or "/var/log/nginx/access.log*"
+        since = int((core.utcnow() - since_delta(ctx)).timestamp())
+        command = (f"find {' '.join(shlex.quote(p) for p in remote_glob.split())} -type f -newermt @{since} -print0 2>/dev/null"
+                   f" | xargs -0 -r ls -tr | tr '\\n' '\\0' | xargs -0 -r zcat -f")
+        proc = subprocess.Popen(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=15", host, command], stdout=subprocess.PIPE,
+                                stderr=subprocess.DEVNULL, text=True, errors="replace")
+        assert proc.stdout is not None
+        with proc.stdout:
+            yield from proc.stdout
+        if proc.wait() not in (0, 123):
+            ctx.degrade("ssh", f"ssh {alias} exited {proc.returncode}")
+        return
+    patterns = args.file or ([cfg["access_path"]] if cfg.get("access_path") else [])
+    if not patterns:
+        raise core.UsageError("no log input: pass --file PATH, --stdin or --ssh ALIAS=user@host (or set logs.access_path in config.json)")
+    files = []
+    for pattern in patterns:
+        matched = sorted(glob.glob(pattern)) if Path(pattern).is_absolute() else (sorted(glob.glob(pattern)) or sorted(glob.glob(str(ctx.project / pattern))))
+        files += [Path(m) for m in matched if Path(m).is_file()]
+    if not files:
+        raise core.UsageError(f"no readable log file matches {', '.join(patterns)}")
+    floor = core.utcnow() - since_delta(ctx)
+    for path in sorted(files, key=lambda p: p.stat().st_mtime):
+        if datetime.fromtimestamp(path.stat().st_mtime, timezone.utc) < floor - timedelta(days=1):
+            continue
+        opener = gzip.open if path.suffix == ".gz" else open
+        try:
+            with opener(path, "rt", encoding="utf-8", errors="replace") as handle:
+                yield from handle
+        except OSError as error:
+            ctx.degrade("logs", f"{path.name}: {error}")
+
+
+def since_delta(ctx: core.Context) -> timedelta:
+    return core.parse_duration(ctx.args.since or "24h")
+
+
+def records(ctx: core.Context):
+    floor = core.utcnow() - since_delta(ctx)
+    excludes = [ipaddress.ip_network(c, strict=False) for c in (ctx.args.exclude_ip or []) + ((ctx.state.config().get("logs") or {}).get("exclude") or [])]
+    vhost = getattr(ctx.args, "vhost", None)
+    stats = {"lines": 0, "parsed": 0, "unparsed": 0, "in_window": 0, "excluded": 0}
+    ctx.run["inputs"]["log_lines"] = stats
+    for line in read_lines(ctx):
+        stats["lines"] += 1
+        rec = parse_line(line)
+        if rec is None:
+            stats["unparsed"] += 1
+            continue
+        stats["parsed"] += 1
+        if rec.time < floor or rec.time > core.utcnow() + timedelta(minutes=5):
+            continue
+        if vhost and rec.vhost and rec.vhost != vhost:
+            continue
+        if excludes:
+            try:
+                address = ipaddress.ip_address(rec.ip)
+                if any(address in net for net in excludes):
+                    stats["excluded"] += 1
+                    continue
+            except ValueError:
+                pass
+        stats["in_window"] += 1
+        yield rec
+
+
+def redact_line(rec: Record) -> str:
+    target = core.mask_query(rec.target)
+    return core.clean_excerpt(f'{rec.ip} [{rec.time.strftime("%Y-%m-%dT%H:%M:%SZ")}] "{rec.method} {target}" {rec.status} {rec.bytes} "{rec.ua[:80]}"')
+
+
+def edge_only(ctx: core.Context) -> bool:
+    if ctx.args.file or ctx.args.stdin or ctx.args.ssh or (ctx.state.config().get("logs") or {}).get("access_path"):
+        return False
+    stack = ctx.state.path("inventory", "stack.json")
+    site_type = core.read_json(stack)["site"].get("type") if stack.is_file() else core.detect_site_type(ctx.project)
+    return site_type == "workers"
+
+
+def edge_only_finding(ctx: core.Context) -> dict:
+    return core.make_finding(kind="availability", command=ctx.command, primary_id="logs-unavailable-edge-only", site=ctx.site,
+                             title="This site runs only on Cloudflare's edge: there are no origin access logs to read", severity="info",
+                             severity_source="maggie", confidence="confirmed", target={"site": ctx.site, "host": "edge"},
+                             remediation={"summary": "Use Cloudflare's Security Analytics, or Logpush to a file and pass it with --file.", "steps": [],
+                                          "snippets": [], "references": ["https://developers.cloudflare.com/logs/"], "cheat_sheet": None})
+
+
+# --------------------------------------------------------------------------- logs-traffic
+
+
+def traffic(ctx: core.Context):
+    if edge_only(ctx):
+        return ctx.finish([edge_only_finding(ctx)])
+    counts = collections.Counter()
+    status_classes = collections.Counter()
+    paths, not_found, ips, uas = collections.Counter(), collections.Counter(), collections.Counter(), collections.Counter()
+    hourly: dict[str, list[int]] = collections.defaultdict(lambda: [0, 0])
+    crawler = re.compile(r"(?i)(bot|crawl|spider|slurp|bingpreview|facebookexternalhit|curl|wget|python-requests|go-http-client)")
+    first = last = None
+    bandwidth = 0
+    for rec in records(ctx):
+        counts["requests"] += 1
+        status_classes[f"{rec.status // 100}xx"] += 1
+        paths[rec.path] += 1
+        ips[rec.ip] += 1
+        bandwidth += rec.bytes
+        if rec.status == 404:
+            not_found[rec.path] += 1
+        if crawler.search(rec.ua or ""):
+            counts["crawler"] += 1
+        hour = rec.time.strftime("%Y-%m-%dT%H")
+        hourly[hour][0] += 1
+        hourly[hour][1] += rec.status >= 500
+        first = rec.time if first is None or rec.time < first else first
+        last = rec.time if last is None or rec.time > last else last
+        uas[rec.ua[:120]] += 1
+    total = counts["requests"]
+    window = {"from": core.iso(first) if first else None, "to": core.iso(last) if last else None, "since": ctx.args.since or "24h"}
+    share = lambda n: round(n / total, 4) if total else 0  # noqa: E731
+    summary = {"window": window, "requests": total, "unique_visitors": len(ips), "bandwidth_bytes": bandwidth,
+               "status": {k: share(v) for k, v in sorted(status_classes.items())},
+               "top_paths": [{"path": core.mask_query(p)[:200], "hits": h} for p, h in paths.most_common(10)],
+               "top_404": [{"path": core.mask_query(p)[:200], "hits": h} for p, h in not_found.most_common(10)],
+               "top_ips": [{"ip": ip, "hits": h, "share": share(h)} for ip, h in ips.most_common(10)],
+               "top_user_agents": [{"ua": ua, "hits": h} for ua, h in uas.most_common(5)],
+               "crawler_share": share(counts["crawler"])}
+    previous = previous_traffic(ctx)
+    if previous and previous.get("requests"):
+        summary["baseline_delta"] = {"requests": f"{(total - previous['requests']) / previous['requests']:+.0%}",
+                                     "5xx": f"{(summary['status'].get('5xx', 0) - previous['status'].get('5xx', 0)) * 100:+.1f}pp"}
+    ctx.run["inputs"]["traffic"] = summary
+    thresholds = {"5xx_share": 0.02, "5xx_hour": 0.10, "single_ip": 0.05, "share_404": 0.30, **((ctx.state.config().get("logs") or {}).get("thresholds") or {})}
+    findings = []
+    if total:
+        five = summary["status"].get("5xx", 0)
+        bad_hours = [h for h, (n, e) in hourly.items() if n >= 20 and e / n > thresholds["5xx_hour"]]
+        if five > thresholds["5xx_share"] or bad_hours:
+            findings.append(core.make_finding(
+                kind="availability", command="logs-traffic", primary_id="traffic-5xx-burst", site=ctx.site,
+                title=f"{five:.1%} of requests failed with a server error" + (f"; worst hours: {', '.join(sorted(bad_hours)[:3])}" if bad_hours else ""),
+                severity="high" if len(bad_hours) > 1 else "medium", severity_source="maggie", confidence="confirmed", target={"site": ctx.site, "host": "origin"},
+                remediation={"summary": "Check the application and server error logs for those hours.", "steps": [], "snippets": [], "references": [], "cheat_sheet": None}))
+        top_ip, top_hits = ips.most_common(1)[0]
+        if share(top_hits) > thresholds["single_ip"] and total >= 200:
+            findings.append(core.make_finding(
+                kind="attack-signal", command="logs-traffic", primary_id="traffic-single-source-dominant", site=ctx.site,
+                title=f"One address made {share(top_hits):.0%} of all requests", severity="low", severity_source="maggie", confidence="possible",
+                target={"site": ctx.site, "host": "origin"},
+                signal={"type": "attempt", "count": top_hits, "top_sources": [top_ip], "window": ctx.args.since or "24h"},
+                remediation={"summary": "If it is not your own monitor or office, consider rate limiting it (see protect-plan).", "steps": [], "snippets": [],
+                             "references": [], "cheat_sheet": None}))
+        if share(sum(not_found.values())) > thresholds["share_404"]:
+            findings.append(core.make_finding(
+                kind="attack-signal", command="logs-traffic", primary_id="traffic-404-heavy", site=ctx.site,
+                title=f"{share(sum(not_found.values())):.0%} of requests were for pages that do not exist (scanner noise)", severity="info",
+                severity_source="maggie", confidence="possible", target={"site": ctx.site, "host": "origin"},
+                signal={"type": "attempt", "count": sum(not_found.values()), "top_sources": [ip for ip, _ in ips.most_common(3)], "window": ctx.args.since or "24h"},
+                remediation={"summary": "Normal for public sites; no action unless it slows the server.", "steps": [], "snippets": [], "references": [], "cheat_sheet": None}))
+        if previous and previous.get("requests") and total > 3 * previous["requests"] and total >= 1000:
+            findings.append(core.make_finding(
+                kind="availability", command="logs-traffic", primary_id="traffic-spike", site=ctx.site,
+                title=f"Traffic is {total / previous['requests']:.1f}× the previous window", severity="low", severity_source="maggie",
+                confidence="possible", target={"site": ctx.site, "host": "origin"},
+                remediation={"summary": "Check whether it is a campaign, a crawler or an attack.", "steps": [], "snippets": [], "references": [], "cheat_sheet": None}))
+    elif ctx.run["inputs"]["log_lines"]["lines"] and not ctx.run["inputs"]["log_lines"]["parsed"]:
+        raise core.UsageError("no line could be parsed; supported formats: nginx/Apache combined or vcombined, Caddy JSON, Cloudflare Logpush JSON")
+    ctx.run["tools"].append({"name": "maggie-logs-native", "version": core.maggie_version(), "sha256": None})
+    return ctx.finish(findings, covered=["*"])
+
+
+def previous_traffic(ctx: core.Context) -> dict | None:
+    runs = ctx.state.path("runs")
+    for path in sorted(runs.glob("*logs-traffic*.json"), reverse=True) if runs.is_dir() else []:
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+        except ValueError:
+            continue
+        summary = (data["run"].get("inputs") or {}).get("traffic")
+        if summary and summary.get("window", {}).get("since") == (ctx.args.since or "24h"):
+            return summary
+    return None
+
+
+# --------------------------------------------------------------------------- Sigma
+
+
+class Rule:
+    def __init__(self, data: dict) -> None:
+        self.data = data
+        detection = dict(data["detection"])
+        self.condition = str(detection.pop("condition"))
+        detection.pop("timeframe", None)
+        self.blocks = {name: self._compile(block) for name, block in detection.items()}
+        self.tokens = re.findall(r"\(|\)|[A-Za-z0-9_*]+", self.condition.replace("1 of", "1_of").replace("all of", "all_of"))
+
+    @staticmethod
+    def _compile(block):
+        if isinstance(block, list) and all(not isinstance(x, dict) for x in block):
+            return ("keywords", [str(x).lower() for x in block])
+        alternatives = block if isinstance(block, list) else [block]
+        compiled = []
+        for alt in alternatives:
+            fields = []
+            for key, values in (alt or {}).items():
+                name, *mods = key.split("|")
+                values = values if isinstance(values, list) else [values]
+                if "re" in mods:
+                    matchers = [re.compile(str(v), re.I) for v in values]
+                elif "cidr" in mods:
+                    matchers = [ipaddress.ip_network(str(v), strict=False) for v in values]
+                else:
+                    matchers = [("" if v is None else str(v)).lower() for v in values]
+                fields.append((name, set(mods), matchers))
+            compiled.append(fields)
+        return ("map", compiled)
+
+    @staticmethod
+    def _value_matches(value: str, mods: set, matcher) -> bool:
+        if "re" in mods:
+            return bool(matcher.search(value))
+        if "cidr" in mods:
+            try:
+                return ipaddress.ip_address(value) in matcher
+            except ValueError:
+                return False
+        value = value.lower()
+        if "contains" in mods:
+            return matcher in value
+        if "startswith" in mods:
+            return value.startswith(matcher)
+        if "endswith" in mods:
+            return value.endswith(matcher)
+        if "*" in matcher or "?" in matcher:
+            return re.fullmatch(re.escape(matcher).replace(r"\*", ".*").replace(r"\?", "."), value) is not None
+        return value == matcher
+
+    def _block(self, name: str, rec: Record, variants: list[str]) -> bool:
+        kind, payload = self.blocks[name]
+        if kind == "keywords":
+            return any(kw in v for v in variants for kw in payload)
+        for fields in payload:
+            ok = True
+            for field, mods, matchers in fields:
+                if field == "":  # keywords with modifiers ("|all")
+                    hits = [any(m in v for v in variants) for m in matchers]
+                else:
+                    raw = rec.field(field)
+                    candidates = [raw, urllib.parse.unquote_plus(raw)] if field in ("cs-uri", "c-uri", "cs-uri-query", "c-uri-query", "cs-uri-stem") else [raw]
+                    hits = [any(self._value_matches(c, mods, m) for c in candidates) for m in matchers]
+                if not (all(hits) if "all" in mods else any(hits)):
+                    ok = False
+                    break
+            if ok:
+                return True
+        return False
+
+    def matches(self, rec: Record) -> bool:
+        lowered = rec.raw.lower()
+        variants = [lowered, urllib.parse.unquote_plus(lowered)]
+        pos = 0
+        tokens = self.tokens
+
+        def expr() -> bool:
+            nonlocal pos
+            value = term()
+            while pos < len(tokens) and tokens[pos] == "or":
+                pos += 1
+                rhs = term()
+                value = value or rhs
+            return value
+
+        def term() -> bool:
+            nonlocal pos
+            value = factor()
+            while pos < len(tokens) and tokens[pos] == "and":
+                pos += 1
+                rhs = factor()
+                value = value and rhs
+            return value
+
+        def factor() -> bool:
+            nonlocal pos
+            token = tokens[pos]
+            pos += 1
+            if token == "not":
+                return not factor()
+            if token == "(":
+                value = expr()
+                pos += 1  # ')'
+                return value
+            if token in ("1_of", "all_of"):
+                pattern = tokens[pos]
+                pos += 1
+                names = list(self.blocks) if pattern == "them" else [n for n in self.blocks if re.fullmatch(pattern.replace("*", ".*"), n)]
+                results = (self._block(n, rec, variants) for n in names)
+                return any(results) if token == "1_of" else (bool(names) and all(results))
+            return self._block(token, rec, variants)
+
+        return expr()
+
+
+def load_rules(groups: set[str], rule_ids: list[str] | None, min_level: str) -> tuple[list[Rule], dict]:
+    pack = core.read_json(core.DATA_DIR / "sigma-web.json")
+    order = ["informational", "low", "medium", "high", "critical"]
+    floor = order.index(min_level)
+    rules, skipped = [], {"unsupported": 0, "below_level": 0, "other_group": 0}
+    for data in pack["rules"]:
+        if rule_ids and data["id"] not in rule_ids:
+            continue
+        if not rule_ids and data["group"] not in groups:
+            skipped["other_group"] += 1
+            continue
+        if data["status"] == "unsupported":
+            skipped["unsupported"] += 1
+            continue
+        if order.index(data.get("level", "medium") if data.get("level") in order else "medium") < floor:
+            skipped["below_level"] += 1
+            continue
+        try:
+            rules.append(Rule(data))
+        except (KeyError, re.error, ValueError):
+            skipped["unsupported"] += 1
+    return rules, {"commit": pack["commit"], **skipped}
+
+
+def open_cve_aliases(ctx: core.Context) -> set[str]:
+    """CVE ids of vulnerabilities vuln-watch currently reports as open for this site."""
+    ledger = core.Ledger(ctx.state.path("ledger.jsonl"))
+    open_ids = {fid for fid, events in ledger.history().items() if core.Ledger.derive(events)["status"] in ("new", "open")}
+    aliases = set()
+    runs = ctx.state.path("runs")
+    for path in sorted(runs.glob("*vuln-watch*.json"))[-3:] if runs.is_dir() else []:
+        for finding in json.loads(path.read_text(encoding="utf-8")).get("findings") or []:
+            if finding["id"] in open_ids:
+                aliases |= {a.upper() for a in finding.get("aliases") or [] if a.upper().startswith("CVE-")}
+    return aliases
+
+
+def sigma(ctx: core.Context):
+    args = ctx.args
+    groups = set((args.rules or "web,proxy").split(","))
+    rules, info = load_rules(groups, args.rule_id, args.min_level)
+    if args.list_rules:
+        pack = core.read_json(core.DATA_DIR / "sigma-web.json")
+        return {"_raw": json.dumps({"commit": pack["commit"], "licence": pack["licence"], "counts": pack["counts"],
+                                    "rules": [{k: r[k] for k in ("id", "title", "group", "level", "status", "unsupported_reasons")} for r in pack["rules"]]},
+                                   indent=2)}
+    if edge_only(ctx):
+        return ctx.finish([edge_only_finding(ctx)])
+    allow = (ctx.state.config().get("logs") or {}).get("sigma_allow") or {}
+    allow_paths = [re.compile(p) for p in allow.get("paths") or []]
+    allow_ids = set(allow.get("rule_ids") or [])
+    hits: dict[str, dict] = {}
+    for rec in records(ctx):
+        if allow_paths and any(p.search(rec.path) for p in allow_paths):
+            continue
+        for rule in rules:
+            if rule.data["id"] in allow_ids or not rule.matches(rec):
+                continue
+            entry = hits.setdefault(rule.data["id"], {"rule": rule, "count": 0, "ips": collections.Counter(), "status": collections.Counter(),
+                                                      "samples": [], "ok_samples": [], "first": rec.time, "last": rec.time, "ok_bodies": 0, "ok_paths": set()})
+            entry["count"] += 1
+            entry["ips"][rec.ip] += 1
+            entry["status"][str(rec.status)] += 1
+            entry["first"], entry["last"] = min(entry["first"], rec.time), max(entry["last"], rec.time)
+            if len(entry["samples"]) < args.samples:
+                entry["samples"].append(redact_line(rec))
+            if 200 <= rec.status < 300:
+                if rec.bytes > 0:
+                    entry["ok_bodies"] += 1
+                entry["ok_paths"].add(rec.path)
+                if len(entry["ok_samples"]) < args.samples:
+                    entry["ok_samples"].append(redact_line(rec))
+    affected_cves = open_cve_aliases(ctx)
+    findings = []
+    for rule_id, entry in hits.items():
+        data = entry["rule"].data
+        success = sum(v for k, v in entry["status"].items() if k.startswith("2"))
+        level = LEVEL_SEVERITY.get(data.get("level"), "medium")
+        title = data["title"]
+        rule_cves = {t.upper().replace("CVE.", "CVE-") for t in data.get("tags") or [] if "cve" in t.lower()} | \
+            {m.upper() for m in re.findall(r"(?i)cve-\d{4}-\d+", title + " " + " ".join(data.get("references") or []))}
+        compromise_reason = None
+        if success and entry["ok_bodies"]:
+            if WEBSHELL_HINT.search(title) and any(p.lower().endswith((".php", ".jsp", ".aspx", ".asp")) for p in entry["ok_paths"]):
+                compromise_reason = "a webshell path answered 200 with content"
+            elif any(SECRET_FILE.search(p) for p in entry["ok_paths"]):
+                compromise_reason = "a secret file (.env, .git/config, config backup) was served with content"
+            elif rule_cves & affected_cves:
+                compromise_reason = f"an exploit for {', '.join(sorted(rule_cves & affected_cves))}, which this site is affected by, got a normal response"
+        if compromise_reason:
+            severity, signal_type, confidence = "critical", "compromise-indicator", "likely"
+        elif success:
+            severity = {"info": "low", "low": "medium", "medium": "high", "high": "high", "critical": "critical"}[level] if level in ("high", "critical") else \
+                {"info": "low", "low": "medium", "medium": "medium"}.get(level, "medium")
+            signal_type, confidence = "attempt", "possible"
+        else:
+            severity = level if level in ("info", "low") else "medium"
+            signal_type, confidence = "attempt", "possible"
+        top = [ip for ip, _ in entry["ips"].most_common(10)]
+        wording = (f"Possible break-in: {compromise_reason}" if compromise_reason else
+                   f"{entry['count']} attempt{'s' if entry['count'] != 1 else ''}: {title}"
+                   + (f" — {success} got a normal response; check {'them' if success != 1 else 'it'}" if success else ""))
+        findings.append(core.make_finding(
+            kind="attack-signal", command="logs-sigma", primary_id=rule_id, site=ctx.site, title=wording[:300], severity=severity,
+            severity_source="tool", confidence=confidence, target={"site": ctx.site, "host": "origin"},
+            signal={"type": signal_type, "count": entry["count"], "top_sources": top, "window": args.since or "24h"},
+            evidence=[{"path": "access log", "excerpt": s} for s in (entry["ok_samples"] if success else entry["samples"])[:args.samples]],
+            tool="sigma", rule_id=rule_id, asvs_ids=[],
+            remediation={"summary": ("Check the listed requests and the server now; treat the site as possibly compromised until checked."
+                                     if compromise_reason else "These are attempts; nothing indicates they worked. Rate limiting or a WAF reduces the noise (protect-plan)."),
+                         "steps": [f"False positives Sigma lists: {'; '.join(map(str, data.get('falsepositives') or []))[:300]}"] if data.get("falsepositives") else [],
+                         "snippets": [], "references": (data.get("references") or [])[:4], "cheat_sheet": None},
+            untrusted={"sigma": {"title": data["title"], "author": data.get("author"), "id": rule_id, "licence": "DRL-1.1",
+                                 "url": f"https://github.com/SigmaHQ/sigma/blob/{info['commit']}/{data.get('source_path', '')}"}}))
+        findings[-1]["signal"].update({"success_count": success, "status_breakdown": dict(entry["status"]), "first": core.iso(entry["first"]),
+                                       "last": core.iso(entry["last"])})
+    ctx.run["inputs"]["sigma"] = {**info, "rules_run": len(rules), "groups": sorted(groups)}
+    ctx.run["tools"].append({"name": "sigma-rules", "version": info["commit"][:12], "sha256": None})
+    doc = ctx.finish(findings, covered=["*"])
+    if any((f.get("signal") or {}).get("type") == "compromise-indicator" and f["status"] in ("new", "open") for f in doc["findings"]) and args.ci:
+        args.fail_on = "info"
+    return doc
+
+
+# --------------------------------------------------------------------------- IOC
+
+
+class IocLists:
+    def __init__(self, ctx: core.Context, feeds: list[str]) -> None:
+        data = core.read_json(core.DATA_DIR / "maltrail-lists.json")
+        self.mass_ips = set(data["mass_scanner_ips"])
+        self.mass_nets = [ipaddress.ip_network(c, strict=False) for c in data["mass_scanner_cidrs"] if "/" in c]
+        self.ua = re.compile("|".join(f"(?:{p})" for p in data["suspicious_ua"]))
+        self.feeds: dict[str, set[str]] = {}
+        self.nets: dict[str, list] = {}
+        for feed_id in feeds:
+            spec = IOC_FEEDS.get(feed_id)
+            if not spec:
+                raise core.UsageError(f"unknown IOC feed {feed_id}; known: {', '.join(IOC_FEEDS)}")
+            body = ctx.cache.get(f"ioc-{feed_id}", spec["url"], ttl="12h")
+            if body is None:
+                continue
+            ips, nets = set(), []
+            for line in body.decode("utf-8", errors="replace").splitlines():
+                item = line.split("#", 1)[0].split(";", 1)[0].strip()
+                if not item:
+                    continue
+                if "/" in item:
+                    try:
+                        nets.append(ipaddress.ip_network(item, strict=False))
+                    except ValueError:
+                        pass
+                else:
+                    ips.add(item)
+            self.feeds[feed_id], self.nets[feed_id] = ips, nets
+
+    def mass_scanner(self, ip: str) -> bool:
+        if ip in self.mass_ips:
+            return True
+        try:
+            address = ipaddress.ip_address(ip)
+        except ValueError:
+            return False
+        return any(address in net for net in self.mass_nets)
+
+    def listed_in(self, ip: str) -> list[str]:
+        out = []
+        try:
+            address = ipaddress.ip_address(ip)
+        except ValueError:
+            return out
+        for feed_id, ips in self.feeds.items():
+            if ip in ips or any(address in net for net in self.nets[feed_id]):
+                out.append(feed_id)
+        return out
+
+
+# Feeds that need a terms review before default use (PRD 06 section 6.3.1); none is on by default.
+IOC_FEEDS = {
+    "blocklist-de": {"url": "https://lists.blocklist.de/lists/all.txt", "host": "lists.blocklist.de"},
+    "greensnow": {"url": "https://blocklist.greensnow.co/greensnow.txt", "host": "blocklist.greensnow.co"},
+    "et-compromised": {"url": "https://rules.emergingthreats.net/open/suricata/rules/compromised-ips.txt", "host": "rules.emergingthreats.net"},
+    "firehol-dshield-1d": {"url": "https://iplists.firehol.org/files/dshield_1d.netset", "host": "iplists.firehol.org"},
+}
+for _id, _spec in IOC_FEEDS.items():
+    core.SOURCES.setdefault(f"ioc-{_id}", {"hosts": [_spec["host"]], "ttl": "12h", "url": _spec["url"]})
+
+
+def ioc(ctx: core.Context):
+    args = ctx.args
+    if edge_only(ctx):
+        return ctx.finish([edge_only_finding(ctx)])
+    feeds = [f for f in (args.feeds or "").split(",") if f]
+    lists = IocLists(ctx, feeds)
+    per_ip: dict[str, dict] = {}
+    ua_hits: dict[str, dict] = {}
+    for rec in records(ctx):
+        entry = per_ip.setdefault(rec.ip, {"count": 0, "samples": [], "ua": False})
+        entry["count"] += 1
+        if len(entry["samples"]) < 2:
+            entry["samples"].append(redact_line(rec))
+        if rec.ua and lists.ua.search(rec.ua):
+            entry["ua"] = True
+            group = ua_hits.setdefault(rec.ua[:60], {"count": 0, "ips": collections.Counter(), "sample": redact_line(rec)})
+            group["count"] += 1
+            group["ips"][rec.ip] += 1
+    sigma_ips = set()
+    runs = ctx.state.path("runs")
+    for path in sorted(runs.glob("*logs-sigma*.json"))[-1:] if runs.is_dir() else []:
+        for finding in json.loads(path.read_text(encoding="utf-8")).get("findings") or []:
+            sigma_ips |= set((finding.get("signal") or {}).get("top_sources") or [])
+    findings, aggregate = [], collections.Counter()
+    for ip, entry in per_ip.items():
+        listed = lists.listed_in(ip)
+        score, label = 0, None
+        if listed:
+            score = 40 + 15 * (len(listed) - 1)
+        if entry["ua"]:
+            score += 15
+        if ip in sigma_ips and score:
+            score += 15
+        if not listed and lists.mass_scanner(ip):
+            score, label = 30, "mass-scanner"
+        score = min(score, 100)
+        if score == 0:
+            continue
+        label = label or ("high-confidence" if score >= 70 else "medium-confidence" if score >= 55 else "low-confidence")
+        if score < args.min_confidence:
+            aggregate[label] += entry["count"]
+            continue
+        findings.append(core.make_finding(
+            kind="attack-signal", command="logs-ioc", primary_id=f"ioc-ip-{ip}", site=ctx.site,
+            title=f"{entry['count']} requests from {ip}, listed by {', '.join(listed) or 'no IP feed'} (score {score}, {label})",
+            severity="medium" if score >= 70 else "low", severity_source="maggie", confidence="likely" if score >= 70 else "possible",
+            target={"site": ctx.site, "host": "origin"}, signal={"type": "attempt", "count": entry["count"], "top_sources": [ip], "window": args.since or "24h"},
+            evidence=[{"path": "access log", "excerpt": s} for s in entry["samples"]], rule_id="ioc-ip",
+            remediation={"summary": "Block recommended (protect-plan can add it to CrowdSec or fail2ban)." if score >= 70 else "Watch; block if it repeats next week.",
+                         "steps": [], "snippets": [], "references": [], "cheat_sheet": None}))
+    for index, (ua, group) in enumerate(sorted(ua_hits.items(), key=lambda kv: -kv[1]["count"])[:10], 1):
+        findings.append(core.make_finding(
+            kind="attack-signal", command="logs-ioc", primary_id=f"ioc-ua:{ua}", site=ctx.site,
+            title=f"{group['count']} requests with a scanner or exploit user agent ({ua[:50]})", severity="low", severity_source="maggie",
+            confidence="possible", target={"site": ctx.site, "host": "origin"}, rule_id=f"ioc-ua:{index}",
+            signal={"type": "attempt", "count": group["count"], "top_sources": [ip for ip, _ in group["ips"].most_common(5)], "window": args.since or "24h"},
+            evidence=[{"path": "access log", "excerpt": group["sample"]}],
+            remediation={"summary": "Automated scanning; no action unless it hurts performance.", "steps": [], "snippets": [], "references": [], "cheat_sheet": None},
+            untrusted={"user_agent": ua}))
+    if aggregate:
+        findings.append(core.make_finding(
+            kind="attack-signal", command="logs-ioc", primary_id="ioc-aggregate", site=ctx.site,
+            title="Low-confidence and mass-scanner traffic: " + ", ".join(f"{n} requests {label}" for label, n in aggregate.items()),
+            severity="info", severity_source="maggie", confidence="possible", target={"site": ctx.site, "host": "origin"},
+            signal={"type": "attempt", "count": sum(aggregate.values()), "top_sources": [], "window": args.since or "24h"},
+            remediation={"summary": "Background noise of the internet; informational.", "steps": [], "snippets": [], "references": [], "cheat_sheet": None}))
+    ctx.run["inputs"]["ioc"] = {"feeds": feeds, "lists": ["maltrail-mass-scanner", "maltrail-ua"], "ips_seen": len(per_ip)}
+    return ctx.finish(findings, covered=["*"])
+
+
+# --------------------------------------------------------------------------- logs-remote (summary only, nothing written on the host)
+
+
+REMOTE_AWK = r'''
+{ ip=$1; n++; split($0, q, "\""); split(q[2], r, " "); path=r[2]; sub(/\?.*/, "", path); st=$(NF>=9?9:NF);
+  match($0, /" [0-9][0-9][0-9] /); if (RSTART) st=substr($0, RSTART+2, 3);
+  ips[ip]++; paths[path]++; status[substr(st,1,1) "xx"]++; if (st=="404") nf[path]++;
+  if (pat != "" && index(tolower($0), tolower(pat))) { ph++; phip[ip]++ } }
+END { printf "{\"requests\":%d", n;
+  printf ",\"status\":{"; s=""; for (k in status) { printf "%s\"%s\":%d", s, k, status[k]; s="," } printf "}";
+  printf ",\"top_ips\":["; s=""; c=0; for (k in ips) if (ips[k] > n/200 || c < 10) { printf "%s[\"%s\",%d]", s, k, ips[k]; s=","; c++ } printf "]";
+  printf ",\"top_paths\":["; s=""; c=0; for (k in paths) if (paths[k] > n/200 || c < 10) { gsub(/["\\]/, "", k); printf "%s[\"%s\",%d]", s, k, paths[k]; s=","; c++ } printf "]";
+  printf ",\"pattern_hits\":%d}\n", ph }
+'''
+
+
+def remote(ctx: core.Context):
+    args = ctx.args
+    alias, _, target = (args.ssh or "").partition("=")
+    if not target:
+        raise core.UsageError("--ssh ALIAS=user@host[:path-glob] is required")
+    core.require_consent(ctx.state, alias, "ssh-read")
+    host, _, remote_glob = target.partition(":")
+    remote_glob = remote_glob or (ctx.state.config().get("logs") or {}).get("access_path") or "/var/log/nginx/access.log*"
+    since = int((core.utcnow() - since_delta(ctx)).timestamp())
+    command = (f"find {' '.join(shlex.quote(p) for p in remote_glob.split())} -type f -newermt @{since} -print0 2>/dev/null | xargs -0 -r zcat -f"
+               f" | awk -v pat={shlex.quote(args.pattern or '')} {shlex.quote(REMOTE_AWK)}")
+    proc = subprocess.run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=15", host, command], capture_output=True, text=True, timeout=900, check=False)
+    try:
+        raw = json.loads(proc.stdout.strip().splitlines()[-1])
+    except (ValueError, IndexError):
+        raise core.DegradedError(f"ssh {alias}: no summary returned ({core.clean_excerpt(proc.stderr, 200)})")
+    total = raw.get("requests", 0)
+    summary = {"requests": total, "status": {k: round(v / total, 4) for k, v in raw.get("status", {}).items()} if total else {},
+               "top_ips": sorted(({"ip": ip, "hits": h} for ip, h in raw.get("top_ips", [])), key=lambda x: -x["hits"])[:10],
+               "top_paths": sorted(({"path": core.mask_query(p)[:200], "hits": h} for p, h in raw.get("top_paths", [])), key=lambda x: -x["hits"])[:10],
+               "pattern": args.pattern, "pattern_hits": raw.get("pattern_hits", 0), "note": "summary computed on the host; no log line left it"}
+    ctx.run["inputs"]["traffic"] = summary
+    ctx.run["inputs"]["remote_cleanup"] = "nothing written"
+    return ctx.finish([], covered=[])
+
+
+# --------------------------------------------------------------------------- registration
+
+
+def _common(p: argparse.ArgumentParser) -> None:
+    p.add_argument("--file", action="append", help="log file or glob (repeatable; .gz is fine)")
+    p.add_argument("--stdin", action="store_true")
+    p.add_argument("--ssh", help="ALIAS=user@host[:path-glob]; needs ssh-read consent")
+    p.add_argument("--vhost")
+    p.add_argument("--exclude-ip", action="append", help="CIDR to ignore (your office, uptime probes)")
+
+
+def _traffic(p: argparse.ArgumentParser) -> None:
+    _common(p)
+
+
+def _sigma(p: argparse.ArgumentParser) -> None:
+    _common(p)
+    p.add_argument("--rules", help="groups: web,proxy,emerging (default web,proxy)")
+    p.add_argument("--rule-id", action="append")
+    p.add_argument("--min-level", choices=["informational", "low", "medium", "high", "critical"], default="medium")
+    p.add_argument("--samples", type=int, default=3)
+    p.add_argument("--list-rules", action="store_true")
+
+
+def _ioc(p: argparse.ArgumentParser) -> None:
+    _common(p)
+    p.add_argument("--feeds", help=f"extra IP feeds (terms to be reviewed by you): {', '.join(IOC_FEEDS)}")
+    p.add_argument("--min-confidence", type=int, default=55)
+
+
+def _remote(p: argparse.ArgumentParser) -> None:
+    p.add_argument("--ssh", required=True)
+    p.add_argument("--pattern", help="count lines containing this text (e.g. /xmlrpc.php)")
+
+
+COMMANDS = [
+    core.Command("logs-traffic", "traffic summary from access logs: requests, errors, top paths and addresses, spikes", ["dns"], _traffic, traffic),
+    core.Command("logs-sigma", "attack patterns in access logs (SigmaHQ web rules); attempts are never called break-ins without evidence", [], _sigma, sigma),
+    core.Command("logs-ioc", "known scanners, suspicious user agents and (opt-in) attacker IP lists in access logs",
+                 [f"ioc-{i}" for i in IOC_FEEDS], _ioc, ioc),
+    core.Command("logs-remote", "summary of a server's access logs computed on the server (only the summary comes back)", [], _remote, remote),
+]