@simbimbo/brainstem 0.0.1 → 0.0.3

package/brainstem/ingest.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+from dataclasses import asdict
+from datetime import datetime
+from pathlib import Path
+from typing import Iterable, List
+
+from .fingerprint import fingerprint_event, normalize_message
+from .models import CanonicalEvent, Event, RawInputEnvelope, Signature
+
+
+def parse_syslog_line(line: str, *, tenant_id: str, source_path: str = "") -> CanonicalEvent:
+    return canonicalize_raw_input_envelope(
+        parse_syslog_envelope(line, tenant_id=tenant_id, source_path=source_path)
+    )
+
+
+def parse_syslog_envelope(line: str, *, tenant_id: str, source_path: str = "") -> RawInputEnvelope:
+    text = (line or "").rstrip("\n")
+    timestamp = datetime.utcnow().isoformat() + "Z"
+    host = ""
+    service = ""
+    message = text
+
+    parts = text.split()
+    if len(parts) >= 5:
+        host = parts[3]
+        rest = " ".join(parts[4:])
+        if ":" in rest:
+            svc, _, msg = rest.partition(":")
+            service = svc.strip()
+            message = msg.strip() or rest.strip()
+        else:
+            message = rest.strip()
+
+    return RawInputEnvelope(
+        tenant_id=tenant_id,
+        source_type="syslog",
+        timestamp=timestamp,
+        message_raw=message,
+        host=host,
+        service=service,
+        source_path=source_path,
+        metadata={"raw_line": text},
+    )
+
+
+def parse_syslog_envelopes(lines: Iterable[str], *, tenant_id: str, source_path: str = "") -> List[RawInputEnvelope]:
+    return [parse_syslog_envelope(line, tenant_id=tenant_id, source_path=source_path) for line in lines if str(line).strip()]
+
+
+def canonicalize_raw_input_envelope(raw: RawInputEnvelope) -> CanonicalEvent:
+    parse_error = (raw.metadata or {}).get("parse_error")
+    if parse_error:
+        raise ValueError(f"parse_error: {parse_error}")
+
+    if not (raw.message_raw or "").strip():
+        raise ValueError("message_raw is empty and cannot be canonicalized")
+
+    message_normalized = normalize_message(raw.message_raw)
+    metadata = dict(raw.metadata or {})
+    metadata.setdefault("canonicalization_source", raw.source_type)
+    metadata["raw_input_seen"] = True
+    return CanonicalEvent(
+        tenant_id=raw.tenant_id,
+        source_type=raw.source_type,
+        timestamp=raw.timestamp,
+        host=raw.host,
+        service=raw.service,
+        severity=raw.severity,
+        asset_id=raw.asset_id,
+        source_path=raw.source_path,
+        facility=raw.facility,
+        message_raw=raw.message_raw,
+        structured_fields=dict(raw.structured_fields),
+        correlation_keys=dict(raw.correlation_keys),
+        message_normalized=message_normalized,
+        signature_input=message_normalized,
+        ingest_metadata={
+            "canonicalized_at": datetime.utcnow().isoformat() + "Z",
+            "source_timestamp": raw.timestamp,
+            **metadata,
+        },
+    )
+
+
+def canonicalize_raw_input_envelopes(events: Iterable[RawInputEnvelope]) -> List[CanonicalEvent]:
+    return [canonicalize_raw_input_envelope(raw_event) for raw_event in events]
+
+
+def ingest_syslog_lines(lines: Iterable[str], *, tenant_id: str, source_path: str = "") -> List[CanonicalEvent]:
+    return canonicalize_raw_input_envelopes(
+        parse_syslog_envelopes(lines, tenant_id=tenant_id, source_path=source_path),
+    )
+
+
+def ingest_syslog_file(path: str, *, tenant_id: str) -> List[Event]:
+    file_path = Path(path)
+    lines = file_path.read_text(encoding="utf-8", errors="ignore").splitlines()
+    return ingest_syslog_lines(lines, tenant_id=tenant_id, source_path=str(file_path))
+
+
+def signatures_for_events(events: Iterable[Event]) -> List[Signature]:
+    return [fingerprint_event(event) for event in events]
+
+
+def events_as_dicts(events: Iterable[Event]) -> List[dict]:
+    return [asdict(event) for event in events]

package/brainstem/instrumentation.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+import json
+import sys
+import time
+from contextlib import contextmanager
+from typing import Any, Dict, Iterator
+
+
+def emit(event: str, **fields: Any) -> None:
+    payload: Dict[str, Any] = {
+        "event": event,
+        "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        **fields,
+    }
+    print(json.dumps(payload, ensure_ascii=False), file=sys.stderr)
+
+
+@contextmanager
+def span(event: str, **fields: Any) -> Iterator[None]:
+    started = time.perf_counter()
+    emit(f"{event}_start", **fields)
+    try:
+        yield
+    except Exception as exc:
+        emit(
+            f"{event}_failed",
+            error_type=type(exc).__name__,
+            error=str(exc),
+            elapsed_ms=round((time.perf_counter() - started) * 1000, 3),
+            **fields,
+        )
+        raise
+    emit(
+        f"{event}_complete",
+        elapsed_ms=round((time.perf_counter() - started) * 1000, 3),
+        **fields,
+    )

package/brainstem/interesting.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+from typing import Iterable, List, Dict, Any
+
+from .models import Candidate
+
+
+def _attention_band(decision_band: str) -> str:
+    mapping = {
+        "ignore": "ignore_fast",
+        "watch": "background",
+        "review": "watch",
+        "urgent_human_review": "investigate",
+        "promote_to_incident_memory": "promote",
+    }
+    return mapping.get(decision_band, "watch")
+
+
+def _why_it_matters(candidate: Candidate) -> str:
+    count = int((candidate.metadata or {}).get("count") or 0)
+    service = str((candidate.metadata or {}).get("service") or "").strip()
+    family = candidate.candidate_type.replace("_", " ")
+    pieces = []
+    if count:
+        pieces.append(f"observed {count} times")
+    if service:
+        pieces.append(f"around {service}")
+    band = _attention_band(candidate.decision_band)
+    if band == "promote":
+        level = "has earned high operator attention"
+    elif band == "investigate":
+        level = "has earned active operator attention"
+    elif band == "watch":
+        level = "has earned watch-level attention"
+    elif band == "background":
+        level = "is worth keeping in the background"
+    else:
+        level = "is low-attention noise"
+    detail = ", ".join(pieces) if pieces else family
+    return f"{detail}; {level}."
+
+
+def interesting_items(candidates: Iterable[Candidate], *, limit: int = 5) -> List[Dict[str, Any]]:
+    ordered = sorted(candidates, key=lambda c: (c.score_total, c.confidence), reverse=True)
+    items: List[Dict[str, Any]] = []
+    for candidate in ordered[: max(limit, 1)]:
+        attention_band = _attention_band(candidate.decision_band)
+        items.append(
+            {
+                "title": candidate.title,
+                "summary": candidate.summary,
+                "decision_band": candidate.decision_band,
+                "attention_band": attention_band,
+                "attention_score": candidate.score_total,
+                "score_total": candidate.score_total,
+                "confidence": candidate.confidence,
+                "why_it_matters": _why_it_matters(candidate),
+                "signals": dict(candidate.score_breakdown),
+                "metadata": dict(candidate.metadata),
+            }
+        )
+    return items

package/brainstem/models.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, List
+
+
+@dataclass
+class RawInputEnvelope:
+    tenant_id: str
+    source_type: str
+    timestamp: str
+    message_raw: str
+    source_id: str = ""
+    source_name: str = ""
+    host: str = ""
+    service: str = ""
+    severity: str = "info"
+    asset_id: str = ""
+    source_path: str = ""
+    facility: str = ""
+    structured_fields: Dict[str, Any] = field(default_factory=dict)
+    correlation_keys: Dict[str, Any] = field(default_factory=dict)
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class CanonicalEvent:
+    tenant_id: str
+    source_type: str
+    timestamp: str
+    message_raw: str
+    host: str = ""
+    service: str = ""
+    severity: str = "info"
+    asset_id: str = ""
+    source_path: str = ""
+    facility: str = ""
+    structured_fields: Dict[str, Any] = field(default_factory=dict)
+    correlation_keys: Dict[str, Any] = field(default_factory=dict)
+    message_normalized: str = ""
+    signature_input: str = ""
+    ingest_metadata: Dict[str, Any] = field(default_factory=dict)
+
+
+# Backward-compatible name used throughout existing code.
+Event = CanonicalEvent
+
+
+@dataclass
+class Signature:
+    signature_key: str
+    event_family: str
+    normalized_pattern: str
+    service: str = ""
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class Candidate:
+    candidate_type: str
+    title: str
+    summary: str
+    score_total: float
+    score_breakdown: Dict[str, float]
+    decision_band: str
+    source_signature_ids: List[str] = field(default_factory=list)
+    source_event_ids: List[str] = field(default_factory=list)
+    confidence: float = 0.0
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class IncidentMemory:
+    title: str
+    summary: str
+    incident_type: str
+    source_candidate_ids: List[str] = field(default_factory=list)
+    recurrence_count: int = 0
+    confidence: float = 0.0
+    metadata: Dict[str, Any] = field(default_factory=dict)

package/brainstem/recurrence.py

@@ -0,0 +1,112 @@
+from __future__ import annotations
+
+from collections import Counter
+from dataclasses import asdict
+from typing import Iterable, List
+
+from .models import Candidate, Event, Signature
+from .scoring import score_candidate
+
+
+FAMILY_TITLES = {
+    "failure": "Recurring failure pattern",
+    "auth": "Recurring authentication failure pattern",
+    "service_lifecycle": "Recurring service lifecycle instability",
+    "generic": "Recurring operational pattern",
+}
+
+
+def _candidate_title(signature: Signature) -> str:
+    normalized = signature.normalized_pattern
+    service = (signature.service or "").strip()
+    if "vpn" in normalized or service == "charon":
+        return "Recurring VPN tunnel instability"
+    if "failed password" in normalized or "auth" in normalized or service == "sshd":
+        return "Recurring SSH authentication failures"
+    if signature.event_family == "service_lifecycle" and service:
+        return f"Recurring {service} service instability"
+    return FAMILY_TITLES.get(signature.event_family, "Recurring operational pattern")
+
+
+def _candidate_summary(signature: Signature, count: int) -> str:
+    normalized = signature.normalized_pattern
+    service = (signature.service or "").strip()
+    if "vpn" in normalized or service == "charon":
+        return f"VPN tunnel instability was observed {count} times and may deserve more operator attention if it continues."
+    if "failed password" in normalized or "auth" in normalized or service == "sshd":
+        return f"SSH authentication failures were observed {count} times and are worth background attention if the pattern continues."
+    if service:
+        return f"A recurring {service} pattern was observed {count} times in the current event stream."
+    return f"A recurring operational pattern was observed {count} times in the current event stream."
+
+
+def signature_counts(signatures: Iterable[Signature]) -> Counter:
+    return Counter(sig.signature_key for sig in signatures)
+
+
+def build_recurrence_candidates(events: List[Event], signatures: List[Signature], *, threshold: int = 2) -> List[Candidate]:
+    counts = signature_counts(signatures)
+    candidates: List[Candidate] = []
+    for signature in signatures:
+        count = counts[signature.signature_key]
+        if count < threshold:
+            continue
+        recurrence = min(count / 10.0, 1.0)
+        recovery = 0.4
+        spread = 0.2
+        novelty = 0.3
+        impact = 0.5 if signature.event_family in {"failure", "auth"} else 0.2
+        precursor = 0.3
+        memory_weight = 0.4
+        candidate = score_candidate(
+            recurrence=recurrence,
+            recovery=recovery,
+            spread=spread,
+            novelty=novelty,
+            impact=impact,
+            precursor=precursor,
+            memory_weight=memory_weight,
+        )
+        candidate.title = _candidate_title(signature)
+        candidate.summary = _candidate_summary(signature, count)
+        candidate.source_signature_ids = [signature.signature_key]
+        candidate.source_event_ids = [str(i) for i, sig in enumerate(signatures) if sig.signature_key == signature.signature_key]
+        candidate.metadata = {"count": count, "service": signature.service}
+        candidates.append(candidate)
+    # dedupe by signature key/title
+    seen = set()
+    unique: List[Candidate] = []
+    for candidate in candidates:
+        key = tuple(candidate.source_signature_ids)
+        if key in seen:
+            continue
+        seen.add(key)
+        unique.append(candidate)
+    return unique
+
+
+def _attention_band(decision_band: str) -> str:
+    mapping = {
+        "ignore": "ignore_fast",
+        "watch": "background",
+        "review": "watch",
+        "urgent_human_review": "investigate",
+        "promote_to_incident_memory": "promote",
+    }
+    return mapping.get(decision_band, "watch")
+
+
+def digest_items(candidates: Iterable[Candidate]) -> List[dict]:
+    return [
+        {
+            "title": c.title,
+            "summary": c.summary,
+            "decision_band": c.decision_band,
+            "attention_band": _attention_band(c.decision_band),
+            "attention_score": c.score_total,
+            "score_total": c.score_total,
+            "score_breakdown": c.score_breakdown,
+            "metadata": c.metadata,
+        }
+        for c in candidates
+    ]

package/brainstem/scoring.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from .models import Candidate
+
+
+def decision_band(score_total: float) -> str:
+    if score_total >= 0.85:
+        return "promote_to_incident_memory"
+    if score_total >= 0.65:
+        return "urgent_human_review"
+    if score_total >= 0.45:
+        return "review"
+    if score_total >= 0.25:
+        return "watch"
+    return "ignore"
+
+
+def score_candidate(*, recurrence: float, recovery: float, spread: float, novelty: float, impact: float, precursor: float, memory_weight: float) -> Candidate:
+    score_breakdown = {
+        "recurrence": recurrence,
+        "recovery": recovery,
+        "spread": spread,
+        "novelty": novelty,
+        "impact": impact,
+        "precursor": precursor,
+        "memory_weight": memory_weight,
+    }
+    total = sum(score_breakdown.values()) / len(score_breakdown)
+    band = decision_band(total)
+    return Candidate(
+        candidate_type="recurrence",
+        title="Derived operational candidate",
+        summary="A weak-signal candidate derived from recurring events.",
+        score_total=round(total, 3),
+        score_breakdown=score_breakdown,
+        decision_band=band,
+        confidence=round(total, 3),
+    )