@simbimbo/brainstem 0.0.1 → 0.0.2

package/brainstem/models.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, List
+
+
+@dataclass
+class RawInputEnvelope:
+    tenant_id: str
+    source_type: str
+    timestamp: str
+    message_raw: str
+    host: str = ""
+    service: str = ""
+    severity: str = "info"
+    asset_id: str = ""
+    source_path: str = ""
+    facility: str = ""
+    structured_fields: Dict[str, Any] = field(default_factory=dict)
+    correlation_keys: Dict[str, Any] = field(default_factory=dict)
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class CanonicalEvent:
+    tenant_id: str
+    source_type: str
+    timestamp: str
+    message_raw: str
+    host: str = ""
+    service: str = ""
+    severity: str = "info"
+    asset_id: str = ""
+    source_path: str = ""
+    facility: str = ""
+    structured_fields: Dict[str, Any] = field(default_factory=dict)
+    correlation_keys: Dict[str, Any] = field(default_factory=dict)
+    message_normalized: str = ""
+    signature_input: str = ""
+    ingest_metadata: Dict[str, Any] = field(default_factory=dict)
+
+
+# Backward-compatible name used throughout existing code.
+Event = CanonicalEvent
+
+
+@dataclass
+class Signature:
+    signature_key: str
+    event_family: str
+    normalized_pattern: str
+    service: str = ""
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class Candidate:
+    candidate_type: str
+    title: str
+    summary: str
+    score_total: float
+    score_breakdown: Dict[str, float]
+    decision_band: str
+    source_signature_ids: List[str] = field(default_factory=list)
+    source_event_ids: List[str] = field(default_factory=list)
+    confidence: float = 0.0
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class IncidentMemory:
+    title: str
+    summary: str
+    incident_type: str
+    source_candidate_ids: List[str] = field(default_factory=list)
+    recurrence_count: int = 0
+    confidence: float = 0.0
+    metadata: Dict[str, Any] = field(default_factory=dict)

package/brainstem/recurrence.py

@@ -0,0 +1,112 @@
+from __future__ import annotations
+
+from collections import Counter
+from dataclasses import asdict
+from typing import Iterable, List
+
+from .models import Candidate, Event, Signature
+from .scoring import score_candidate
+
+
+FAMILY_TITLES = {
+    "failure": "Recurring failure pattern",
+    "auth": "Recurring authentication failure pattern",
+    "service_lifecycle": "Recurring service lifecycle instability",
+    "generic": "Recurring operational pattern",
+}
+
+
+def _candidate_title(signature: Signature) -> str:
+    normalized = signature.normalized_pattern
+    service = (signature.service or "").strip()
+    if "vpn" in normalized or service == "charon":
+        return "Recurring VPN tunnel instability"
+    if "failed password" in normalized or "auth" in normalized or service == "sshd":
+        return "Recurring SSH authentication failures"
+    if signature.event_family == "service_lifecycle" and service:
+        return f"Recurring {service} service instability"
+    return FAMILY_TITLES.get(signature.event_family, "Recurring operational pattern")
+
+
+def _candidate_summary(signature: Signature, count: int) -> str:
+    normalized = signature.normalized_pattern
+    service = (signature.service or "").strip()
+    if "vpn" in normalized or service == "charon":
+        return f"VPN tunnel instability was observed {count} times and may deserve more operator attention if it continues."
+    if "failed password" in normalized or "auth" in normalized or service == "sshd":
+        return f"SSH authentication failures were observed {count} times and are worth background attention if the pattern continues."
+    if service:
+        return f"A recurring {service} pattern was observed {count} times in the current event stream."
+    return f"A recurring operational pattern was observed {count} times in the current event stream."
+
+
+def signature_counts(signatures: Iterable[Signature]) -> Counter:
+    return Counter(sig.signature_key for sig in signatures)
+
+
+def build_recurrence_candidates(events: List[Event], signatures: List[Signature], *, threshold: int = 2) -> List[Candidate]:
+    counts = signature_counts(signatures)
+    candidates: List[Candidate] = []
+    for signature in signatures:
+        count = counts[signature.signature_key]
+        if count < threshold:
+            continue
+        recurrence = min(count / 10.0, 1.0)
+        recovery = 0.4
+        spread = 0.2
+        novelty = 0.3
+        impact = 0.5 if signature.event_family in {"failure", "auth"} else 0.2
+        precursor = 0.3
+        memory_weight = 0.4
+        candidate = score_candidate(
+            recurrence=recurrence,
+            recovery=recovery,
+            spread=spread,
+            novelty=novelty,
+            impact=impact,
+            precursor=precursor,
+            memory_weight=memory_weight,
+        )
+        candidate.title = _candidate_title(signature)
+        candidate.summary = _candidate_summary(signature, count)
+        candidate.source_signature_ids = [signature.signature_key]
+        candidate.source_event_ids = [str(i) for i, sig in enumerate(signatures) if sig.signature_key == signature.signature_key]
+        candidate.metadata = {"count": count, "service": signature.service}
+        candidates.append(candidate)
+    # dedupe by signature key/title
+    seen = set()
+    unique: List[Candidate] = []
+    for candidate in candidates:
+        key = tuple(candidate.source_signature_ids)
+        if key in seen:
+            continue
+        seen.add(key)
+        unique.append(candidate)
+    return unique
+
+
+def _attention_band(decision_band: str) -> str:
+    mapping = {
+        "ignore": "ignore_fast",
+        "watch": "background",
+        "review": "watch",
+        "urgent_human_review": "investigate",
+        "promote_to_incident_memory": "promote",
+    }
+    return mapping.get(decision_band, "watch")
+
+
+def digest_items(candidates: Iterable[Candidate]) -> List[dict]:
+    return [
+        {
+            "title": c.title,
+            "summary": c.summary,
+            "decision_band": c.decision_band,
+            "attention_band": _attention_band(c.decision_band),
+            "attention_score": c.score_total,
+            "score_total": c.score_total,
+            "score_breakdown": c.score_breakdown,
+            "metadata": c.metadata,
+        }
+        for c in candidates
+    ]

package/brainstem/scoring.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from .models import Candidate
+
+
+def decision_band(score_total: float) -> str:
+    if score_total >= 0.85:
+        return "promote_to_incident_memory"
+    if score_total >= 0.65:
+        return "urgent_human_review"
+    if score_total >= 0.45:
+        return "review"
+    if score_total >= 0.25:
+        return "watch"
+    return "ignore"
+
+
+def score_candidate(*, recurrence: float, recovery: float, spread: float, novelty: float, impact: float, precursor: float, memory_weight: float) -> Candidate:
+    score_breakdown = {
+        "recurrence": recurrence,
+        "recovery": recovery,
+        "spread": spread,
+        "novelty": novelty,
+        "impact": impact,
+        "precursor": precursor,
+        "memory_weight": memory_weight,
+    }
+    total = sum(score_breakdown.values()) / len(score_breakdown)
+    band = decision_band(total)
+    return Candidate(
+        candidate_type="recurrence",
+        title="Derived operational candidate",
+        summary="A weak-signal candidate derived from recurring events.",
+        score_total=round(total, 3),
+        score_breakdown=score_breakdown,
+        decision_band=band,
+        confidence=round(total, 3),
+    )

package/brainstem/storage.py

@@ -0,0 +1,182 @@
+from __future__ import annotations
+
+import json
+import sqlite3
+from dataclasses import asdict
+from pathlib import Path
+from typing import Iterable, List
+
+from .models import Candidate, Event, Signature
+
+
+def default_db_path() -> Path:
+    return Path('.brainstem-state') / 'brainstem.sqlite3'
+
+
+def connect(db_path: str | None = None) -> sqlite3.Connection:
+    path = Path(db_path) if db_path else default_db_path()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    conn = sqlite3.connect(path)
+    conn.row_factory = sqlite3.Row
+    return conn
+
+
+def init_db(db_path: str | None = None) -> None:
+    conn = connect(db_path)
+    try:
+        conn.executescript(
+            '''
+            CREATE TABLE IF NOT EXISTS events (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                tenant_id TEXT NOT NULL,
+                source_type TEXT NOT NULL,
+                timestamp TEXT NOT NULL,
+                host TEXT,
+                service TEXT,
+                severity TEXT,
+                asset_id TEXT,
+                source_path TEXT,
+                facility TEXT,
+                message_raw TEXT NOT NULL,
+                structured_fields_json TEXT NOT NULL,
+                correlation_keys_json TEXT NOT NULL
+            );
+
+            CREATE TABLE IF NOT EXISTS signatures (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                signature_key TEXT NOT NULL UNIQUE,
+                event_family TEXT NOT NULL,
+                normalized_pattern TEXT NOT NULL,
+                service TEXT,
+                metadata_json TEXT NOT NULL,
+                occurrence_count INTEGER NOT NULL DEFAULT 0
+            );
+
+            CREATE TABLE IF NOT EXISTS candidates (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                candidate_type TEXT NOT NULL,
+                title TEXT NOT NULL,
+                summary TEXT NOT NULL,
+                score_total REAL NOT NULL,
+                score_breakdown_json TEXT NOT NULL,
+                decision_band TEXT NOT NULL,
+                source_signature_ids_json TEXT NOT NULL,
+                source_event_ids_json TEXT NOT NULL,
+                confidence REAL NOT NULL,
+                metadata_json TEXT NOT NULL
+            );
+            '''
+        )
+        conn.commit()
+    finally:
+        conn.close()
+
+
+def store_events(events: Iterable[Event], db_path: str | None = None) -> int:
+    conn = connect(db_path)
+    count = 0
+    try:
+        for event in events:
+            conn.execute(
+                '''
+                INSERT INTO events (
+                    tenant_id, source_type, timestamp, host, service, severity,
+                    asset_id, source_path, facility, message_raw,
+                    structured_fields_json, correlation_keys_json
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                ''',
+                (
+                    event.tenant_id,
+                    event.source_type,
+                    event.timestamp,
+                    event.host,
+                    event.service,
+                    event.severity,
+                    event.asset_id,
+                    event.source_path,
+                    event.facility,
+                    event.message_raw,
+                    json.dumps(event.structured_fields, ensure_ascii=False),
+                    json.dumps(event.correlation_keys, ensure_ascii=False),
+                ),
+            )
+            count += 1
+        conn.commit()
+        return count
+    finally:
+        conn.close()
+
+
+def store_signatures(signatures: Iterable[Signature], db_path: str | None = None) -> int:
+    conn = connect(db_path)
+    count = 0
+    try:
+        for signature in signatures:
+            conn.execute(
+                '''
+                INSERT INTO signatures (
+                    signature_key, event_family, normalized_pattern, service, metadata_json, occurrence_count
+                ) VALUES (?, ?, ?, ?, ?, 1)
+                ON CONFLICT(signature_key) DO UPDATE SET
+                    occurrence_count = occurrence_count + 1,
+                    metadata_json = excluded.metadata_json,
+                    service = excluded.service
+                ''',
+                (
+                    signature.signature_key,
+                    signature.event_family,
+                    signature.normalized_pattern,
+                    signature.service,
+                    json.dumps(signature.metadata, ensure_ascii=False),
+                ),
+            )
+            count += 1
+        conn.commit()
+        return count
+    finally:
+        conn.close()
+
+
+def store_candidates(candidates: Iterable[Candidate], db_path: str | None = None) -> int:
+    conn = connect(db_path)
+    count = 0
+    try:
+        for candidate in candidates:
+            conn.execute(
+                '''
+                INSERT INTO candidates (
+                    candidate_type, title, summary, score_total, score_breakdown_json,
+                    decision_band, source_signature_ids_json, source_event_ids_json,
+                    confidence, metadata_json
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                ''',
+                (
+                    candidate.candidate_type,
+                    candidate.title,
+                    candidate.summary,
+                    candidate.score_total,
+                    json.dumps(candidate.score_breakdown, ensure_ascii=False),
+                    candidate.decision_band,
+                    json.dumps(candidate.source_signature_ids, ensure_ascii=False),
+                    json.dumps(candidate.source_event_ids, ensure_ascii=False),
+                    candidate.confidence,
+                    json.dumps(candidate.metadata, ensure_ascii=False),
+                ),
+            )
+            count += 1
+        conn.commit()
+        return count
+    finally:
+        conn.close()
+
+
+def list_candidates(db_path: str | None = None, limit: int = 20) -> List[sqlite3.Row]:
+    conn = connect(db_path)
+    try:
+        rows = conn.execute(
+            'SELECT * FROM candidates ORDER BY score_total DESC, id DESC LIMIT ?',
+            (max(1, limit),),
+        ).fetchall()
+        return rows
+    finally:
+        conn.close()