@simbimbo/brainstem 0.0.2 → 0.0.4

package/brainstem/ingest.py

@@ -1,63 +1,158 @@
 from __future__ import annotations
 
-from dataclasses import asdict
+import json
+from dataclasses import asdict, dataclass, field
 from datetime import datetime
 from pathlib import Path
-from typing import Iterable, List
+from typing import Callable, Iterable, List, Optional
 
 from .fingerprint import fingerprint_event, normalize_message
-from .models import CanonicalEvent, Event, RawInputEnvelope, Signature
+from .models import Candidate, CanonicalEvent, Event, RawInputEnvelope, Signature
+from .recurrence import build_recurrence_candidates
+from .source_drivers import parse_source_payloads
+from .storage import (
+    get_raw_envelopes_by_ids,
+    init_db,
+    RAW_ENVELOPE_STATUSES,
+    set_raw_envelope_status,
+    store_candidates,
+    store_events,
+    store_raw_envelopes,
+    store_signatures,
+)
 
 
-def parse_syslog_line(line: str, *, tenant_id: str, source_path: str = "") -> CanonicalEvent:
-    return canonicalize_raw_input_envelope(
-        parse_syslog_envelope(line, tenant_id=tenant_id, source_path=source_path)
+ErrorHandler = Callable[[Exception, str], None]
+
+
+@dataclass
+class IngestionItemResult:
+    index: int
+    status: str
+    tenant_id: str
+    source_type: str
+    source_id: str
+    source_name: str
+    raw_envelope_id: int | None
+    failure_reason: str | None = None
+
+
+@dataclass
+class IngestionResult:
+    raw_envelopes: List[RawInputEnvelope]
+    raw_envelope_ids: List[int]
+    events: List[CanonicalEvent]
+    signatures: List[Signature]
+    candidates: List[Candidate]
+    parse_failed: int
+    item_results: List[IngestionItemResult] = field(default_factory=list)
+
+
+@dataclass
+class ReplayAttempt:
+    raw_envelope_id: int
+    reason: str
+    status: str | None = None
+
+
+@dataclass
+class ReplayResult:
+    requested_raw_envelope_ids: List[int]
+    attempted_raw_envelope_ids: List[int]
+    skipped: List[ReplayAttempt]
+    events: List[CanonicalEvent]
+    signatures: List[Signature]
+    candidates: List[Candidate]
+    parse_failed: int
+
+
+def _ingestion_item_result_from_event(
+    index: int,
+    raw_event: RawInputEnvelope,
+    raw_envelope_id: int | None,
+    status: str,
+    *,
+    failure_reason: str | None = None,
+) -> IngestionItemResult:
+    return IngestionItemResult(
+        index=index,
+        status=status,
+        tenant_id=raw_event.tenant_id,
+        source_type=raw_event.source_type,
+        source_id=raw_event.source_id,
+        source_name=raw_event.source_name,
+        raw_envelope_id=raw_envelope_id,
+        failure_reason=failure_reason,
     )
 
 
+def parse_syslog_line(line: str, *, tenant_id: str, source_path: str = "") -> CanonicalEvent:
+    return canonicalize_raw_input_envelope(parse_syslog_envelope(line, tenant_id=tenant_id, source_path=source_path))
+
+
 def parse_syslog_envelope(line: str, *, tenant_id: str, source_path: str = "") -> RawInputEnvelope:
-    text = (line or "").rstrip("\n")
-    timestamp = datetime.utcnow().isoformat() + "Z"
-    host = ""
-    service = ""
-    message = text
-
-    parts = text.split()
-    if len(parts) >= 5:
-        host = parts[3]
-        rest = " ".join(parts[4:])
-        if ":" in rest:
-            svc, _, msg = rest.partition(":")
-            service = svc.strip()
-            message = msg.strip() or rest.strip()
-        else:
-            message = rest.strip()
+    return parse_source_payloads("syslog", [line], tenant_id=tenant_id, source_path=source_path)[0]
 
-    return RawInputEnvelope(
+
+def parse_file_line(line: str, *, tenant_id: str, source_path: str = "") -> RawInputEnvelope:
+    return parse_source_payloads("file", [line], tenant_id=tenant_id, source_path=source_path)[0]
+
+
+def parse_syslog_envelopes(lines: Iterable[str], *, tenant_id: str, source_path: str = "") -> List[RawInputEnvelope]:
+    parsed = [parse_syslog_envelope(line, tenant_id=tenant_id, source_path=source_path) for line in lines if str(line).strip()]
+    return parsed
+
+
+def parse_file_envelopes(lines: Iterable[str], *, tenant_id: str, source_path: str = "") -> List[RawInputEnvelope]:
+    return parse_source_payloads(
+        "file",
+        [line for line in lines if str(line).strip()],
         tenant_id=tenant_id,
-        source_type="syslog",
-        timestamp=timestamp,
-        message_raw=message,
-        host=host,
-        service=service,
         source_path=source_path,
-        metadata={"raw_line": text},
     )
 
 
-def parse_syslog_envelopes(lines: Iterable[str], *, tenant_id: str, source_path: str = "") -> List[RawInputEnvelope]:
-    return [parse_syslog_envelope(line, tenant_id=tenant_id, source_path=source_path) for line in lines if str(line).strip()]
+def _coerce_raw_envelope_id(value: object) -> int | None:
+    if isinstance(value, bool):
+        return None
+    if isinstance(value, int):
+        return value
+    if isinstance(value, str):
+        value = value.strip()
+        if not value.isdigit():
+            return None
+        return int(value)
+    return None
 
 
-def canonicalize_raw_input_envelope(raw: RawInputEnvelope) -> CanonicalEvent:
+def canonicalize_raw_input_envelope(
+    raw: RawInputEnvelope,
+    *,
+    raw_envelope_id: int | None = None,
+) -> CanonicalEvent:
+    parse_error = (raw.metadata or {}).get("parse_error")
+    if parse_error:
+        raise ValueError(f"parse_error: {parse_error}")
+
+    if not (raw.message_raw or "").strip():
+        raise ValueError("message_raw is empty and cannot be canonicalized")
+
+    resolved_raw_envelope_id = _coerce_raw_envelope_id(raw_envelope_id)
+    if resolved_raw_envelope_id is None:
+        resolved_raw_envelope_id = _coerce_raw_envelope_id(raw.metadata.get("raw_envelope_id"))
+
     message_normalized = normalize_message(raw.message_raw)
     metadata = dict(raw.metadata or {})
     metadata.setdefault("canonicalization_source", raw.source_type)
     metadata["raw_input_seen"] = True
+    if resolved_raw_envelope_id is not None:
+        metadata["raw_envelope_id"] = resolved_raw_envelope_id
+
     return CanonicalEvent(
         tenant_id=raw.tenant_id,
         source_type=raw.source_type,
         timestamp=raw.timestamp,
+        raw_envelope_id=resolved_raw_envelope_id,
         host=raw.host,
         service=raw.service,
         severity=raw.severity,
@@ -81,20 +176,310 @@ def canonicalize_raw_input_envelopes(events: Iterable[RawInputEnvelope]) -> List
     return [canonicalize_raw_input_envelope(raw_event) for raw_event in events]
 
 
+def _parse_json_map(value: str | None) -> dict:
+    if not value:
+        return {}
+    try:
+        parsed = json.loads(value)
+    except json.JSONDecodeError:
+        return {}
+    if isinstance(parsed, dict):
+        return parsed
+    return {}
+
+
+def _raw_envelope_from_row(row) -> RawInputEnvelope:
+    metadata = _parse_json_map(row["metadata_json"])
+    metadata["raw_envelope_id"] = int(row["id"])
+    return RawInputEnvelope(
+        tenant_id=row["tenant_id"],
+        source_type=row["source_type"],
+        source_id=row["source_id"] or "",
+        source_name=row["source_name"] or "",
+        timestamp=row["timestamp"],
+        host=row["host"] or "",
+        service=row["service"] or "",
+        severity=row["severity"] or "info",
+        asset_id=row["asset_id"] or "",
+        source_path=row["source_path"] or "",
+        message_raw=row["message_raw"] or "",
+        facility=row["facility"] or "",
+        structured_fields=_parse_json_map(row["structured_fields_json"]),
+        correlation_keys=_parse_json_map(row["correlation_keys_json"]),
+        metadata=metadata,
+    )
+
+
+def replay_raw_envelopes_by_ids(
+    raw_envelope_ids: Iterable[int | str | object],
+    *,
+    db_path: str,
+    threshold: int = 2,
+    on_event: Optional[Callable[[CanonicalEvent], None]] = None,
+    on_parse_error: Optional[ErrorHandler] = None,
+    force: bool = False,
+    allowed_statuses: Iterable[str] = ("received", "parse_failed"),
+) -> ReplayResult:
+    requested_raw_envelope_ids = list(dict.fromkeys([_coerce_raw_envelope_id(item) for item in raw_envelope_ids]))
+    requested_raw_envelope_ids = [item for item in requested_raw_envelope_ids if item is not None]
+
+    if not requested_raw_envelope_ids:
+        return ReplayResult(
+            requested_raw_envelope_ids=[],
+            attempted_raw_envelope_ids=[],
+            skipped=[],
+            events=[],
+            signatures=[],
+            candidates=[],
+            parse_failed=0,
+        )
+
+    allowed_status_set = set(allowed_statuses)
+    if any(status not in RAW_ENVELOPE_STATUSES for status in allowed_status_set):
+        raise ValueError(
+            "allowed_statuses must only include one of: "
+            + ", ".join(RAW_ENVELOPE_STATUSES)
+        )
+
+    raw_rows_by_id = {
+        int(row["id"]): row
+        for row in get_raw_envelopes_by_ids(requested_raw_envelope_ids, db_path=db_path)
+    }
+
+    replay_rows = []
+    skipped: List[ReplayAttempt] = []
+    for raw_envelope_id in requested_raw_envelope_ids:
+        row = raw_rows_by_id.get(raw_envelope_id)
+        if row is None:
+            skipped.append(
+                ReplayAttempt(
+                    raw_envelope_id=raw_envelope_id,
+                    reason="not_found",
+                    status="missing",
+                )
+            )
+            continue
+        if not force and row["canonicalization_status"] not in allowed_status_set:
+            skipped.append(
+                ReplayAttempt(
+                    raw_envelope_id=raw_envelope_id,
+                    reason="not_replayable",
+                    status=row["canonicalization_status"],
+                )
+            )
+            continue
+        replay_rows.append(row)
+
+    replay_envelopes = [_raw_envelope_from_row(row) for row in replay_rows]
+    raw_pipeline_result = run_ingest_pipeline(
+        replay_envelopes,
+        threshold=threshold,
+        db_path=db_path,
+        on_event=on_event,
+        on_parse_error=on_parse_error,
+        store_raw=False,
+    )
+
+    return ReplayResult(
+        requested_raw_envelope_ids=requested_raw_envelope_ids,
+        attempted_raw_envelope_ids=[row["id"] for row in replay_rows],
+        skipped=skipped,
+        events=raw_pipeline_result.events,
+        signatures=raw_pipeline_result.signatures,
+        candidates=raw_pipeline_result.candidates,
+        parse_failed=raw_pipeline_result.parse_failed,
+    )
+
+
+def run_ingest_pipeline(
+    raw_envelopes: Iterable[RawInputEnvelope],
+    *,
+    threshold: int = 2,
+    db_path: str | None = None,
+    store_raw: bool = True,
+    on_event: Optional[Callable[[CanonicalEvent], None]] = None,
+    on_parse_error: Optional[ErrorHandler] = None,
+) -> IngestionResult:
+    raw_envelopes_list = list(raw_envelopes)
+    raw_envelope_ids: List[int] = []
+    if db_path:
+        init_db(db_path)
+        if store_raw:
+            raw_envelope_ids = store_raw_envelopes(raw_envelopes_list, db_path)
+
+    canonical_events: List[CanonicalEvent] = []
+    parse_failed = 0
+    item_results: List[IngestionItemResult] = []
+    for idx, raw_event in enumerate(raw_envelopes_list):
+        raw_envelope_id = raw_envelope_ids[idx] if idx < len(raw_envelope_ids) else None
+        if raw_envelope_id is None:
+            raw_envelope_id = _coerce_raw_envelope_id(raw_event.metadata.get("raw_envelope_id"))
+        try:
+            canonical_event = canonicalize_raw_input_envelope(raw_event, raw_envelope_id=raw_envelope_id)
+        except Exception as exc:
+            parse_failed += 1
+            item_results.append(
+                _ingestion_item_result_from_event(
+                    idx,
+                    raw_event,
+                    raw_envelope_id=raw_envelope_id,
+                    status="parse_failed",
+                    failure_reason=str(exc),
+                )
+            )
+            if raw_envelope_id is not None:
+                set_raw_envelope_status(
+                    raw_envelope_id,
+                    "parse_failed",
+                    db_path=db_path,
+                    failure_reason=str(exc),
+                )
+            if on_parse_error is not None:
+                on_parse_error(exc, raw_event.metadata.get("raw_line", raw_event.message_raw))
+            continue
+
+        canonical_events.append(canonical_event)
+        item_results.append(
+            _ingestion_item_result_from_event(
+                idx,
+                raw_event,
+                raw_envelope_id=raw_envelope_id,
+                status="canonicalized",
+            )
+        )
+        if raw_envelope_id is not None:
+            set_raw_envelope_status(raw_envelope_id, "canonicalized", db_path=db_path)
+        if on_event is not None:
+            on_event(canonical_event)
+
+    if not canonical_events:
+        return IngestionResult(
+            raw_envelopes=raw_envelopes_list,
+            raw_envelope_ids=raw_envelope_ids,
+            events=[],
+            signatures=[],
+            candidates=[],
+            parse_failed=parse_failed,
+            item_results=item_results,
+        )
+
+    signatures = signatures_for_events(canonical_events)
+    candidates = build_recurrence_candidates(canonical_events, signatures, threshold=threshold)
+    if db_path:
+        store_events(canonical_events, db_path)
+        store_signatures(signatures, db_path)
+        store_candidates(candidates, db_path)
+
+    return IngestionResult(
+        raw_envelopes=raw_envelopes_list,
+        raw_envelope_ids=raw_envelope_ids,
+        events=canonical_events,
+        signatures=signatures,
+        candidates=candidates,
+        parse_failed=parse_failed,
+        item_results=item_results,
+    )
+
+
+def run_ingest_source_payload(
+    source_type: str,
+    payload: object,
+    *,
+    tenant_id: str,
+    source_path: str,
+    threshold: int = 2,
+    db_path: Optional[str] = None,
+    on_event: Optional[Callable[[CanonicalEvent], None]] = None,
+    on_parse_error: Optional[ErrorHandler] = None,
+) -> IngestionResult:
+    return run_ingest_pipeline(
+        parse_source_payloads(
+            source_type,
+            payload,
+            tenant_id=tenant_id,
+            source_path=source_path,
+            on_parse_error=on_parse_error,
+        ),
+        threshold=threshold,
+        db_path=db_path,
+        on_event=on_event,
+        on_parse_error=on_parse_error,
+    )
+
+
+def run_ingest_file_lines(
+    lines: Iterable[str],
+    *,
+    tenant_id: str,
+    source_path: str,
+    threshold: int = 2,
+    db_path: str | None = None,
+    on_event: Optional[Callable[[CanonicalEvent], None]] = None,
+    on_parse_error: Optional[ErrorHandler] = None,
+) -> IngestionResult:
+    return run_ingest_source_payload(
+        "file",
+        [line for line in lines if str(line).strip()],
+        tenant_id=tenant_id,
+        source_path=source_path,
+        threshold=threshold,
+        db_path=db_path,
+        on_event=on_event,
+        on_parse_error=on_parse_error,
+    )
+
+
 def ingest_syslog_lines(lines: Iterable[str], *, tenant_id: str, source_path: str = "") -> List[CanonicalEvent]:
     return canonicalize_raw_input_envelopes(
         parse_syslog_envelopes(lines, tenant_id=tenant_id, source_path=source_path),
     )
 
 
+def ingest_file_lines(lines: Iterable[str], *, tenant_id: str, source_path: str = "") -> List[CanonicalEvent]:
+    return canonicalize_raw_input_envelopes(
+        parse_file_envelopes(lines, tenant_id=tenant_id, source_path=source_path),
+    )
+
+
 def ingest_syslog_file(path: str, *, tenant_id: str) -> List[Event]:
     file_path = Path(path)
     lines = file_path.read_text(encoding="utf-8", errors="ignore").splitlines()
     return ingest_syslog_lines(lines, tenant_id=tenant_id, source_path=str(file_path))
 
 
+def run_ingest_file(
+    path: str,
+    *,
+    tenant_id: str,
+    threshold: int = 2,
+    db_path: Optional[str] = None,
+    on_event: Optional[Callable[[CanonicalEvent], None]] = None,
+    on_parse_error: Optional[ErrorHandler] = None,
+) -> IngestionResult:
+    file_path = Path(path)
+    lines = file_path.read_text(encoding="utf-8", errors="ignore").splitlines()
+    return run_ingest_file_lines(
+        lines,
+        tenant_id=tenant_id,
+        source_path=str(file_path),
+        threshold=threshold,
+        db_path=db_path,
+        on_event=on_event,
+        on_parse_error=on_parse_error,
+    )
+
+
 def signatures_for_events(events: Iterable[Event]) -> List[Signature]:
-    return [fingerprint_event(event) for event in events]
+    signatures = []
+    for event in events:
+        signature = fingerprint_event(event)
+        source_raw_envelope_id = getattr(event, "raw_envelope_id", None)
+        if source_raw_envelope_id is not None:
+            signature.metadata = dict(signature.metadata)
+            signature.metadata["source_raw_envelope_id"] = int(source_raw_envelope_id)
+            signature.metadata["source_raw_envelope_ids"] = [int(source_raw_envelope_id)]
+        signatures.append(signature)
+    return signatures
 
 
 def events_as_dicts(events: Iterable[Event]) -> List[dict]:

package/brainstem/interesting.py

@@ -1,9 +1,29 @@
 from __future__ import annotations
 
-from typing import Iterable, List, Dict, Any
+from typing import Any, Dict, Iterable, List
 
 from .models import Candidate
 
+ATTN_SIGNAL_LABELS = {
+    "recurrence": "recurrence",
+    "recovery": "recovery",
+    "spread": "spread",
+    "novelty": "novelty",
+    "impact": "human-impact",
+    "precursor": "precursor",
+    "memory_weight": "memory",
+}
+
+ATTN_SIGNAL_RATIONALES = {
+    "recurrence": "recurrence indicates repeated observation",
+    "recovery": "recovery suggests a pattern that often resets",
+    "spread": "spread shows similar behavior across context",
+    "novelty": "novelty indicates non-routine pattern shape",
+    "impact": "impact shows likely operator visibility value",
+    "precursor": "precursor score indicates early warning behavior",
+    "memory_weight": "memory_weight reflects previous recurrence context",
+}
+
 
 def _attention_band(decision_band: str) -> str:
     mapping = {
@@ -16,10 +36,42 @@ def _attention_band(decision_band: str) -> str:
     return mapping.get(decision_band, "watch")
 
 
+def _dominant_attention_signals(score_breakdown: Dict[str, float], *, limit: int = 3) -> List[Dict[str, Any]]:
+    ordered = sorted(score_breakdown.items(), key=lambda item: (float(item[1]), item[0]), reverse=True)
+    dominant = ordered[:limit]
+    return [
+        {
+            "signal": name,
+            "value": round(float(value), 3),
+            "label": ATTN_SIGNAL_LABELS.get(name, name.replace("_", "-")),
+            "rationale": ATTN_SIGNAL_RATIONALES.get(name, "prototype attention component"),
+        }
+        for name, value in dominant
+        if float(value) > 0
+    ]
+
+
+def _attention_explanation(candidate: Candidate) -> Dict[str, Any]:
+    attention_band = _attention_band(candidate.decision_band)
+    dominant_signals = _dominant_attention_signals(candidate.score_breakdown)
+    signal_summary = ", ".join(f"{item['label']}:{item['value']}" for item in dominant_signals)
+    if signal_summary:
+        summary = f"{attention_band} attention is driven by {signal_summary}."
+    else:
+        summary = f"{attention_band} attention is currently low; no dominant attention signals are available."
+    return {
+        "attention_band": attention_band,
+        "dominant_signals": dominant_signals,
+        "summary": summary,
+    }
+
+
 def _why_it_matters(candidate: Candidate) -> str:
     count = int((candidate.metadata or {}).get("count") or 0)
     service = str((candidate.metadata or {}).get("service") or "").strip()
     family = candidate.candidate_type.replace("_", " ")
+    attention_explanation = _attention_explanation(candidate)
+    top_signals = ", ".join(item["label"] for item in attention_explanation["dominant_signals"])
     pieces = []
     if count:
         pieces.append(f"observed {count} times")
@@ -37,6 +89,8 @@ def _why_it_matters(candidate: Candidate) -> str:
     else:
         level = "is low-attention noise"
     detail = ", ".join(pieces) if pieces else family
+    if top_signals:
+        detail = f"{detail} ({top_signals})" if detail else top_signals
     return f"{detail}; {level}."
 
 
@@ -55,6 +109,7 @@ def interesting_items(candidates: Iterable[Candidate], *, limit: int = 5) -> Lis
                 "score_total": candidate.score_total,
                 "confidence": candidate.confidence,
                 "why_it_matters": _why_it_matters(candidate),
+                "attention_explanation": _attention_explanation(candidate),
                 "signals": dict(candidate.score_breakdown),
                 "metadata": dict(candidate.metadata),
             }