@simbimbo/brainstem 0.0.3 → 0.0.4

package/brainstem/listener.py

@@ -0,0 +1,175 @@
+from __future__ import annotations
+
+import argparse
+import json
+import socket
+from dataclasses import asdict
+from typing import Any, Callable, List, Optional
+
+from .ingest import IngestionResult, run_ingest_source_payload
+from .ingest import canonicalize_raw_input_envelope
+from .config import get_runtime_config
+from .source_drivers import parse_source_payloads
+from .models import CanonicalEvent
+from .models import RawInputEnvelope
+
+EventHandler = Callable[[CanonicalEvent], None]
+ErrorHandler = Callable[[Exception, str], None]
+LISTENER_CONFIG = get_runtime_config().listener
+
+
+def parse_syslog_datagram(
+    payload: bytes,
+    *,
+    tenant_id: str,
+    source_path: str = LISTENER_CONFIG.syslog_source_path,
+    on_parse_error: Optional[ErrorHandler] = None,
+) -> List[CanonicalEvent]:
+    raw_events = parse_source_payloads(
+        "syslog",
+        payload,
+        tenant_id=tenant_id,
+        source_path=source_path,
+        on_parse_error=on_parse_error,
+    )
+    events: List[CanonicalEvent] = []
+    for raw_event in raw_events:
+        try:
+            events.append(canonicalize_raw_input_envelope(raw_event))
+        except Exception as exc:
+            if on_parse_error is None:
+                raise
+            on_parse_error(exc, raw_event.metadata.get("raw_line", raw_event.message_raw))
+    return events
+
+
+def parse_syslog_raw_datagram(
+    payload: bytes,
+    *,
+    tenant_id: str,
+    source_path: str = LISTENER_CONFIG.syslog_source_path,
+    on_parse_error: Optional[ErrorHandler] = None,
+) -> List[RawInputEnvelope]:
+    return parse_source_payloads(
+        "syslog",
+        payload,
+        tenant_id=tenant_id,
+        source_path=source_path,
+        on_parse_error=on_parse_error,
+    )
+
+
+def run_ingest_syslog_datagram(
+    payload: bytes,
+    *,
+    tenant_id: str,
+    source_path: str = LISTENER_CONFIG.syslog_source_path,
+    threshold: int = LISTENER_CONFIG.ingest_threshold,
+    db_path: Optional[str] = None,
+    on_event: Optional[EventHandler] = None,
+    on_parse_error: Optional[ErrorHandler] = None,
+) -> IngestionResult:
+    return run_ingest_source_payload(
+        "syslog",
+        payload,
+        tenant_id=tenant_id,
+        source_path=source_path,
+        threshold=threshold,
+        db_path=db_path,
+        on_event=on_event,
+        on_parse_error=on_parse_error,
+    )
+
+
+def run_udp_syslog_listener(
+    tenant_id: str,
+    *,
+    host: str = LISTENER_CONFIG.syslog_host,
+    port: int = LISTENER_CONFIG.syslog_port,
+    source_path: Optional[str] = None,
+    on_event: Optional[EventHandler] = None,
+    on_parse_error: Optional[ErrorHandler] = None,
+    db_path: Optional[str] = None,
+    threshold: int = LISTENER_CONFIG.ingest_threshold,
+    max_datagrams: Optional[int] = None,
+    socket_timeout: float = LISTENER_CONFIG.syslog_socket_timeout,
+    socket_obj: Optional[socket.socket] = None,
+) -> int:
+    sock = socket_obj
+    owns_socket = sock is None
+    if sock is None:
+        sock = socket.socket(family=socket.AF_INET, type=socket.SOCK_DGRAM)
+    try:
+        bound_host, bound_port = sock.getsockname()
+        if bound_port == 0:
+            sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+            sock.bind((host, port))
+            bound_host, bound_port = sock.getsockname()
+        if source_path is None:
+            source_path = f"udp://{bound_host}:{bound_port}"
+
+        received_events = 0
+        sock.settimeout(socket_timeout)
+
+        processed_datagrams = 0
+        while True:
+            if max_datagrams is not None and processed_datagrams >= max_datagrams:
+                break
+            try:
+                payload, _ = sock.recvfrom(65535)
+            except socket.timeout:
+                continue
+            processed_datagrams += 1
+            if db_path is None:
+                events = parse_syslog_datagram(
+                    payload,
+                    tenant_id=tenant_id,
+                    source_path=source_path,
+                    on_parse_error=on_parse_error,
+                )
+            else:
+                result = run_ingest_syslog_datagram(
+                    payload,
+                    tenant_id=tenant_id,
+                    source_path=source_path,
+                    threshold=threshold,
+                    db_path=db_path,
+                    on_event=None,
+                    on_parse_error=on_parse_error,
+                )
+                events = result.events
+            for event in events:
+                received_events += 1
+                if on_event is not None:
+                    on_event(event)
+        return received_events
+    finally:
+        if owns_socket:
+            sock.close()
+
+
+def main(argv: Optional[list[str]] = None) -> int:
+    parser = argparse.ArgumentParser(description="Run the brAInstem UDP syslog listener.")
+    parser.add_argument("--host", default=LISTENER_CONFIG.syslog_host, help="Listen host")
+    parser.add_argument("--port", type=int, default=LISTENER_CONFIG.syslog_port, help="UDP port")
+    parser.add_argument("--tenant", default="demo-tenant", help="Tenant identifier")
+    parser.add_argument("--source-path", default="", help="Override source_path metadata on parsed envelopes")
+    args = parser.parse_args(argv)
+
+    source_path = args.source_path or f"udp://{args.host}:{args.port}"
+
+    def _emit(event: CanonicalEvent) -> None:
+        print(json.dumps(asdict(event), default=str))
+
+    run_udp_syslog_listener(
+        args.tenant,
+        host=args.host,
+        port=args.port,
+        source_path=source_path,
+        on_event=_emit,
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/brainstem/models.py

@@ -29,6 +29,7 @@ class CanonicalEvent:
     source_type: str
     timestamp: str
     message_raw: str
+    raw_envelope_id: int | None = None
     host: str = ""
     service: str = ""
     severity: str = "info"

package/brainstem/recurrence.py

@@ -5,6 +5,7 @@ from dataclasses import asdict
 from typing import Iterable, List
 
 from .models import Candidate, Event, Signature
+from .interesting import _attention_explanation
 from .scoring import score_candidate
 
 
@@ -44,9 +45,40 @@ def signature_counts(signatures: Iterable[Signature]) -> Counter:
     return Counter(sig.signature_key for sig in signatures)
 
 
+def _coerce_raw_envelope_id(value: object) -> int | None:
+    if isinstance(value, bool):
+        return None
+    if isinstance(value, int):
+        return value
+    if isinstance(value, str):
+        value = value.strip()
+        if not value.isdigit():
+            return None
+        return int(value)
+    return None
+
+
+def _signature_lineage_index(
+    events: List[Event],
+    signatures: List[Signature],
+) -> dict[str, list[int]]:
+    per_signature_raw_ids: dict[str, list[int]] = {}
+    seen = {}
+    for event, signature in zip(events, signatures):
+        raw_envelope_id = _coerce_raw_envelope_id(getattr(event, "raw_envelope_id", None))
+        if raw_envelope_id is None:
+            continue
+        seen.setdefault(signature.signature_key, set()).add(raw_envelope_id)
+
+    for key, values in seen.items():
+        per_signature_raw_ids[key] = sorted(values)
+    return per_signature_raw_ids
+
+
 def build_recurrence_candidates(events: List[Event], signatures: List[Signature], *, threshold: int = 2) -> List[Candidate]:
     counts = signature_counts(signatures)
     candidates: List[Candidate] = []
+    signature_raw_envelope_ids = _signature_lineage_index(events, signatures)
     for signature in signatures:
         count = counts[signature.signature_key]
         if count < threshold:
@@ -71,7 +103,11 @@ def build_recurrence_candidates(events: List[Event], signatures: List[Signature]
         candidate.summary = _candidate_summary(signature, count)
         candidate.source_signature_ids = [signature.signature_key]
         candidate.source_event_ids = [str(i) for i, sig in enumerate(signatures) if sig.signature_key == signature.signature_key]
-        candidate.metadata = {"count": count, "service": signature.service}
+        candidate.metadata = {
+            "count": count,
+            "service": signature.service,
+            "source_raw_envelope_ids": signature_raw_envelope_ids.get(signature.signature_key, []),
+        }
         candidates.append(candidate)
     # dedupe by signature key/title
     seen = set()
@@ -105,6 +141,7 @@ def digest_items(candidates: Iterable[Candidate]) -> List[dict]:
             "attention_band": _attention_band(c.decision_band),
             "attention_score": c.score_total,
             "score_total": c.score_total,
+            "attention_explanation": _attention_explanation(c),
             "score_breakdown": c.score_breakdown,
             "metadata": c.metadata,
         }

package/brainstem/source_drivers.py

@@ -0,0 +1,150 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Callable, Iterable, List, Protocol, runtime_checkable
+
+from .adapters import get_raw_input_adapter
+from .models import RawInputEnvelope
+
+ErrorHandler = Callable[[Exception, str], None]
+
+
+def iter_syslog_lines(payload: bytes | str) -> list[str]:
+    text = payload.decode("utf-8", errors="replace") if isinstance(payload, (bytes, bytearray)) else str(payload)
+    return [line.rstrip("\r") for line in text.splitlines() if line.strip()]
+
+
+@runtime_checkable
+class SourceDriver(Protocol):
+    """Contract for a small source-to-raw-envelope driver."""
+
+    source_type: str
+
+    def parse_payload(
+        self,
+        payload: Any,
+        *,
+        tenant_id: str,
+        source_path: str = "",
+        on_parse_error: ErrorHandler | None = None,
+    ) -> List[RawInputEnvelope]:
+        """Parse one input payload into one or more raw envelopes."""
+
+
+_SOURCE_DRIVER_REGISTRY: dict[str, SourceDriver] = {}
+
+
+def register_source_driver(driver: SourceDriver) -> None:
+    _SOURCE_DRIVER_REGISTRY[driver.source_type] = driver
+
+
+def get_source_driver(source_type: str) -> SourceDriver:
+    try:
+        return _SOURCE_DRIVER_REGISTRY[source_type]
+    except KeyError as exc:
+        raise ValueError(f"unsupported source_type: {source_type}") from exc
+
+
+def list_source_driver_types() -> list[str]:
+    return sorted(_SOURCE_DRIVER_REGISTRY.keys())
+
+
+def _iter_payload_items(payload: Any) -> Iterable[Any]:
+    if payload is None or isinstance(payload, (bytes, bytearray, str)):
+        yield payload
+        return
+    try:
+        iter(payload)
+    except TypeError:
+        yield payload
+        return
+    yield from payload
+
+
+def parse_source_payloads(
+    source_type: str,
+    payloads: Any,
+    *,
+    tenant_id: str,
+    source_path: str = "",
+    on_parse_error: ErrorHandler | None = None,
+) -> List[RawInputEnvelope]:
+    driver = get_source_driver(source_type)
+    raw_envelopes: List[RawInputEnvelope] = []
+    for payload in _iter_payload_items(payloads):
+        raw_envelopes.extend(
+            driver.parse_payload(
+                payload,
+                tenant_id=tenant_id,
+                source_path=source_path,
+                on_parse_error=on_parse_error,
+            )
+        )
+    return raw_envelopes
+
+
+@dataclass(frozen=True)
+class FileSourceDriver:
+    """Driver for file-line payloads."""
+
+    source_type: str = "file"
+
+    def parse_payload(
+        self,
+        payload: Any,
+        *,
+        tenant_id: str,
+        source_path: str = "",
+        on_parse_error: ErrorHandler | None = None,
+    ) -> List[RawInputEnvelope]:
+        adapter = get_raw_input_adapter(self.source_type)
+        text = "" if payload is None else str(payload)
+        try:
+            return [adapter.parse_raw_input(text, tenant_id=tenant_id, source_path=source_path)]
+        except Exception as exc:
+            if on_parse_error is None:
+                raise
+            on_parse_error(exc, text)
+            return []
+
+
+@dataclass(frozen=True)
+class SyslogSourceDriver:
+    """Driver for syslog payload chunks and line-oriented payloads."""
+
+    source_type: str = "syslog"
+
+    def parse_payload(
+        self,
+        payload: Any,
+        *,
+        tenant_id: str,
+        source_path: str = "",
+        on_parse_error: ErrorHandler | None = None,
+    ) -> List[RawInputEnvelope]:
+        adapter = get_raw_input_adapter(self.source_type)
+        raw_envelopes: List[RawInputEnvelope] = []
+
+        if isinstance(payload, (bytes, bytearray)):
+            payload_lines = iter_syslog_lines(payload)
+            for line in payload_lines:
+                try:
+                    raw_envelopes.append(adapter.parse_raw_input(line, tenant_id=tenant_id, source_path=source_path))
+                except Exception as exc:
+                    if on_parse_error is None:
+                        raise
+                    on_parse_error(exc, line)
+            return raw_envelopes
+
+        payload_text = "" if payload is None else str(payload)
+        try:
+            raw_envelopes.append(adapter.parse_raw_input(payload_text, tenant_id=tenant_id, source_path=source_path))
+        except Exception as exc:
+            if on_parse_error is None:
+                raise
+            on_parse_error(exc, payload_text)
+        return raw_envelopes
+
+
+register_source_driver(FileSourceDriver())
+register_source_driver(SyslogSourceDriver())