@simbimbo/brainstem 0.0.2 → 0.0.4

package/tests/test_canonicalization.py

@@ -14,6 +14,7 @@ def test_canonicalization_from_raw_envelope_is_explicit() -> None:
     raw = parse_syslog_envelope("Mar 22 00:00:01 fw-01 charon: VPN tunnel dropped and recovered", tenant_id="client-a", source_path="/var/log/syslog")
     canonical = canonicalize_raw_input_envelope(raw)
     assert isinstance(canonical, CanonicalEvent)
+    assert canonical.raw_envelope_id is None
     assert canonical.message_normalized == "vpn tunnel dropped and recovered"
     assert canonical.signature_input == canonical.message_normalized
     assert canonical.ingest_metadata["canonicalization_source"] == "syslog"
@@ -21,6 +22,13 @@ def test_canonicalization_from_raw_envelope_is_explicit() -> None:
     assert canonical.ingest_metadata["raw_line"] == "Mar 22 00:00:01 fw-01 charon: VPN tunnel dropped and recovered"
 
 
+def test_canonicalization_accepts_and_carries_raw_envelope_id() -> None:
+    raw = parse_syslog_envelope("Mar 22 00:00:01 fw-01 charon: VPN tunnel dropped and recovered", tenant_id="client-a", source_path="/var/log/syslog")
+    canonical = canonicalize_raw_input_envelope(raw, raw_envelope_id=99)
+    assert canonical.raw_envelope_id == 99
+    assert canonical.ingest_metadata["raw_envelope_id"] == 99
+
+
 def test_parse_syslog_line_still_returns_canonical_event() -> None:
     event = parse_syslog_line("Mar 22 00:00:01 fw-01 charon: VPN tunnel dropped and recovered", tenant_id="client-a", source_path="/var/log/syslog")
     assert isinstance(event, CanonicalEvent)

package/tests/test_config.py

@@ -0,0 +1,24 @@
+from pathlib import Path
+
+import pytest
+
+from brainstem.config import get_runtime_config, resolve_default_db_path
+
+
+def test_runtime_config_exposes_shared_defaults() -> None:
+    cfg = get_runtime_config()
+    assert cfg.api_token_env_var == "BRAINSTEM_API_TOKEN"
+    assert cfg.listener.syslog_host == "127.0.0.1"
+    assert cfg.listener.syslog_port == 5514
+    assert cfg.listener.syslog_source_path == "/dev/udp"
+    assert cfg.defaults.ingest_threshold == 2
+    assert cfg.defaults.interesting_limit == 5
+    assert cfg.limits.replay_raw_max_ids == 32
+
+
+def test_resolve_default_db_path_uses_env_override(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
+    custom_db = tmp_path / "override.sqlite3"
+    monkeypatch.setenv("BRAINSTEM_DB_PATH", str(custom_db))
+    assert resolve_default_db_path() == str(custom_db)
+    monkeypatch.delenv("BRAINSTEM_DB_PATH", raising=False)
+    assert resolve_default_db_path() == ".brainstem-state/brainstem.sqlite3"

package/tests/test_file_ingest.py

@@ -0,0 +1,77 @@
+from pathlib import Path
+
+from brainstem.adapters import FileRawInputAdapter, get_raw_input_adapter, register_raw_input_adapter
+from brainstem.ingest import run_ingest_file, run_ingest_file_lines
+from brainstem.models import RawInputEnvelope
+from brainstem.storage import get_ingest_stats, list_recent_raw_envelopes
+
+
+def test_run_ingest_file_persists_accounting_and_generates_candidates(tmp_path: Path) -> None:
+    path = tmp_path / "file.log"
+    path.write_text("vpn tunnel dropped and recovered\nvpn tunnel dropped and recovered\n", encoding="utf-8")
+
+    db_path = tmp_path / "brainstem.sqlite3"
+    result = run_ingest_file(str(path), tenant_id="client-a", threshold=2, db_path=str(db_path))
+
+    assert len(result.events) == 2
+    assert result.parse_failed == 0
+    assert len(result.signatures) >= 1
+    assert len(result.candidates) >= 1
+
+    stats = get_ingest_stats(str(db_path))
+    assert stats["received"] == 2
+    assert stats["canonicalized"] == 2
+    assert stats["parse_failed"] == 0
+    assert stats["candidates_generated"] >= 1
+
+    rows = list_recent_raw_envelopes(str(db_path), limit=10)
+    assert rows[0]["source_type"] == "file"
+    assert rows[0]["source_path"] == str(path)
+    assert all(row["canonicalization_status"] == "canonicalized" for row in rows)
+
+
+def test_run_ingest_file_lines_records_parse_failures_without_breaking_pipeline(tmp_path: Path) -> None:
+    db_path = tmp_path / "brainstem.sqlite3"
+    original = get_raw_input_adapter("file")
+
+    class BrokenFileAdapter:
+        source_type = "file"
+        calls = 0
+
+        def parse_raw_input(self, payload, *, tenant_id: str, source_path: str = "") -> RawInputEnvelope:
+            BrokenFileAdapter.calls += 1
+            if payload == "BROKEN":
+                return RawInputEnvelope(
+                    tenant_id=tenant_id,
+                    source_type="file",
+                    timestamp="2026-03-22T00:00:00Z",
+                    message_raw="",
+                    source_path=source_path,
+                    metadata={"raw_line": payload},
+                )
+            return FileRawInputAdapter().parse_raw_input(payload, tenant_id=tenant_id, source_path=source_path)
+
+    register_raw_input_adapter(BrokenFileAdapter())
+    try:
+        result = run_ingest_file_lines(
+            ["vpn tunnel dropped and recovered", "BROKEN", "vpn tunnel dropped and recovered"],
+            tenant_id="client-a",
+            source_path="/tmp/test.log",
+            threshold=2,
+            db_path=str(db_path),
+        )
+    finally:
+        register_raw_input_adapter(original)
+
+    assert result.parse_failed == 1
+    assert len(result.events) == 2
+    assert result.raw_envelope_ids
+
+    stats = get_ingest_stats(str(db_path))
+    assert stats["received"] == 3
+    assert stats["canonicalized"] == 2
+    assert stats["parse_failed"] == 1
+    assert stats["candidates_generated"] >= 1
+
+    failed_rows = list_recent_raw_envelopes(str(db_path), status="parse_failed", limit=10)
+    assert len(failed_rows) == 1

package/tests/test_interesting.py

@@ -18,7 +18,17 @@ def test_interesting_items_returns_ranked_observations() -> None:
     assert items
     assert items[0]['title']
     assert 'why_it_matters' in items[0]
+    assert 'attention_explanation' in items[0]
     assert items[0]['signals']
+    explanation = items[0]["attention_explanation"]
+    assert explanation["attention_band"] in {"ignore_fast", "background", "watch", "investigate", "promote"}
+    assert isinstance(explanation["dominant_signals"], list)
+    assert explanation["dominant_signals"]
+    first_signal = explanation["dominant_signals"][0]
+    assert {"signal", "value", "label", "rationale"} <= set(first_signal.keys())
+    values = [signal["value"] for signal in explanation["dominant_signals"]]
+    assert values == sorted(values, reverse=True)
+    assert explanation["dominant_signals"][0]["label"] in items[0]["why_it_matters"]
 
 
 def test_interesting_items_respects_limit() -> None:

package/tests/test_listener.py

@@ -0,0 +1,253 @@
+import socket
+import threading
+from pathlib import Path
+
+import brainstem.listener
+from brainstem.listener import parse_syslog_datagram, run_ingest_syslog_datagram, run_udp_syslog_listener
+from brainstem.adapters import get_raw_input_adapter, register_raw_input_adapter
+from brainstem.models import RawInputEnvelope
+from brainstem.storage import get_ingest_stats, list_candidates, list_recent_raw_envelopes
+
+
+def test_parse_syslog_datagram_pushes_through_canonicalization() -> None:
+    payload = (
+        b"Mar 22 00:00:01 fw-01 charon: VPN tunnel dropped and recovered\n"
+        b"Mar 22 00:00:02 fw-01 charon: IPsec SA rekey succeeded\n"
+    )
+    events = parse_syslog_datagram(payload, tenant_id="client-a", source_path="/var/log/syslog")
+
+    assert len(events) == 2
+    assert events[0].tenant_id == "client-a"
+    assert events[0].source_path == "/var/log/syslog"
+    assert events[0].host == "fw-01"
+    assert events[0].service == "charon"
+    assert events[0].message_normalized == "vpn tunnel dropped and recovered"
+    assert events[0].ingest_metadata["canonicalization_source"] == "syslog"
+    assert events[1].ingest_metadata["raw_line"].startswith("Mar 22 00:00:02")
+
+
+class _FakeDatagramSocket:
+    def __init__(self, datagrams: list[bytes]) -> None:
+        self.datagrams = list(datagrams)
+        self.closed = False
+
+    def getsockname(self) -> tuple[str, int]:
+        return ("127.0.0.1", 5514)
+
+    def settimeout(self, timeout: float) -> None:
+        self.timeout = timeout
+
+    def recvfrom(self, size: int) -> tuple[bytes, tuple[str, int]]:
+        if not self.datagrams:
+            raise socket.timeout("no more datagrams")
+        return self.datagrams.pop(0), ("127.0.0.1", 514)
+
+    def close(self) -> None:
+        self.closed = True
+
+
+def test_udp_listener_processes_local_datagrams_without_external_network() -> None:
+    datagrams = [
+        b"Mar 22 00:00:01 fw-01 charon: VPN tunnel dropped and recovered",
+        b"Mar 22 00:00:02 fw-01 charon: IPsec SA rekey succeeded",
+    ]
+
+    fake_socket = _FakeDatagramSocket(datagrams)
+
+    received_events = []
+    parse_errors = []
+
+    processed = threading.Event()
+
+    def on_event(event) -> None:
+        received_events.append(event)
+        if len(received_events) >= 2:
+            processed.set()
+
+    def on_parse_error(exc, line: str) -> None:
+        parse_errors.append((exc, line))
+
+    thread = threading.Thread(
+        target=run_udp_syslog_listener,
+        kwargs={
+            "tenant_id": "client-a",
+            "host": "127.0.0.1",
+            "port": 5514,
+            "source_path": "/var/log/syslog",
+            "socket_obj": fake_socket,
+            "max_datagrams": 2,
+            "on_event": on_event,
+            "on_parse_error": on_parse_error,
+        },
+        daemon=True,
+    )
+    thread.start()
+
+    processed.wait(1.0)
+    thread.join(timeout=1.0)
+
+    assert len(received_events) == 2
+    assert [event.message_normalized for event in received_events] == [
+        "vpn tunnel dropped and recovered",
+        "ipsec sa rekey succeeded",
+    ]
+    assert parse_errors == []
+
+
+def test_udp_listener_can_persist_datagrams_and_update_accounting(tmp_path: Path) -> None:
+    datagrams = [
+        b"Mar 22 00:00:01 fw-01 charon: VPN tunnel dropped and recovered\n"
+        b"Mar 22 00:00:02 fw-01 charon: VPN tunnel dropped and recovered",
+    ]
+    db_path = tmp_path / "brainstem.sqlite3"
+
+    fake_socket = _FakeDatagramSocket(datagrams)
+    received_events = []
+    processed = threading.Event()
+
+    def on_event(event) -> None:
+        received_events.append(event)
+        if len(received_events) >= 2:
+            processed.set()
+
+    thread = threading.Thread(
+        target=run_udp_syslog_listener,
+        kwargs={
+            "tenant_id": "client-a",
+            "host": "127.0.0.1",
+            "port": 5514,
+            "source_path": "/var/log/syslog",
+            "socket_obj": fake_socket,
+            "max_datagrams": 1,
+            "on_event": on_event,
+            "db_path": str(db_path),
+            "threshold": 2,
+        },
+        daemon=True,
+    )
+    thread.start()
+
+    processed.wait(1.0)
+    thread.join(timeout=1.0)
+
+    assert len(received_events) == 2
+    stats = get_ingest_stats(str(db_path))
+    assert stats["received"] == 2
+    assert stats["canonicalized"] == 2
+    assert stats["parse_failed"] == 0
+    assert stats["candidates_generated"] >= 1
+
+    rows = list_recent_raw_envelopes(str(db_path), limit=10)
+    raw_ids = [row["id"] for row in rows[:2]]
+    assert len(raw_ids) == 2
+    assert all(event.raw_envelope_id is not None for event in received_events)
+    assert all(event.raw_envelope_id in raw_ids for event in received_events)
+    assert len(rows) == 2
+    assert rows[0]["canonicalization_status"] == "canonicalized"
+    assert rows[1]["canonicalization_status"] == "canonicalized"
+
+
+def test_udp_listener_marks_parse_failures_via_pipeline(tmp_path: Path) -> None:
+    datagrams = [
+        b"Mar 22 00:00:01 fw-01 charon: VPN tunnel dropped and recovered\n"
+        b"BROKEN syslog payload with empty parsed body",
+    ]
+    db_path = tmp_path / "brainstem.sqlite3"
+
+    original_adapter = get_raw_input_adapter("syslog")
+
+    class BrokenSyslogAdapter:
+        source_type = "syslog"
+
+        def parse_raw_input(self, payload, *, tenant_id: str, source_path: str = "") -> RawInputEnvelope:
+            if str(payload).startswith("BROKEN"):
+                return RawInputEnvelope(
+                    tenant_id=tenant_id,
+                    source_type="syslog",
+                    timestamp="2026-03-22T00:00:00Z",
+                    message_raw="",
+                    source_path=source_path,
+                    metadata={"raw_line": str(payload)},
+                )
+            return original_adapter.parse_raw_input(payload, tenant_id=tenant_id, source_path=source_path)
+
+    register_raw_input_adapter(BrokenSyslogAdapter())
+    try:
+        fake_socket = _FakeDatagramSocket(datagrams)
+        received_events = []
+        parse_errors = []
+        processed = threading.Event()
+
+        def on_event(event) -> None:
+            received_events.append(event)
+            if len(received_events) >= 1:
+                processed.set()
+
+        def on_parse_error(exc, line: str) -> None:
+            parse_errors.append((exc, line))
+
+        thread = threading.Thread(
+            target=run_udp_syslog_listener,
+            kwargs={
+                "tenant_id": "client-a",
+                "host": "127.0.0.1",
+                "port": 5514,
+                "source_path": "/var/log/syslog",
+                "socket_obj": fake_socket,
+                "max_datagrams": 1,
+                "on_event": on_event,
+                "on_parse_error": on_parse_error,
+                "db_path": str(db_path),
+                "threshold": 2,
+            },
+            daemon=True,
+        )
+        thread.start()
+
+        processed.wait(1.0)
+        thread.join(timeout=1.0)
+
+        assert len(received_events) == 1
+        assert parse_errors
+        assert parse_errors[0][1].startswith("BROKEN")
+
+        stats = get_ingest_stats(str(db_path))
+        assert stats["received"] == 2
+        assert stats["canonicalized"] == 1
+        assert stats["parse_failed"] == 1
+        assert stats["candidates_generated"] == 0
+
+        failed = list_recent_raw_envelopes(str(db_path), status="parse_failed", limit=10)
+        assert len(failed) == 1
+    finally:
+        register_raw_input_adapter(original_adapter)
+
+
+def test_listener_ingest_helper_persists_canonical_events_and_candidates(tmp_path: Path) -> None:
+    payload = (
+        b"Mar 22 00:00:01 fw-01 charon: VPN tunnel dropped and recovered\n"
+        b"Mar 22 00:00:02 fw-01 charon: VPN tunnel dropped and recovered\n"
+    )
+    db_path = tmp_path / "brainstem.sqlite3"
+
+    result = run_ingest_syslog_datagram(
+        payload,
+        tenant_id="client-a",
+        source_path="/var/log/syslog",
+        db_path=str(db_path),
+        threshold=2,
+    )
+
+    assert len(result.events) == 2
+    assert all(event.raw_envelope_id == expected for event, expected in zip(result.events, result.raw_envelope_ids))
+    assert len(result.signatures) >= 1
+    assert len(result.candidates) >= 1
+    assert result.parse_failed == 0
+
+    stats = get_ingest_stats(str(db_path))
+    assert stats["received"] == 2
+    assert stats["canonicalized"] == 2
+    assert stats["candidates_generated"] >= 1
+
+    candidates = list_candidates(str(db_path), limit=10)
+    assert candidates

package/tests/test_recurrence.py

@@ -13,4 +13,6 @@ def test_build_recurrence_candidates_emits_candidate_for_repeated_signature() ->
     candidates = build_recurrence_candidates(events, signatures, threshold=2)
     assert candidates
     digest = digest_items(candidates)
+    assert digest[0]["attention_explanation"]["dominant_signals"]
+    assert "summary" in digest[0]["attention_explanation"]
     assert digest[0]['decision_band'] in {'watch', 'review', 'urgent_human_review', 'promote_to_incident_memory'}

package/tests/test_source_drivers.py

@@ -0,0 +1,95 @@
+from pathlib import Path
+
+import pytest
+
+from brainstem.adapters import get_raw_input_adapter, register_raw_input_adapter
+from brainstem.ingest import run_ingest_source_payload
+from brainstem.models import RawInputEnvelope
+from brainstem.storage import get_ingest_stats
+
+
+@pytest.mark.parametrize(
+    "source_type,payload,source_path",
+    [
+        ("file", ["vpn tunnel dropped and recovered", "vpn tunnel dropped and recovered"], "/tmp/sample.log"),
+        ("syslog", b"Mar 22 00:00:01 fw-01 charon: VPN tunnel dropped and recovered\nMar 22 00:00:02 fw-01 charon: VPN tunnel dropped and recovered\n", "/var/log/syslog"),
+    ],
+)
+def test_run_ingest_source_payload_routes_file_and_syslog_payloads_through_same_pipeline(
+    tmp_path: Path,
+    source_type: str,
+    payload,
+    source_path: str,
+) -> None:
+    db_path = tmp_path / f"{source_type}.sqlite3"
+
+    result = run_ingest_source_payload(
+        source_type,
+        payload,
+        tenant_id="client-a",
+        source_path=source_path,
+        threshold=2,
+        db_path=str(db_path),
+    )
+
+    assert len(result.events) == 2
+    assert result.parse_failed == 0
+    assert len(result.signatures) >= 1
+    assert len(result.candidates) >= 1
+
+    stats = get_ingest_stats(str(db_path))
+    assert stats["received"] == 2
+    assert stats["canonicalized"] == 2
+    assert stats["parse_failed"] == 0
+    assert stats["candidates_generated"] >= 1
+
+
+@pytest.mark.parametrize("source_type,payload,source_path", [
+    ("file", ["vpn tunnel dropped and recovered", "BROKEN syslog payload with empty parsed body", "vpn tunnel dropped and recovered"], "/tmp/sample.log"),
+    ("syslog", b"Mar 22 00:00:01 fw-01 charon: VPN tunnel dropped and recovered\nBROKEN syslog payload with empty parsed body\nMar 22 00:00:02 fw-01 charon: VPN tunnel dropped and recovered\n", "/var/log/syslog"),
+])
+def test_run_ingest_source_payload_preserves_parse_failure_contract(
+    source_type: str,
+    payload,
+    source_path: str,
+    tmp_path: Path,
+) -> None:
+    original_adapter = get_raw_input_adapter(source_type)
+    db_path = tmp_path / f"{source_type}-failures.sqlite3"
+
+    class BrokenAdapter:
+        def __init__(self, source_type: str) -> None:
+            self.source_type = source_type
+
+        def parse_raw_input(self, payload, *, tenant_id: str, source_path: str = "") -> RawInputEnvelope:
+            if str(payload).startswith("BROKEN"):
+                return RawInputEnvelope(
+                    tenant_id=tenant_id,
+                    source_type=source_type,
+                    timestamp="2026-03-22T00:00:00Z",
+                    message_raw="",
+                    source_path=source_path,
+                    metadata={"raw_line": str(payload)},
+                )
+            return original_adapter.parse_raw_input(payload, tenant_id=tenant_id, source_path=source_path)
+
+    register_raw_input_adapter(BrokenAdapter(source_type))
+    try:
+        result = run_ingest_source_payload(
+            source_type,
+            payload,
+            tenant_id="client-a",
+            source_path=source_path,
+            threshold=2,
+            db_path=str(db_path),
+        )
+    finally:
+        register_raw_input_adapter(original_adapter)
+
+    assert result.parse_failed == 1
+    assert len(result.events) == 2
+
+    stats = get_ingest_stats(str(db_path))
+    assert stats["received"] == 3
+    assert stats["canonicalized"] == 2
+    assert stats["parse_failed"] == 1