@simbimbo/brainstem 0.0.1 → 0.0.3

package/tests/test_api.py

@@ -0,0 +1,319 @@
+from pathlib import Path
+
+from fastapi.testclient import TestClient
+
+from brainstem.api import app
+from brainstem.models import RawInputEnvelope
+from brainstem.storage import (
+    init_db,
+    set_raw_envelope_status,
+    store_raw_envelopes,
+)
+
+
+def test_ingest_event_endpoint_round_trip(tmp_path: Path) -> None:
+    client = TestClient(app)
+    db_path = tmp_path / "brainstem.sqlite3"
+    response = client.post(
+        "/ingest/event?threshold=2&db_path=" + str(db_path),
+        json={
+            "tenant_id": "client-a",
+            "source_type": "syslog",
+            "message_raw": "VPN tunnel dropped and recovered",
+            "host": "fw-01",
+            "service": "charon",
+            "severity": "info",
+        },
+    )
+    assert response.status_code == 200
+    payload = response.json()
+    assert payload["ok"] is True
+    assert payload["event_count"] == 1
+    assert payload["signature_count"] == 1
+
+
+def test_ingest_batch_and_interesting(tmp_path: Path) -> None:
+    client = TestClient(app)
+    db_path = tmp_path / "brainstem.sqlite3"
+    response = client.post(
+        "/ingest/batch",
+        json={
+            "threshold": 2,
+            "db_path": str(db_path),
+            "events": [
+                {
+                    "tenant_id": "client-a",
+                    "source_type": "syslog",
+                    "message_raw": "Failed password for admin from 10.1.2.3",
+                    "host": "fw-01",
+                    "service": "sshd",
+                },
+                {
+                    "tenant_id": "client-a",
+                    "source_type": "syslog",
+                    "message_raw": "Failed password for admin from 10.1.2.3",
+                    "host": "fw-01",
+                    "service": "sshd",
+                },
+            ],
+        },
+    )
+    assert response.status_code == 200
+    payload = response.json()
+    assert payload["ok"] is True
+    assert payload["event_count"] == 2
+    assert payload["candidate_count"] >= 1
+
+    interesting = client.get(f"/interesting?db_path={db_path}&limit=10")
+    assert interesting.status_code == 200
+    interesting_payload = interesting.json()
+    assert interesting_payload["ok"] is True
+    assert interesting_payload["items"]
+
+
+def test_stats_after_successful_and_failed_ingest(tmp_path: Path) -> None:
+    client = TestClient(app)
+    db_path = tmp_path / "brainstem.sqlite3"
+    batch_response = client.post(
+        "/ingest/batch",
+        json={
+            "threshold": 2,
+            "db_path": str(db_path),
+            "events": [
+                {
+                    "tenant_id": "client-a",
+                    "source_type": "syslog",
+                    "message_raw": "Failed password for admin from 10.1.2.3",
+                    "host": "fw-01",
+                    "service": "sshd",
+                },
+                {
+                    "tenant_id": "client-a",
+                    "source_type": "syslog",
+                    "message_raw": "Failed password for admin from 10.1.2.3",
+                    "host": "fw-01",
+                    "service": "sshd",
+                },
+                {
+                    "tenant_id": "client-a",
+                    "source_type": "syslog",
+                    "message_raw": "",
+                    "host": "fw-01",
+                    "service": "sshd",
+                },
+            ],
+        },
+    )
+    assert batch_response.status_code == 200
+    batch_payload = batch_response.json()
+    assert batch_payload["ok"] is True
+    assert batch_payload["event_count"] == 2
+    assert batch_payload["parse_failed"] == 1
+
+    stats = client.get(f"/stats?db_path={db_path}")
+    assert stats.status_code == 200
+    stats_payload = stats.json()
+    assert stats_payload["ok"] is True
+    assert stats_payload["received"] == 3
+    assert stats_payload["canonicalized"] == 2
+    assert stats_payload["parse_failed"] == 1
+    assert stats_payload["candidates_generated"] >= 1
+
+
+def test_healthz_is_ready() -> None:
+    client = TestClient(app)
+    response = client.get("/healthz")
+    assert response.status_code == 200
+    assert response.json()["ok"] is True
+
+
+def test_failures_endpoint_lists_recent_parse_failures(tmp_path: Path) -> None:
+    client = TestClient(app)
+    db_path = tmp_path / "brainstem.sqlite3"
+    client.post(
+        "/ingest/batch",
+        json={
+            "threshold": 2,
+            "db_path": str(db_path),
+            "events": [
+                {
+                    "tenant_id": "client-a",
+                    "source_type": "syslog",
+                    "message_raw": "",
+                    "host": "fw-01",
+                    "service": "sshd",
+                },
+                {
+                    "tenant_id": "client-a",
+                    "source_type": "syslog",
+                    "message_raw": "VPN tunnel dropped and recovered",
+                    "host": "fw-01",
+                    "service": "charon",
+                },
+            ],
+        },
+    )
+
+    response = client.get(f"/failures?db_path={db_path}&limit=10")
+    assert response.status_code == 200
+    payload = response.json()
+    assert payload["ok"] is True
+    assert payload["count"] == 1
+    assert payload["items"][0]["canonicalization_status"] == "parse_failed"
+
+
+def test_failures_endpoint_filters_by_status_and_fetches_single_record(tmp_path: Path) -> None:
+    client = TestClient(app)
+    db_path = tmp_path / "brainstem.sqlite3"
+    init_db(str(db_path))
+    raw_ids = store_raw_envelopes(
+        [
+            RawInputEnvelope(
+                tenant_id="client-a",
+                source_type="syslog",
+                timestamp="2026-03-22T00:00:01Z",
+                message_raw="first",
+                host="fw-01",
+                service="sshd",
+            ),
+            RawInputEnvelope(
+                tenant_id="client-a",
+                source_type="syslog",
+                timestamp="2026-03-22T00:00:02Z",
+                message_raw="second",
+                host="fw-01",
+                service="sshd",
+            ),
+        ],
+        db_path=str(db_path),
+    )
+    set_raw_envelope_status(raw_ids[0], "parse_failed", db_path=str(db_path), failure_reason="bad parse")
+    set_raw_envelope_status(raw_ids[1], "unsupported", db_path=str(db_path), failure_reason="unsupported source")
+
+    failed_only = client.get(f"/failures?db_path={db_path}&status=parse_failed&limit=10")
+    assert failed_only.status_code == 200
+    failed_payload = failed_only.json()
+    assert failed_payload["count"] == 1
+    assert failed_payload["items"][0]["id"] == raw_ids[0]
+
+    unsupported = client.get(f"/failures?db_path={db_path}&status=unsupported&limit=10")
+    assert unsupported.status_code == 200
+    unsupported_payload = unsupported.json()
+    assert unsupported_payload["count"] == 1
+    assert unsupported_payload["items"][0]["id"] == raw_ids[1]
+
+    single = client.get(f"/failures/{raw_ids[1]}?db_path={db_path}")
+    assert single.status_code == 200
+    single_payload = single.json()
+    assert single_payload["ok"] is True
+    assert single_payload["item"]["id"] == raw_ids[1]
+    assert single_payload["item"]["failure_reason"] == "unsupported source"
+
+    invalid = client.get(f"/failures?db_path={db_path}&status=bogus")
+    assert invalid.status_code == 422
+
+
+def test_sources_endpoint_summarizes_ingest_dimensions(tmp_path: Path) -> None:
+    client = TestClient(app)
+    db_path = tmp_path / "brainstem.sqlite3"
+    batch_response = client.post(
+        "/ingest/batch",
+        json={
+            "threshold": 1,
+            "db_path": str(db_path),
+            "events": [
+                {
+                    "tenant_id": "client-a",
+                    "source_type": "syslog",
+                    "source_id": "fw-01",
+                    "source_name": "edge-fw-01",
+                    "source_path": "/var/log/syslog",
+                    "message_raw": "Failed password for admin from 10.1.2.3",
+                    "host": "fw-01",
+                    "service": "sshd",
+                    "severity": "info",
+                },
+                {
+                    "tenant_id": "client-a",
+                    "source_type": "syslog",
+                    "source_id": "fw-01",
+                    "source_name": "edge-fw-01",
+                    "source_path": "/var/log/syslog",
+                    "message_raw": "Failed password for admin from 10.1.2.3",
+                    "host": "fw-01",
+                    "service": "sshd",
+                    "severity": "info",
+                },
+                {
+                    "tenant_id": "client-a",
+                    "source_type": "logicmonitor",
+                    "source_id": "lm-01",
+                    "source_name": "edge-lm-01",
+                    "source_path": "/alerts",
+                    "message_raw": "Disk space low",
+                    "host": "lm-01",
+                    "service": "logicmonitor",
+                    "severity": "warning",
+                },
+            ],
+        },
+    )
+    assert batch_response.status_code == 200
+
+    response = client.get(f"/sources?db_path={db_path}&limit=10")
+    assert response.status_code == 200
+    payload = response.json()
+    assert payload["ok"] is True
+    assert payload["items"]["source_type"] == [
+        {"value": "syslog", "count": 2},
+        {"value": "logicmonitor", "count": 1},
+    ]
+    assert dict((entry["value"], entry["count"]) for entry in payload["items"]["source_name"]) == {
+        "edge-fw-01": 2,
+        "edge-lm-01": 1,
+    }
+
+
+def test_ingest_recent_endpoint_returns_recent_intake_and_allows_status_filter(tmp_path: Path) -> None:
+    client = TestClient(app)
+    db_path = tmp_path / "brainstem.sqlite3"
+    client.post(
+        "/ingest/batch",
+        json={
+            "threshold": 1,
+            "db_path": str(db_path),
+            "events": [
+                {
+                    "tenant_id": "client-a",
+                    "source_type": "syslog",
+                    "source_id": "fw-01",
+                    "source_name": "edge-fw-01",
+                    "message_raw": "service restarted",
+                    "host": "fw-01",
+                    "service": "systemd",
+                },
+                {
+                    "tenant_id": "client-a",
+                    "source_type": "syslog",
+                    "source_id": "fw-01",
+                    "source_name": "edge-fw-01",
+                    "message_raw": "",
+                    "host": "fw-01",
+                    "service": "systemd",
+                },
+            ],
+        },
+    )
+
+    response = client.get(f"/ingest/recent?db_path={db_path}&limit=10")
+    assert response.status_code == 200
+    payload = response.json()
+    assert payload["ok"] is True
+    assert payload["count"] == 2
+    assert len({item["canonicalization_status"] for item in payload["items"]}) == 2
+
+    failed = client.get(f"/ingest/recent?db_path={db_path}&status=parse_failed&limit=10")
+    assert failed.status_code == 200
+    failed_payload = failed.json()
+    assert failed_payload["count"] == 1
+    assert failed_payload["items"][0]["canonicalization_status"] == "parse_failed"

package/tests/test_canonicalization.py

@@ -0,0 +1,28 @@
+from brainstem.ingest import canonicalize_raw_input_envelope, parse_syslog_envelope, parse_syslog_line
+from brainstem.models import CanonicalEvent, RawInputEnvelope
+
+
+def test_parse_syslog_envelope_preserves_raw_fields() -> None:
+    raw = parse_syslog_envelope("Mar 22 00:00:01 fw-01 charon: VPN tunnel dropped and recovered", tenant_id="client-a", source_path="/var/log/syslog")
+    assert isinstance(raw, RawInputEnvelope)
+    assert raw.host == "fw-01"
+    assert raw.service == "charon"
+    assert raw.metadata["raw_line"]
+
+
+def test_canonicalization_from_raw_envelope_is_explicit() -> None:
+    raw = parse_syslog_envelope("Mar 22 00:00:01 fw-01 charon: VPN tunnel dropped and recovered", tenant_id="client-a", source_path="/var/log/syslog")
+    canonical = canonicalize_raw_input_envelope(raw)
+    assert isinstance(canonical, CanonicalEvent)
+    assert canonical.message_normalized == "vpn tunnel dropped and recovered"
+    assert canonical.signature_input == canonical.message_normalized
+    assert canonical.ingest_metadata["canonicalization_source"] == "syslog"
+    assert canonical.ingest_metadata["raw_input_seen"] is True
+    assert canonical.ingest_metadata["raw_line"] == "Mar 22 00:00:01 fw-01 charon: VPN tunnel dropped and recovered"
+
+
+def test_parse_syslog_line_still_returns_canonical_event() -> None:
+    event = parse_syslog_line("Mar 22 00:00:01 fw-01 charon: VPN tunnel dropped and recovered", tenant_id="client-a", source_path="/var/log/syslog")
+    assert isinstance(event, CanonicalEvent)
+    assert event.message_normalized
+    assert event.ingest_metadata["canonicalization_source"] == "syslog"

package/tests/test_demo.py

@@ -0,0 +1,25 @@
+from pathlib import Path
+
+from brainstem.demo import run_syslog_demo
+
+
+FIXTURE = Path(__file__).parent / "fixtures" / "sample_syslog.log"
+
+
+def test_run_syslog_demo_emits_digest_for_recurrence(tmp_path: Path) -> None:
+    payload = run_syslog_demo(str(FIXTURE), tenant_id="client-a", threshold=2, db_path=str(tmp_path / 'brainstem.sqlite3'))
+    assert payload["ok"] is True
+    assert payload["event_count"] == 6
+    assert payload["signature_count"] >= 2
+    assert payload["candidate_count"] >= 1
+    assert payload["digest"]
+    assert payload["interesting_items"]
+    assert payload["top_candidate"] is not None
+
+
+def test_run_syslog_demo_threshold_can_suppress_candidates() -> None:
+    payload = run_syslog_demo(str(FIXTURE), tenant_id="client-a", threshold=10)
+    assert payload["ok"] is True
+    assert payload["candidate_count"] == 0
+    assert payload["digest"] == []
+    assert payload["top_candidate"] is None

package/tests/test_fingerprint.py

@@ -0,0 +1,22 @@
+from brainstem.fingerprint import normalize_message, fingerprint_event
+from brainstem.models import Event
+
+
+def test_normalize_message_replaces_ips_and_numbers() -> None:
+    text = normalize_message("Auth failure from 10.1.2.3 after 17 retries")
+    assert "<ip>" in text
+    assert "<n>" in text
+
+
+def test_fingerprint_event_builds_signature() -> None:
+    event = Event(
+        tenant_id="client-a",
+        source_type="syslog",
+        timestamp="2026-03-22T00:00:00Z",
+        host="fw-01",
+        service="sshd",
+        message_raw="Auth failure from 10.1.2.3 after 17 retries",
+    )
+    sig = fingerprint_event(event)
+    assert sig.signature_key.startswith("failure|sshd|") or sig.signature_key.startswith("auth|sshd|")
+    assert sig.normalized_pattern

package/tests/test_ingest.py

@@ -0,0 +1,15 @@
+from brainstem.ingest import ingest_syslog_lines, signatures_for_events
+
+
+def test_ingest_syslog_lines_parses_basic_fields() -> None:
+    lines = [
+        'Mar 22 00:00:01 fw-01 charon: VPN tunnel dropped and recovered',
+        'Mar 22 00:00:03 fw-01 charon: VPN tunnel dropped and recovered',
+    ]
+    events = ingest_syslog_lines(lines, tenant_id='client-a', source_path='/var/log/syslog')
+    assert len(events) == 2
+    assert events[0].host == 'fw-01'
+    assert events[0].service == 'charon'
+    sigs = signatures_for_events(events)
+    assert len(sigs) == 2
+    assert sigs[0].signature_key == sigs[1].signature_key

package/tests/test_instrumentation.py

@@ -0,0 +1,16 @@
+from brainstem.instrumentation import emit, span
+
+
+def test_emit_writes_json_log(capsys) -> None:
+    emit("demo_event", count=2)
+    captured = capsys.readouterr()
+    assert '"event": "demo_event"' in captured.err
+    assert '"count": 2' in captured.err
+
+
+def test_span_emits_start_and_complete(capsys) -> None:
+    with span("unit_work", item_count=3):
+        pass
+    captured = capsys.readouterr()
+    assert '"event": "unit_work_start"' in captured.err
+    assert '"event": "unit_work_complete"' in captured.err

package/tests/test_interesting.py

@@ -0,0 +1,36 @@
+from brainstem.ingest import ingest_syslog_lines, signatures_for_events
+from brainstem.interesting import interesting_items
+from brainstem.recurrence import build_recurrence_candidates
+
+
+def test_interesting_items_returns_ranked_observations() -> None:
+    lines = [
+        'Mar 22 00:00:01 fw-01 charon: VPN tunnel dropped and recovered',
+        'Mar 22 00:00:03 fw-01 charon: VPN tunnel dropped and recovered',
+        'Mar 22 00:00:05 fw-01 charon: VPN tunnel dropped and recovered',
+        'Mar 22 00:01:10 fw-01 sshd: Failed password for admin from 10.1.2.3 port 54422',
+        'Mar 22 00:01:12 fw-01 sshd: Failed password for admin from 10.1.2.3 port 54425',
+    ]
+    events = ingest_syslog_lines(lines, tenant_id='client-a')
+    signatures = signatures_for_events(events)
+    candidates = build_recurrence_candidates(events, signatures, threshold=2)
+    items = interesting_items(candidates, limit=3)
+    assert items
+    assert items[0]['title']
+    assert 'why_it_matters' in items[0]
+    assert items[0]['signals']
+
+
+def test_interesting_items_respects_limit() -> None:
+    lines = [
+        'Mar 22 00:00:01 fw-01 charon: VPN tunnel dropped and recovered',
+        'Mar 22 00:00:03 fw-01 charon: VPN tunnel dropped and recovered',
+        'Mar 22 00:00:05 fw-01 charon: VPN tunnel dropped and recovered',
+        'Mar 22 00:01:10 fw-01 sshd: Failed password for admin from 10.1.2.3 port 54422',
+        'Mar 22 00:01:12 fw-01 sshd: Failed password for admin from 10.1.2.3 port 54425',
+    ]
+    events = ingest_syslog_lines(lines, tenant_id='client-a')
+    signatures = signatures_for_events(events)
+    candidates = build_recurrence_candidates(events, signatures, threshold=2)
+    items = interesting_items(candidates, limit=1)
+    assert len(items) == 1

package/tests/test_logicmonitor.py

@@ -0,0 +1,22 @@
+from brainstem.connectors.logicmonitor import map_logicmonitor_event
+
+
+def test_map_logicmonitor_event_normalizes_payload() -> None:
+    event = map_logicmonitor_event(
+        {
+            'resource_id': 123,
+            'resource_name': 'edge-fw-01',
+            'severity': 'warning',
+            'timestamp': '2026-03-22T00:00:00Z',
+            'message_raw': 'VPN tunnel dropped and recovered',
+            'metadata': {
+                'datasource': 'IPSec Tunnel',
+                'instance_name': 'site-b',
+                'acknowledged': False,
+            },
+        },
+        tenant_id='client-a',
+    )
+    assert event.source_type == 'logicmonitor'
+    assert event.host == 'edge-fw-01'
+    assert event.metadata['resource_id'] == 123

package/tests/test_recurrence.py

@@ -0,0 +1,16 @@
+from brainstem.ingest import ingest_syslog_lines, signatures_for_events
+from brainstem.recurrence import build_recurrence_candidates, digest_items
+
+
+def test_build_recurrence_candidates_emits_candidate_for_repeated_signature() -> None:
+    lines = [
+        'Mar 22 00:00:01 fw-01 charon: VPN tunnel dropped and recovered',
+        'Mar 22 00:00:03 fw-01 charon: VPN tunnel dropped and recovered',
+        'Mar 22 00:00:05 fw-01 charon: VPN tunnel dropped and recovered',
+    ]
+    events = ingest_syslog_lines(lines, tenant_id='client-a', source_path='/var/log/syslog')
+    signatures = signatures_for_events(events)
+    candidates = build_recurrence_candidates(events, signatures, threshold=2)
+    assert candidates
+    digest = digest_items(candidates)
+    assert digest[0]['decision_band'] in {'watch', 'review', 'urgent_human_review', 'promote_to_incident_memory'}

package/tests/test_scoring.py

@@ -0,0 +1,21 @@
+from brainstem.scoring import score_candidate
+
+
+def test_score_candidate_returns_band() -> None:
+    candidate = score_candidate(
+        recurrence=0.8,
+        recovery=0.7,
+        spread=0.4,
+        novelty=0.5,
+        impact=0.8,
+        precursor=0.7,
+        memory_weight=0.6,
+    )
+    assert candidate.score_total > 0
+    assert candidate.decision_band in {
+        "ignore",
+        "watch",
+        "review",
+        "urgent_human_review",
+        "promote_to_incident_memory",
+    }