@simbimbo/brainstem 0.0.4 → 0.0.5

package/tests/test_api.py

@@ -132,6 +132,74 @@ def test_ingest_batch_mixed_success_and_failure_returns_per_item_accounting(tmp_
         assert "failure_reason" in item
 
 
+def test_ingest_logicmonitor_endpoint_persists_and_surfaces_via_runtime_inspection(tmp_path: Path) -> None:
+    client = TestClient(app)
+    db_path = tmp_path / "logicmonitor.sqlite3"
+
+    response = client.post(
+        "/ingest/logicmonitor",
+        json={
+            "tenant_id": "client-a",
+            "source_path": "/logicmonitor/webhook",
+            "threshold": 2,
+            "db_path": str(db_path),
+            "events": [
+                {
+                    "resource_id": 123,
+                    "resource_name": "edge-fw-01",
+                    "severity": "warning",
+                    "timestamp": "2026-03-22T00:00:00Z",
+                    "message_raw": "VPN tunnel dropped and recovered",
+                    "metadata": {
+                        "datasource": "IPSec Tunnel",
+                        "instance_name": "site-b",
+                    },
+                },
+                {
+                    "resource_id": 124,
+                    "resource_name": "edge-fw-02",
+                    "severity": "warning",
+                    "timestamp": "2026-03-22T00:00:01Z",
+                    "message_raw": "VPN tunnel dropped and recovered",
+                    "metadata": {
+                        "datasource": "IPSec Tunnel",
+                        "instance_name": "site-c",
+                    },
+                },
+            ],
+        },
+    )
+    assert response.status_code == 200
+    payload = response.json()
+    assert payload["ok"] is True
+    assert payload["event_count"] == 2
+    assert payload["signature_count"] >= 1
+
+    raw_response = client.get(f"/raw_envelopes?db_path={db_path}&source_type=logicmonitor&limit=10")
+    assert raw_response.status_code == 200
+    raw_payload = raw_response.json()
+    assert raw_payload["count"] == 2
+    assert all(item["source_type"] == "logicmonitor" for item in raw_payload["items"])
+    assert all(item["canonicalization_status"] == "canonicalized" for item in raw_payload["items"])
+
+    canonical_response = client.get(f"/canonical_events?db_path={db_path}&source=logicmonitor&limit=10")
+    assert canonical_response.status_code == 200
+    canonical_payload = canonical_response.json()
+    assert canonical_payload["count"] == 2
+    assert all(item["source"] == "logicmonitor" for item in canonical_payload["items"])
+
+    sources_response = client.get(f"/sources?db_path={db_path}&limit=10")
+    assert sources_response.status_code == 200
+    sources_payload = sources_response.json()
+    source_types = [entry["value"] for entry in sources_payload["items"]["source_type"]]
+    assert "logicmonitor" in source_types
+
+    runtime_response = client.get("/runtime")
+    assert runtime_response.status_code == 200
+    runtime_payload = runtime_response.json()
+    assert runtime_payload["runtime"]["capability_flags"]["ingest_endpoints"]["logicmonitor_events"] is True
+
+
 def test_candidates_endpoint_returns_candidate_inspection_payload_and_supports_filtering(tmp_path: Path) -> None:
     client = TestClient(app)
     db_path = tmp_path / "brainstem_candidates.sqlite3"
@@ -483,6 +551,24 @@ def test_status_and_healthz_are_coherent() -> None:
     assert status_response.json() == healthz_response.json()
 
 
+def test_runtime_endpoint_includes_source_capability_matrix() -> None:
+    client = TestClient(app)
+    response = client.get("/runtime")
+    assert response.status_code == 200
+    source_capabilities = response.json()["runtime"]["capability_flags"]["source_capabilities"]
+    source_types = source_capabilities["source_types"]
+
+    assert "syslog" in source_types
+    assert "file" in source_types
+    assert "logicmonitor" in source_types
+
+    per_source = {item["source_type"]: item["modes"] for item in source_capabilities["ingest_modes_by_source_type"]}
+    assert {"mode": "udp_listener", "endpoint": "brainstem.listener"} in per_source["syslog"]
+    assert {"mode": "single_event_api", "endpoint": "/ingest/event"} in per_source["file"]
+    assert {"mode": "batch_api", "endpoint": "/ingest/batch"} in per_source["file"]
+    assert {"mode": "logicmonitor_webhook", "endpoint": "/ingest/logicmonitor"} in per_source["logicmonitor"]
+
+
 def test_runtime_endpoint_reports_config_object(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
     custom_db = tmp_path / "runtime.sqlite3"
     monkeypatch.setenv("BRAINSTEM_DB_PATH", str(custom_db))

package/tests/test_config.py

@@ -12,10 +12,25 @@ def test_runtime_config_exposes_shared_defaults() -> None:
     assert cfg.listener.syslog_port == 5514
     assert cfg.listener.syslog_source_path == "/dev/udp"
     assert cfg.defaults.ingest_threshold == 2
+    assert cfg.defaults.recurrence_threshold == 2
     assert cfg.defaults.interesting_limit == 5
+    assert cfg.candidate_attention.recurrence_count_normalizer == 10
+    assert cfg.candidate_attention.decision_band_watch == 0.25
     assert cfg.limits.replay_raw_max_ids == 32
 
 
+def test_runtime_config_allows_threshold_and_attention_overrides(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("BRAINSTEM_INGEST_THRESHOLD", "7")
+    monkeypatch.setenv("BRAINSTEM_RECURRENCE_THRESHOLD", "9")
+    monkeypatch.setenv("BRAINSTEM_CANDIDATE_RECURRENCE_NORMALIZER", "5")
+    monkeypatch.setenv("BRAINSTEM_CANDIDATE_RECOVERY", "0.11")
+    cfg = get_runtime_config()
+    assert cfg.defaults.ingest_threshold == 7
+    assert cfg.defaults.recurrence_threshold == 9
+    assert cfg.candidate_attention.recurrence_count_normalizer == 5
+    assert cfg.candidate_attention.recovery_signal_weight == 0.11
+
+
 def test_resolve_default_db_path_uses_env_override(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
     custom_db = tmp_path / "override.sqlite3"
     monkeypatch.setenv("BRAINSTEM_DB_PATH", str(custom_db))

package/tests/test_fingerprint.py

@@ -1,4 +1,6 @@
-from brainstem.fingerprint import normalize_message, fingerprint_event
+from brainstem.connectors.logicmonitor import map_logicmonitor_payload_to_raw_envelope
+from brainstem.fingerprint import fingerprint_event, normalize_message
+from brainstem.ingest import canonicalize_raw_input_envelope, parse_file_line, parse_syslog_line
 from brainstem.models import Event
 
 
@@ -20,3 +22,51 @@ def test_fingerprint_event_builds_signature() -> None:
     sig = fingerprint_event(event)
     assert sig.signature_key.startswith("failure|sshd|") or sig.signature_key.startswith("auth|sshd|")
     assert sig.normalized_pattern
+
+
+def test_event_family_for_connectivity_syslog_event() -> None:
+    event = parse_syslog_line(
+        "Mar 22 00:00:01 fw-01 charon: VPN tunnel dropped and recovered",
+        tenant_id="client-a",
+        source_path="/var/log/syslog",
+    )
+    signature = fingerprint_event(event)
+    assert signature.event_family == "connectivity"
+
+
+def test_event_family_for_resource_file_event() -> None:
+    event = parse_file_line(
+        "disk pressure warning on /var reached high watermark",
+        tenant_id="client-a",
+        source_path="/tmp/sample.log",
+    )
+    signature = fingerprint_event(event)
+    assert signature.event_family == "resource"
+
+
+def test_event_family_for_logicmonitor_connectivity_event() -> None:
+    raw = map_logicmonitor_payload_to_raw_envelope(
+        {
+            "resource_id": 123,
+            "resource_name": "edge-fw-01",
+            "message_raw": "IPsec tunnel dropped and recovered",
+            "severity": "warning",
+            "timestamp": "2026-03-22T00:00:00Z",
+            "metadata": {"datasource": "IPSec Tunnel"},
+        },
+        tenant_id="client-a",
+        source_path="/logicmonitor/ingest",
+    )
+    event = canonicalize_raw_input_envelope(raw)
+    signature = fingerprint_event(event)
+    assert signature.event_family == "connectivity"
+
+
+def test_event_family_falls_back_to_generic_when_no_clear_signal() -> None:
+    event = parse_file_line(
+        "Routine status check completed without issue",
+        tenant_id="client-a",
+        source_path="/tmp/sample.log",
+    )
+    signature = fingerprint_event(event)
+    assert signature.event_family == "generic"

package/tests/test_logicmonitor.py

@@ -1,4 +1,5 @@
-from brainstem.connectors.logicmonitor import map_logicmonitor_event
+from brainstem.connectors.logicmonitor import map_logicmonitor_event, map_logicmonitor_payload_to_raw_envelope
+from brainstem.ingest import canonicalize_raw_input_envelope
 
 
 def test_map_logicmonitor_event_normalizes_payload() -> None:
@@ -20,3 +21,55 @@ def test_map_logicmonitor_event_normalizes_payload() -> None:
     assert event.source_type == 'logicmonitor'
     assert event.host == 'edge-fw-01'
     assert event.metadata['resource_id'] == 123
+
+
+def test_map_logicmonitor_payload_to_raw_envelope_preserves_shape() -> None:
+    raw = map_logicmonitor_payload_to_raw_envelope(
+        {
+            'resource_id': 123,
+            'resource_name': 'edge-fw-01',
+            'severity': 'warning',
+            'timestamp': '2026-03-22T00:00:00Z',
+            'message_raw': 'VPN tunnel dropped and recovered',
+            'metadata': {
+                'datasource': 'IPSec Tunnel',
+                'instance_name': 'site-b',
+            },
+        },
+        tenant_id='client-a',
+        source_path='/logicmonitor/ingest',
+    )
+
+    assert raw.source_type == 'logicmonitor'
+    assert raw.source_path == '/logicmonitor/ingest'
+    assert raw.host == 'edge-fw-01'
+    assert raw.service == 'IPSec Tunnel'
+    assert raw.severity == 'warning'
+    assert raw.message_raw == 'VPN tunnel dropped and recovered'
+    assert raw.metadata['alert_id'] is None
+    assert raw.timestamp == '2026-03-22T00:00:00Z'
+
+
+def test_canonicalization_from_logicmonitor_raw_envelope_remains_in_ingest_flow() -> None:
+    raw = map_logicmonitor_payload_to_raw_envelope(
+        {
+            'resource_id': 123,
+            'resource_name': 'edge-fw-01',
+            'severity': 'warning',
+            'timestamp': '2026-03-22T00:00:00Z',
+            'message_raw': 'VPN tunnel dropped and recovered',
+            'metadata': {
+                'datasource': 'IPSec Tunnel',
+            },
+        },
+        tenant_id='client-a',
+        source_path='/logicmonitor/ingest',
+    )
+
+    event = canonicalize_raw_input_envelope(raw)
+    assert event.source_type == 'logicmonitor'
+    assert event.host == 'edge-fw-01'
+    assert event.service == 'IPSec Tunnel'
+    assert event.ingest_metadata['canonicalization_source'] == 'logicmonitor'
+    assert event.ingest_metadata['raw_input_seen'] is True
+    assert event.message_normalized == 'vpn tunnel dropped and recovered'

package/tests/test_recurrence.py

@@ -1,3 +1,5 @@
+import pytest
+
 from brainstem.ingest import ingest_syslog_lines, signatures_for_events
 from brainstem.recurrence import build_recurrence_candidates, digest_items
 
@@ -16,3 +18,15 @@ def test_build_recurrence_candidates_emits_candidate_for_repeated_signature() ->
     assert digest[0]["attention_explanation"]["dominant_signals"]
     assert "summary" in digest[0]["attention_explanation"]
     assert digest[0]['decision_band'] in {'watch', 'review', 'urgent_human_review', 'promote_to_incident_memory'}
+
+
+def test_build_recurrence_candidates_uses_default_threshold_from_config(monkeypatch: pytest.MonkeyPatch) -> None:
+    lines = [
+        "Mar 22 00:00:01 fw-01 charon: VPN tunnel dropped and recovered",
+        "Mar 22 00:00:03 fw-01 charon: VPN tunnel dropped and recovered",
+    ]
+    monkeypatch.setenv("BRAINSTEM_RECURRENCE_THRESHOLD", "3")
+    events = ingest_syslog_lines(lines, tenant_id="client-a", source_path="/var/log/syslog")
+    signatures = signatures_for_events(events)
+    candidates = build_recurrence_candidates(events, signatures)
+    assert candidates == []

package/tests/test_storage.py

@@ -6,7 +6,10 @@ from brainstem.models import RawInputEnvelope
 from brainstem.recurrence import build_recurrence_candidates
 from brainstem.storage import (
     extract_source_raw_envelope_ids,
+    create_sqlite_snapshot,
+    clear_storage_tables,
     get_raw_envelope_by_id,
+    get_storage_counts,
     get_ingest_stats,
     init_db,
     list_candidates,
@@ -18,6 +21,7 @@ from brainstem.storage import (
     list_recent_raw_envelopes,
     store_raw_envelopes,
     set_raw_envelope_status,
+    resolve_maintenance_tables,
     store_signatures,
 )
 
@@ -392,3 +396,76 @@ def test_get_raw_envelope_by_id(tmp_path: Path) -> None:
     assert row["id"] == raw_id
     assert row["canonicalization_status"] == "parse_failed"
     assert row["failure_reason"] == "empty message"
+
+
+def test_storage_counts_report_expected_rows_for_ingested_payload(tmp_path: Path) -> None:
+    db_path = tmp_path / 'brainstem.sqlite3'
+    raw_events = parse_syslog_envelopes(
+        [
+            "Mar 22 00:00:01 fw-01 charon: VPN tunnel dropped and recovered",
+            "Mar 22 00:00:02 fw-01 charon: VPN tunnel dropped and recovered",
+        ],
+        tenant_id="client-a",
+        source_path="/var/log/syslog",
+    )
+    result = run_ingest_pipeline(raw_events, threshold=2, db_path=str(db_path))
+    assert result.raw_envelope_ids
+    assert result.events
+    assert result.signatures
+    assert result.candidates
+
+    counts = get_storage_counts(str(db_path))
+    expected_signature_rows = len({sig.signature_key for sig in result.signatures})
+    assert counts["raw_envelopes"] == len(result.raw_envelope_ids)
+    assert counts["events"] == len(result.events)
+    assert counts["signatures"] == expected_signature_rows
+    assert counts["candidates"] == len(result.candidates)
+
+
+def test_resolve_and_clear_storage_tables_are_explicit() -> None:
+    assert resolve_maintenance_tables(["all"]) == ["events", "raw_envelopes", "signatures", "candidates"]
+    assert resolve_maintenance_tables(["raw_envelopes", "events"]) == ["raw_envelopes", "events"]
+    assert resolve_maintenance_tables(None) == ["events", "raw_envelopes", "signatures", "candidates"]
+
+
+def test_clear_storage_tables_only_clears_requested_tables(tmp_path: Path) -> None:
+    db_path = tmp_path / 'brainstem.sqlite3'
+    raw_events = parse_syslog_envelopes(
+        [
+            "Mar 22 00:00:01 fw-01 charon: VPN tunnel dropped and recovered",
+            "Mar 22 00:00:02 fw-01 charon: VPN tunnel dropped and recovered",
+        ],
+        tenant_id="client-a",
+        source_path="/var/log/syslog",
+    )
+    result = run_ingest_pipeline(raw_events, threshold=2, db_path=str(db_path))
+    removed = clear_storage_tables(str(db_path), tables=["raw_envelopes", "events"])
+    expected_signature_rows = len({sig.signature_key for sig in result.signatures})
+    assert removed["raw_envelopes"] == len(result.raw_envelope_ids)
+    assert removed["events"] == len(result.events)
+
+    counts_after = get_storage_counts(str(db_path))
+    assert counts_after["raw_envelopes"] == 0
+    assert counts_after["events"] == 0
+    assert counts_after["signatures"] == expected_signature_rows
+    assert counts_after["candidates"] == len(result.candidates)
+
+
+def test_create_sqlite_snapshot_returns_metadata_and_copies_file(tmp_path: Path) -> None:
+    db_path = tmp_path / "snapshot.sqlite3"
+    init_db(str(db_path))
+
+    first = create_sqlite_snapshot(str(db_path))
+    first_path = Path(first["snapshot_path"])
+    assert first["source_path"] == str(db_path)
+    assert first_path.exists()
+    assert first_path.is_file()
+    assert first_path != db_path
+    assert first["size"] == first_path.stat().st_size
+    assert first["size"] > 0
+    assert first["created_at"].endswith("Z")
+
+    second = create_sqlite_snapshot(str(db_path))
+    second_path = Path(second["snapshot_path"])
+    assert second_path.exists()
+    assert second["snapshot_path"] != first["snapshot_path"]