@simbimbo/brainstem 0.0.1 → 0.0.3

package/tests/test_storage.py

@@ -0,0 +1,294 @@
+import sqlite3
+from pathlib import Path
+
+from brainstem.ingest import ingest_syslog_lines, signatures_for_events
+from brainstem.models import RawInputEnvelope
+from brainstem.recurrence import build_recurrence_candidates
+from brainstem.storage import (
+    get_raw_envelope_by_id,
+    get_ingest_stats,
+    init_db,
+    list_candidates,
+    get_source_dimension_summaries,
+    store_candidates,
+    store_events,
+    list_recent_failed_raw_envelopes,
+    list_recent_raw_envelopes,
+    store_raw_envelopes,
+    set_raw_envelope_status,
+    store_signatures,
+)
+
+
+def test_storage_round_trip(tmp_path: Path) -> None:
+    db_path = tmp_path / 'brainstem.sqlite3'
+    init_db(str(db_path))
+    lines = [
+        'Mar 22 00:00:01 fw-01 charon: VPN tunnel dropped and recovered',
+        'Mar 22 00:00:03 fw-01 charon: VPN tunnel dropped and recovered',
+        'Mar 22 00:00:05 fw-01 charon: VPN tunnel dropped and recovered',
+    ]
+    events = ingest_syslog_lines(lines, tenant_id='client-a', source_path='/var/log/syslog')
+    signatures = signatures_for_events(events)
+    candidates = build_recurrence_candidates(events, signatures, threshold=2)
+
+    assert store_events(events, str(db_path)) == 3
+    assert store_signatures(signatures, str(db_path)) == 3
+    assert store_candidates(candidates, str(db_path)) >= 1
+
+    rows = list_candidates(str(db_path), limit=10)
+    assert rows
+    assert rows[0]['title']
+
+
+def test_raw_envelope_records_are_persisted(tmp_path: Path) -> None:
+    db_path = tmp_path / 'brainstem.sqlite3'
+    init_db(str(db_path))
+    raw_events = [
+        RawInputEnvelope(
+            tenant_id="client-a",
+            source_type="syslog",
+            timestamp="2026-03-22T00:00:01Z",
+            message_raw="VPN tunnel dropped and recovered",
+            host="fw-01",
+            service="charon",
+        ),
+        RawInputEnvelope(
+            tenant_id="client-a",
+            source_type="syslog",
+            timestamp="2026-03-22T00:00:02Z",
+            message_raw="IPsec SA rekey succeeded",
+            host="fw-01",
+            service="charon",
+        ),
+    ]
+    assert store_raw_envelopes(raw_events, str(db_path)) == [1, 2]
+
+    conn = sqlite3.connect(db_path)
+    try:
+        rows = conn.execute(
+            "SELECT tenant_id, source_type, message_raw, canonicalization_status FROM raw_envelopes ORDER BY id ASC"
+        ).fetchall()
+    finally:
+        conn.close()
+
+    assert len(rows) == 2
+    assert rows[0][0] == "client-a"
+    assert rows[0][1] == "syslog"
+    assert rows[0][2] == "VPN tunnel dropped and recovered"
+    assert rows[0][3] == "received"
+    assert rows[1][3] == "received"
+
+
+def test_ingest_stats_from_raw_envelopes(tmp_path: Path) -> None:
+    db_path = tmp_path / 'brainstem.sqlite3'
+    init_db(str(db_path))
+    conn = sqlite3.connect(db_path)
+    try:
+        conn.execute(
+            """
+            INSERT INTO raw_envelopes (
+                tenant_id, source_type, timestamp, host, service, severity,
+                asset_id, source_path, facility, message_raw,
+                structured_fields_json, correlation_keys_json, metadata_json,
+                canonicalization_status, failure_reason
+            ) VALUES (
+                'client-a', 'syslog', '2026-03-22T00:00:00Z',
+                'fw-01', 'charon', 'info', '', '', '', 'ok', '{}', '{}', '{}',
+                'canonicalized', NULL
+            )
+            """
+        )
+        conn.execute(
+            """
+            INSERT INTO raw_envelopes (
+                tenant_id, source_type, timestamp, host, service, severity,
+                asset_id, source_path, facility, message_raw,
+                structured_fields_json, correlation_keys_json, metadata_json,
+                canonicalization_status, failure_reason
+            ) VALUES (
+                'client-a', 'syslog', '2026-03-22T00:00:00Z',
+                'fw-01', 'charon', 'info', '', '', '', 'bad', '{}', '{}', '{}',
+                'parse_failed', 'message empty'
+            )
+            """
+        )
+        conn.execute(
+            "INSERT INTO candidates (candidate_type, title, summary, score_total, score_breakdown_json, decision_band, source_signature_ids_json, source_event_ids_json, confidence, metadata_json) VALUES ('recurrence', 'x', 'y', 1.0, '{}', 'medium', '[]', '[]', 0.1, '{}')"
+        )
+        conn.commit()
+    finally:
+        conn.close()
+
+    stats = get_ingest_stats(str(db_path))
+    assert stats["received"] == 2
+    assert stats["canonicalized"] == 1
+    assert stats["parse_failed"] == 1
+    assert stats["candidates_generated"] == 1
+
+
+def test_source_dimension_summaries(tmp_path: Path) -> None:
+    db_path = tmp_path / 'brainstem.sqlite3'
+    init_db(str(db_path))
+    store_raw_envelopes(
+        [
+            RawInputEnvelope(
+                tenant_id='client-a',
+                source_type='syslog',
+                source_id='fw-01',
+                source_name='edge-fw-01',
+                timestamp='2026-03-22T00:00:01Z',
+                message_raw='VPN tunnel dropped and recovered',
+                source_path='/var/log/syslog',
+                host='fw-01',
+                service='charon',
+            ),
+            RawInputEnvelope(
+                tenant_id='client-a',
+                source_type='syslog',
+                source_id='fw-01',
+                source_name='edge-fw-01',
+                timestamp='2026-03-22T00:00:02Z',
+                message_raw='IPsec SA rekey succeeded',
+                source_path='/var/log/syslog',
+                host='fw-01',
+                service='charon',
+            ),
+            RawInputEnvelope(
+                tenant_id='client-a',
+                source_type='logicmonitor',
+                source_id='lm-1',
+                source_name='edge-lm-01',
+                timestamp='2026-03-22T00:00:03Z',
+                message_raw='CPU usage high',
+                source_path='/alerts',
+                host='lm-01',
+                service='logicmonitor',
+            ),
+        ],
+        db_path=str(db_path),
+    )
+
+    summary = get_source_dimension_summaries(str(db_path), limit=10)
+    assert summary['source_type'][0]['value'] == "syslog"
+    assert summary['source_type'][0]['count'] == 2
+    assert summary['source_type'][1]['value'] == "logicmonitor"
+    assert summary['source_type'][1]['count'] == 1
+    assert dict((entry['value'], entry['count']) for entry in summary['source_path']) == {
+        '/alerts': 1,
+        '/var/log/syslog': 2,
+    }
+    assert dict((entry['value'], entry['count']) for entry in summary['source_id']) == {
+        'fw-01': 2,
+        'lm-1': 1,
+    }
+
+
+def test_list_recent_raw_envelopes_supports_status_filtering(tmp_path: Path) -> None:
+    db_path = tmp_path / 'brainstem.sqlite3'
+    init_db(str(db_path))
+    raw_ids = store_raw_envelopes(
+        [
+            RawInputEnvelope(
+                tenant_id='client-a',
+                source_type='syslog',
+                timestamp='2026-03-22T00:00:01Z',
+                message_raw='first',
+                host='fw-01',
+                service='sshd',
+            ),
+            RawInputEnvelope(
+                tenant_id='client-a',
+                source_type='syslog',
+                timestamp='2026-03-22T00:00:02Z',
+                message_raw='second',
+                host='fw-01',
+                service='sshd',
+            ),
+            RawInputEnvelope(
+                tenant_id='client-a',
+                source_type='syslog',
+                timestamp='2026-03-22T00:00:03Z',
+                message_raw='third',
+                host='fw-01',
+                service='sshd',
+            ),
+        ],
+        db_path=str(db_path),
+    )
+    set_raw_envelope_status(raw_ids[0], 'parse_failed', db_path=str(db_path), failure_reason='empty message')
+    set_raw_envelope_status(raw_ids[1], 'canonicalized', db_path=str(db_path))
+    set_raw_envelope_status(raw_ids[2], 'unsupported', db_path=str(db_path), failure_reason='unsupported source')
+
+    all_rows = list_recent_raw_envelopes(str(db_path), limit=10)
+    assert [row['id'] for row in all_rows] == [raw_ids[2], raw_ids[1], raw_ids[0]]
+
+    parsed_only = list_recent_raw_envelopes(str(db_path), status='parse_failed', limit=10)
+    assert [row['id'] for row in parsed_only] == [raw_ids[0]]
+
+
+def test_query_recent_failed_raw_envelopes_with_status_filter(tmp_path: Path) -> None:
+    db_path = tmp_path / 'brainstem.sqlite3'
+    init_db(str(db_path))
+    raw_events = [
+        RawInputEnvelope(
+            tenant_id="client-a",
+            source_type="syslog",
+            timestamp="2026-03-22T00:00:01Z",
+            message_raw="first",
+            host="fw-01",
+            service="sshd",
+        ),
+        RawInputEnvelope(
+            tenant_id="client-a",
+            source_type="syslog",
+            timestamp="2026-03-22T00:00:02Z",
+            message_raw="second",
+            host="fw-01",
+            service="sshd",
+        ),
+        RawInputEnvelope(
+            tenant_id="client-a",
+            source_type="syslog",
+            timestamp="2026-03-22T00:00:03Z",
+            message_raw="third",
+            host="fw-01",
+            service="sshd",
+        ),
+    ]
+    raw_ids = store_raw_envelopes(raw_events, str(db_path))
+    set_raw_envelope_status(raw_ids[0], "parse_failed", db_path=str(db_path), failure_reason="empty message")
+    set_raw_envelope_status(raw_ids[1], "canonicalized", db_path=str(db_path))
+    set_raw_envelope_status(raw_ids[2], "unsupported", db_path=str(db_path), failure_reason="unsupported source")
+
+    failures = list_recent_failed_raw_envelopes(str(db_path), limit=10)
+    assert [row["id"] for row in failures] == [raw_ids[2], raw_ids[0]]
+    assert failures[0]["canonicalization_status"] == "unsupported"
+    assert failures[1]["canonicalization_status"] == "parse_failed"
+
+    parsed_only = list_recent_failed_raw_envelopes(str(db_path), status="parse_failed", limit=10)
+    assert len(parsed_only) == 1
+    assert parsed_only[0]["id"] == raw_ids[0]
+
+
+def test_get_raw_envelope_by_id(tmp_path: Path) -> None:
+    db_path = tmp_path / 'brainstem.sqlite3'
+    init_db(str(db_path))
+    raw_events = [
+        RawInputEnvelope(
+            tenant_id="client-a",
+            source_type="syslog",
+            timestamp="2026-03-22T00:00:01Z",
+            message_raw="single",
+            host="fw-01",
+            service="charon",
+        )
+    ]
+    (raw_id,) = store_raw_envelopes(raw_events, str(db_path))
+    set_raw_envelope_status(raw_id, "parse_failed", db_path=str(db_path), failure_reason="empty message")
+
+    row = get_raw_envelope_by_id(raw_id, db_path=str(db_path))
+    assert row is not None
+    assert row["id"] == raw_id
+    assert row["canonicalization_status"] == "parse_failed"
+    assert row["failure_reason"] == "empty message"