@simbimbo/brainstem 0.0.2 → 0.0.4

package/tests/test_storage.py

@@ -1,8 +1,25 @@
+import sqlite3
 from pathlib import Path
 
-from brainstem.ingest import ingest_syslog_lines, signatures_for_events
+from brainstem.ingest import ingest_syslog_lines, parse_syslog_envelopes, run_ingest_pipeline, signatures_for_events
+from brainstem.models import RawInputEnvelope
 from brainstem.recurrence import build_recurrence_candidates
-from brainstem.storage import init_db, list_candidates, store_candidates, store_events, store_signatures
+from brainstem.storage import (
+    extract_source_raw_envelope_ids,
+    get_raw_envelope_by_id,
+    get_ingest_stats,
+    init_db,
+    list_candidates,
+    get_source_dimension_summaries,
+    get_source_status_summaries,
+    store_candidates,
+    store_events,
+    list_recent_failed_raw_envelopes,
+    list_recent_raw_envelopes,
+    store_raw_envelopes,
+    set_raw_envelope_status,
+    store_signatures,
+)
 
 
 def test_storage_round_trip(tmp_path: Path) -> None:
@@ -24,3 +41,354 @@ def test_storage_round_trip(tmp_path: Path) -> None:
     rows = list_candidates(str(db_path), limit=10)
     assert rows
     assert rows[0]['title']
+
+
+def test_raw_envelope_lineage_stored_in_signature_and_candidate_metadata(tmp_path: Path) -> None:
+    db_path = tmp_path / 'brainstem.sqlite3'
+    raw_envelopes = parse_syslog_envelopes(
+        [
+            "Mar 22 00:00:01 fw-01 charon: VPN tunnel dropped and recovered",
+            "Mar 22 00:00:03 fw-01 charon: VPN tunnel dropped and recovered",
+            "Mar 22 00:00:05 fw-01 charon: VPN tunnel dropped and recovered",
+        ],
+        tenant_id="client-a",
+        source_path="/var/log/syslog",
+    )
+    result = run_ingest_pipeline(raw_envelopes, threshold=2, db_path=str(db_path))
+
+    assert result.raw_envelope_ids == [1, 2, 3]
+    assert all(event.raw_envelope_id is not None for event in result.events)
+
+    signatures = sqlite3.connect(db_path)
+    try:
+        signature_rows = signatures.execute(
+            "SELECT metadata_json FROM signatures ORDER BY id ASC"
+        ).fetchall()
+    finally:
+        signatures.close()
+    assert signature_rows
+    signature_metadata = extract_source_raw_envelope_ids(signature_rows[0][0])
+    assert set(signature_metadata) == set(result.raw_envelope_ids)
+
+    candidate_rows = list_candidates(str(db_path), limit=10)
+    assert candidate_rows
+    candidate_metadata = extract_source_raw_envelope_ids(candidate_rows[0]["metadata_json"])
+    assert set(candidate_metadata) == set(result.raw_envelope_ids)
+
+
+def test_raw_envelope_records_are_persisted(tmp_path: Path) -> None:
+    db_path = tmp_path / 'brainstem.sqlite3'
+    init_db(str(db_path))
+    raw_events = [
+        RawInputEnvelope(
+            tenant_id="client-a",
+            source_type="syslog",
+            timestamp="2026-03-22T00:00:01Z",
+            message_raw="VPN tunnel dropped and recovered",
+            host="fw-01",
+            service="charon",
+        ),
+        RawInputEnvelope(
+            tenant_id="client-a",
+            source_type="syslog",
+            timestamp="2026-03-22T00:00:02Z",
+            message_raw="IPsec SA rekey succeeded",
+            host="fw-01",
+            service="charon",
+        ),
+    ]
+    assert store_raw_envelopes(raw_events, str(db_path)) == [1, 2]
+
+    conn = sqlite3.connect(db_path)
+    try:
+        rows = conn.execute(
+            "SELECT tenant_id, source_type, message_raw, canonicalization_status FROM raw_envelopes ORDER BY id ASC"
+        ).fetchall()
+    finally:
+        conn.close()
+
+    assert len(rows) == 2
+    assert rows[0][0] == "client-a"
+    assert rows[0][1] == "syslog"
+    assert rows[0][2] == "VPN tunnel dropped and recovered"
+    assert rows[0][3] == "received"
+    assert rows[1][3] == "received"
+
+
+def test_ingest_stats_from_raw_envelopes(tmp_path: Path) -> None:
+    db_path = tmp_path / 'brainstem.sqlite3'
+    init_db(str(db_path))
+    conn = sqlite3.connect(db_path)
+    try:
+        conn.execute(
+            """
+            INSERT INTO raw_envelopes (
+                tenant_id, source_type, timestamp, host, service, severity,
+                asset_id, source_path, facility, message_raw,
+                structured_fields_json, correlation_keys_json, metadata_json,
+                canonicalization_status, failure_reason
+            ) VALUES (
+                'client-a', 'syslog', '2026-03-22T00:00:00Z',
+                'fw-01', 'charon', 'info', '', '', '', 'ok', '{}', '{}', '{}',
+                'canonicalized', NULL
+            )
+            """
+        )
+        conn.execute(
+            """
+            INSERT INTO raw_envelopes (
+                tenant_id, source_type, timestamp, host, service, severity,
+                asset_id, source_path, facility, message_raw,
+                structured_fields_json, correlation_keys_json, metadata_json,
+                canonicalization_status, failure_reason
+            ) VALUES (
+                'client-a', 'syslog', '2026-03-22T00:00:00Z',
+                'fw-01', 'charon', 'info', '', '', '', 'bad', '{}', '{}', '{}',
+                'parse_failed', 'message empty'
+            )
+            """
+        )
+        conn.execute(
+            "INSERT INTO candidates (candidate_type, title, summary, score_total, score_breakdown_json, decision_band, source_signature_ids_json, source_event_ids_json, confidence, metadata_json) VALUES ('recurrence', 'x', 'y', 1.0, '{}', 'medium', '[]', '[]', 0.1, '{}')"
+        )
+        conn.commit()
+    finally:
+        conn.close()
+
+    stats = get_ingest_stats(str(db_path))
+    assert stats["received"] == 2
+    assert stats["canonicalized"] == 1
+    assert stats["parse_failed"] == 1
+    assert stats["candidates_generated"] == 1
+
+
+def test_source_dimension_summaries(tmp_path: Path) -> None:
+    db_path = tmp_path / 'brainstem.sqlite3'
+    init_db(str(db_path))
+    store_raw_envelopes(
+        [
+            RawInputEnvelope(
+                tenant_id='client-a',
+                source_type='syslog',
+                source_id='fw-01',
+                source_name='edge-fw-01',
+                timestamp='2026-03-22T00:00:01Z',
+                message_raw='VPN tunnel dropped and recovered',
+                source_path='/var/log/syslog',
+                host='fw-01',
+                service='charon',
+            ),
+            RawInputEnvelope(
+                tenant_id='client-a',
+                source_type='syslog',
+                source_id='fw-01',
+                source_name='edge-fw-01',
+                timestamp='2026-03-22T00:00:02Z',
+                message_raw='IPsec SA rekey succeeded',
+                source_path='/var/log/syslog',
+                host='fw-01',
+                service='charon',
+            ),
+            RawInputEnvelope(
+                tenant_id='client-a',
+                source_type='logicmonitor',
+                source_id='lm-1',
+                source_name='edge-lm-01',
+                timestamp='2026-03-22T00:00:03Z',
+                message_raw='CPU usage high',
+                source_path='/alerts',
+                host='lm-01',
+                service='logicmonitor',
+            ),
+        ],
+        db_path=str(db_path),
+    )
+
+    summary = get_source_dimension_summaries(str(db_path), limit=10)
+    assert summary['source_type'][0]['value'] == "syslog"
+    assert summary['source_type'][0]['count'] == 2
+    assert summary['source_type'][1]['value'] == "logicmonitor"
+    assert summary['source_type'][1]['count'] == 1
+    assert dict((entry['value'], entry['count']) for entry in summary['source_path']) == {
+        '/alerts': 1,
+        '/var/log/syslog': 2,
+    }
+    assert dict((entry['value'], entry['count']) for entry in summary['source_id']) == {
+        'fw-01': 2,
+        'lm-1': 1,
+    }
+
+
+def test_source_status_summaries_from_raw_envelope_history(tmp_path: Path) -> None:
+    db_path = tmp_path / 'brainstem.sqlite3'
+    init_db(str(db_path))
+    raw_ids = store_raw_envelopes(
+        [
+            RawInputEnvelope(
+                tenant_id='client-a',
+                source_type='syslog',
+                source_id='fw-01',
+                source_name='edge-fw-01',
+                timestamp='2026-03-22T00:00:01Z',
+                message_raw='event 1',
+                source_path='/var/log/syslog',
+                host='fw-01',
+                service='charon',
+            ),
+            RawInputEnvelope(
+                tenant_id='client-a',
+                source_type='syslog',
+                source_id='fw-01',
+                source_name='edge-fw-01',
+                timestamp='2026-03-22T00:00:02Z',
+                message_raw='event 2',
+                source_path='/var/log/syslog',
+                host='fw-01',
+                service='charon',
+            ),
+            RawInputEnvelope(
+                tenant_id='client-a',
+                source_type='syslog',
+                source_id='fw-02',
+                source_name='edge-fw-02',
+                timestamp='2026-03-22T00:00:03Z',
+                message_raw='event 3',
+                source_path='/var/log/auth.log',
+                host='fw-02',
+                service='sshd',
+            ),
+        ],
+        db_path=str(db_path),
+    )
+    set_raw_envelope_status(raw_ids[0], 'parse_failed', db_path=str(db_path), failure_reason='empty event')
+    set_raw_envelope_status(raw_ids[1], 'canonicalized', db_path=str(db_path))
+    set_raw_envelope_status(raw_ids[2], 'unsupported', db_path=str(db_path), failure_reason='bad source payload')
+
+    summaries = get_source_status_summaries(str(db_path), limit=10)
+    assert len(summaries) == 2
+    fw01 = next(item for item in summaries if item['source_id'] == 'fw-01' and item['source_path'] == '/var/log/syslog')
+    fw02 = next(item for item in summaries if item['source_id'] == 'fw-02')
+    assert fw01['raw_count'] == 2
+    assert fw01['canonicalized_count'] == 1
+    assert fw01['parse_failed_count'] == 1
+    assert fw01['unsupported_count'] == 0
+    assert fw01['first_seen_at'] == '2026-03-22T00:00:01Z'
+    assert fw01['last_seen_at'] == '2026-03-22T00:00:02Z'
+    assert fw02['raw_count'] == 1
+    assert fw02['canonicalized_count'] == 0
+    assert fw02['parse_failed_count'] == 0
+    assert fw02['unsupported_count'] == 1
+
+    filtered = get_source_status_summaries(str(db_path), source_type='syslog', source_id='fw-01', source_path='/var/log/syslog')
+    assert len(filtered) == 1
+    assert filtered[0]['source_id'] == 'fw-01'
+
+
+def test_list_recent_raw_envelopes_supports_status_filtering(tmp_path: Path) -> None:
+    db_path = tmp_path / 'brainstem.sqlite3'
+    init_db(str(db_path))
+    raw_ids = store_raw_envelopes(
+        [
+            RawInputEnvelope(
+                tenant_id='client-a',
+                source_type='syslog',
+                timestamp='2026-03-22T00:00:01Z',
+                message_raw='first',
+                host='fw-01',
+                service='sshd',
+            ),
+            RawInputEnvelope(
+                tenant_id='client-a',
+                source_type='syslog',
+                timestamp='2026-03-22T00:00:02Z',
+                message_raw='second',
+                host='fw-01',
+                service='sshd',
+            ),
+            RawInputEnvelope(
+                tenant_id='client-a',
+                source_type='syslog',
+                timestamp='2026-03-22T00:00:03Z',
+                message_raw='third',
+                host='fw-01',
+                service='sshd',
+            ),
+        ],
+        db_path=str(db_path),
+    )
+    set_raw_envelope_status(raw_ids[0], 'parse_failed', db_path=str(db_path), failure_reason='empty message')
+    set_raw_envelope_status(raw_ids[1], 'canonicalized', db_path=str(db_path))
+    set_raw_envelope_status(raw_ids[2], 'unsupported', db_path=str(db_path), failure_reason='unsupported source')
+
+    all_rows = list_recent_raw_envelopes(str(db_path), limit=10)
+    assert [row['id'] for row in all_rows] == [raw_ids[2], raw_ids[1], raw_ids[0]]
+
+    parsed_only = list_recent_raw_envelopes(str(db_path), status='parse_failed', limit=10)
+    assert [row['id'] for row in parsed_only] == [raw_ids[0]]
+
+
+def test_query_recent_failed_raw_envelopes_with_status_filter(tmp_path: Path) -> None:
+    db_path = tmp_path / 'brainstem.sqlite3'
+    init_db(str(db_path))
+    raw_events = [
+        RawInputEnvelope(
+            tenant_id="client-a",
+            source_type="syslog",
+            timestamp="2026-03-22T00:00:01Z",
+            message_raw="first",
+            host="fw-01",
+            service="sshd",
+        ),
+        RawInputEnvelope(
+            tenant_id="client-a",
+            source_type="syslog",
+            timestamp="2026-03-22T00:00:02Z",
+            message_raw="second",
+            host="fw-01",
+            service="sshd",
+        ),
+        RawInputEnvelope(
+            tenant_id="client-a",
+            source_type="syslog",
+            timestamp="2026-03-22T00:00:03Z",
+            message_raw="third",
+            host="fw-01",
+            service="sshd",
+        ),
+    ]
+    raw_ids = store_raw_envelopes(raw_events, str(db_path))
+    set_raw_envelope_status(raw_ids[0], "parse_failed", db_path=str(db_path), failure_reason="empty message")
+    set_raw_envelope_status(raw_ids[1], "canonicalized", db_path=str(db_path))
+    set_raw_envelope_status(raw_ids[2], "unsupported", db_path=str(db_path), failure_reason="unsupported source")
+
+    failures = list_recent_failed_raw_envelopes(str(db_path), limit=10)
+    assert [row["id"] for row in failures] == [raw_ids[2], raw_ids[0]]
+    assert failures[0]["canonicalization_status"] == "unsupported"
+    assert failures[1]["canonicalization_status"] == "parse_failed"
+
+    parsed_only = list_recent_failed_raw_envelopes(str(db_path), status="parse_failed", limit=10)
+    assert len(parsed_only) == 1
+    assert parsed_only[0]["id"] == raw_ids[0]
+
+
+def test_get_raw_envelope_by_id(tmp_path: Path) -> None:
+    db_path = tmp_path / 'brainstem.sqlite3'
+    init_db(str(db_path))
+    raw_events = [
+        RawInputEnvelope(
+            tenant_id="client-a",
+            source_type="syslog",
+            timestamp="2026-03-22T00:00:01Z",
+            message_raw="single",
+            host="fw-01",
+            service="charon",
+        )
+    ]
+    (raw_id,) = store_raw_envelopes(raw_events, str(db_path))
+    set_raw_envelope_status(raw_id, "parse_failed", db_path=str(db_path), failure_reason="empty message")
+
+    row = get_raw_envelope_by_id(raw_id, db_path=str(db_path))
+    assert row is not None
+    assert row["id"] == raw_id
+    assert row["canonicalization_status"] == "parse_failed"
+    assert row["failure_reason"] == "empty message"