@simbimbo/brainstem 0.0.3 → 0.0.5

package/tests/test_storage.py

@@ -1,21 +1,27 @@
 import sqlite3
 from pathlib import Path
 
-from brainstem.ingest import ingest_syslog_lines, signatures_for_events
+from brainstem.ingest import ingest_syslog_lines, parse_syslog_envelopes, run_ingest_pipeline, signatures_for_events
 from brainstem.models import RawInputEnvelope
 from brainstem.recurrence import build_recurrence_candidates
 from brainstem.storage import (
+    extract_source_raw_envelope_ids,
+    create_sqlite_snapshot,
+    clear_storage_tables,
     get_raw_envelope_by_id,
+    get_storage_counts,
     get_ingest_stats,
     init_db,
     list_candidates,
     get_source_dimension_summaries,
+    get_source_status_summaries,
     store_candidates,
     store_events,
     list_recent_failed_raw_envelopes,
     list_recent_raw_envelopes,
     store_raw_envelopes,
     set_raw_envelope_status,
+    resolve_maintenance_tables,
     store_signatures,
 )
 
@@ -41,6 +47,39 @@ def test_storage_round_trip(tmp_path: Path) -> None:
     assert rows[0]['title']
 
 
+def test_raw_envelope_lineage_stored_in_signature_and_candidate_metadata(tmp_path: Path) -> None:
+    db_path = tmp_path / 'brainstem.sqlite3'
+    raw_envelopes = parse_syslog_envelopes(
+        [
+            "Mar 22 00:00:01 fw-01 charon: VPN tunnel dropped and recovered",
+            "Mar 22 00:00:03 fw-01 charon: VPN tunnel dropped and recovered",
+            "Mar 22 00:00:05 fw-01 charon: VPN tunnel dropped and recovered",
+        ],
+        tenant_id="client-a",
+        source_path="/var/log/syslog",
+    )
+    result = run_ingest_pipeline(raw_envelopes, threshold=2, db_path=str(db_path))
+
+    assert result.raw_envelope_ids == [1, 2, 3]
+    assert all(event.raw_envelope_id is not None for event in result.events)
+
+    signatures = sqlite3.connect(db_path)
+    try:
+        signature_rows = signatures.execute(
+            "SELECT metadata_json FROM signatures ORDER BY id ASC"
+        ).fetchall()
+    finally:
+        signatures.close()
+    assert signature_rows
+    signature_metadata = extract_source_raw_envelope_ids(signature_rows[0][0])
+    assert set(signature_metadata) == set(result.raw_envelope_ids)
+
+    candidate_rows = list_candidates(str(db_path), limit=10)
+    assert candidate_rows
+    candidate_metadata = extract_source_raw_envelope_ids(candidate_rows[0]["metadata_json"])
+    assert set(candidate_metadata) == set(result.raw_envelope_ids)
+
+
 def test_raw_envelope_records_are_persisted(tmp_path: Path) -> None:
     db_path = tmp_path / 'brainstem.sqlite3'
     init_db(str(db_path))
@@ -184,6 +223,71 @@ def test_source_dimension_summaries(tmp_path: Path) -> None:
     }
 
 
+def test_source_status_summaries_from_raw_envelope_history(tmp_path: Path) -> None:
+    db_path = tmp_path / 'brainstem.sqlite3'
+    init_db(str(db_path))
+    raw_ids = store_raw_envelopes(
+        [
+            RawInputEnvelope(
+                tenant_id='client-a',
+                source_type='syslog',
+                source_id='fw-01',
+                source_name='edge-fw-01',
+                timestamp='2026-03-22T00:00:01Z',
+                message_raw='event 1',
+                source_path='/var/log/syslog',
+                host='fw-01',
+                service='charon',
+            ),
+            RawInputEnvelope(
+                tenant_id='client-a',
+                source_type='syslog',
+                source_id='fw-01',
+                source_name='edge-fw-01',
+                timestamp='2026-03-22T00:00:02Z',
+                message_raw='event 2',
+                source_path='/var/log/syslog',
+                host='fw-01',
+                service='charon',
+            ),
+            RawInputEnvelope(
+                tenant_id='client-a',
+                source_type='syslog',
+                source_id='fw-02',
+                source_name='edge-fw-02',
+                timestamp='2026-03-22T00:00:03Z',
+                message_raw='event 3',
+                source_path='/var/log/auth.log',
+                host='fw-02',
+                service='sshd',
+            ),
+        ],
+        db_path=str(db_path),
+    )
+    set_raw_envelope_status(raw_ids[0], 'parse_failed', db_path=str(db_path), failure_reason='empty event')
+    set_raw_envelope_status(raw_ids[1], 'canonicalized', db_path=str(db_path))
+    set_raw_envelope_status(raw_ids[2], 'unsupported', db_path=str(db_path), failure_reason='bad source payload')
+
+    summaries = get_source_status_summaries(str(db_path), limit=10)
+    assert len(summaries) == 2
+    fw01 = next(item for item in summaries if item['source_id'] == 'fw-01' and item['source_path'] == '/var/log/syslog')
+    fw02 = next(item for item in summaries if item['source_id'] == 'fw-02')
+    assert fw01['raw_count'] == 2
+    assert fw01['canonicalized_count'] == 1
+    assert fw01['parse_failed_count'] == 1
+    assert fw01['unsupported_count'] == 0
+    assert fw01['first_seen_at'] == '2026-03-22T00:00:01Z'
+    assert fw01['last_seen_at'] == '2026-03-22T00:00:02Z'
+    assert fw02['raw_count'] == 1
+    assert fw02['canonicalized_count'] == 0
+    assert fw02['parse_failed_count'] == 0
+    assert fw02['unsupported_count'] == 1
+
+    filtered = get_source_status_summaries(str(db_path), source_type='syslog', source_id='fw-01', source_path='/var/log/syslog')
+    assert len(filtered) == 1
+    assert filtered[0]['source_id'] == 'fw-01'
+
+
 def test_list_recent_raw_envelopes_supports_status_filtering(tmp_path: Path) -> None:
     db_path = tmp_path / 'brainstem.sqlite3'
     init_db(str(db_path))
@@ -292,3 +396,76 @@ def test_get_raw_envelope_by_id(tmp_path: Path) -> None:
     assert row["id"] == raw_id
     assert row["canonicalization_status"] == "parse_failed"
     assert row["failure_reason"] == "empty message"
+
+
+def test_storage_counts_report_expected_rows_for_ingested_payload(tmp_path: Path) -> None:
+    db_path = tmp_path / 'brainstem.sqlite3'
+    raw_events = parse_syslog_envelopes(
+        [
+            "Mar 22 00:00:01 fw-01 charon: VPN tunnel dropped and recovered",
+            "Mar 22 00:00:02 fw-01 charon: VPN tunnel dropped and recovered",
+        ],
+        tenant_id="client-a",
+        source_path="/var/log/syslog",
+    )
+    result = run_ingest_pipeline(raw_events, threshold=2, db_path=str(db_path))
+    assert result.raw_envelope_ids
+    assert result.events
+    assert result.signatures
+    assert result.candidates
+
+    counts = get_storage_counts(str(db_path))
+    expected_signature_rows = len({sig.signature_key for sig in result.signatures})
+    assert counts["raw_envelopes"] == len(result.raw_envelope_ids)
+    assert counts["events"] == len(result.events)
+    assert counts["signatures"] == expected_signature_rows
+    assert counts["candidates"] == len(result.candidates)
+
+
+def test_resolve_and_clear_storage_tables_are_explicit() -> None:
+    assert resolve_maintenance_tables(["all"]) == ["events", "raw_envelopes", "signatures", "candidates"]
+    assert resolve_maintenance_tables(["raw_envelopes", "events"]) == ["raw_envelopes", "events"]
+    assert resolve_maintenance_tables(None) == ["events", "raw_envelopes", "signatures", "candidates"]
+
+
+def test_clear_storage_tables_only_clears_requested_tables(tmp_path: Path) -> None:
+    db_path = tmp_path / 'brainstem.sqlite3'
+    raw_events = parse_syslog_envelopes(
+        [
+            "Mar 22 00:00:01 fw-01 charon: VPN tunnel dropped and recovered",
+            "Mar 22 00:00:02 fw-01 charon: VPN tunnel dropped and recovered",
+        ],
+        tenant_id="client-a",
+        source_path="/var/log/syslog",
+    )
+    result = run_ingest_pipeline(raw_events, threshold=2, db_path=str(db_path))
+    removed = clear_storage_tables(str(db_path), tables=["raw_envelopes", "events"])
+    expected_signature_rows = len({sig.signature_key for sig in result.signatures})
+    assert removed["raw_envelopes"] == len(result.raw_envelope_ids)
+    assert removed["events"] == len(result.events)
+
+    counts_after = get_storage_counts(str(db_path))
+    assert counts_after["raw_envelopes"] == 0
+    assert counts_after["events"] == 0
+    assert counts_after["signatures"] == expected_signature_rows
+    assert counts_after["candidates"] == len(result.candidates)
+
+
+def test_create_sqlite_snapshot_returns_metadata_and_copies_file(tmp_path: Path) -> None:
+    db_path = tmp_path / "snapshot.sqlite3"
+    init_db(str(db_path))
+
+    first = create_sqlite_snapshot(str(db_path))
+    first_path = Path(first["snapshot_path"])
+    assert first["source_path"] == str(db_path)
+    assert first_path.exists()
+    assert first_path.is_file()
+    assert first_path != db_path
+    assert first["size"] == first_path.stat().st_size
+    assert first["size"] > 0
+    assert first["created_at"].endswith("Z")
+
+    second = create_sqlite_snapshot(str(db_path))
+    second_path = Path(second["snapshot_path"])
+    assert second_path.exists()
+    assert second["snapshot_path"] != first["snapshot_path"]