@simbimbo/brainstem 0.0.3 → 0.0.5

package/brainstem/storage.py

@@ -2,18 +2,111 @@ from __future__ import annotations
 
 import json
 import sqlite3
+from datetime import datetime, timezone
 from pathlib import Path
 from typing import Any, Iterable, List
 
 from .models import Candidate, Event, RawInputEnvelope, Signature
+from .config import resolve_default_db_path
 
 
 def default_db_path() -> Path:
-    return Path('.brainstem-state') / 'brainstem.sqlite3'
+    return Path(resolve_default_db_path())
+
+
+def _snapshot_timestamp() -> tuple[str, str]:
+    now = datetime.now(timezone.utc)
+    created_at = now.isoformat(timespec="seconds").replace("+00:00", "Z")
+    compact_timestamp = now.strftime("%Y%m%dT%H%M%S%fZ")
+    return created_at, compact_timestamp
+
+
+def create_sqlite_snapshot(db_path: str | None = None) -> dict[str, str | int]:
+    source_path = Path(db_path) if db_path else default_db_path()
+    if not source_path.exists():
+        raise FileNotFoundError(f"source database does not exist: {source_path}")
+    if not source_path.is_file():
+        raise ValueError(f"source path is not a file: {source_path}")
+
+    created_at, compact_timestamp = _snapshot_timestamp()
+    snapshot_name = f"{source_path.stem}.snapshot.{compact_timestamp}{source_path.suffix}"
+    snapshot_path = source_path.with_name(snapshot_name)
+
+    source_connection = sqlite3.connect(str(source_path))
+    snapshot_connection = sqlite3.connect(str(snapshot_path))
+    try:
+        source_connection.backup(snapshot_connection)
+    finally:
+        snapshot_connection.close()
+        source_connection.close()
+
+    return {
+        "source_path": str(source_path),
+        "snapshot_path": str(snapshot_path),
+        "size": snapshot_path.stat().st_size,
+        "created_at": created_at,
+    }
 
 
 RAW_ENVELOPE_STATUSES = ("received", "canonicalized", "parse_failed", "unsupported")
 RAW_ENVELOPE_FAILURE_STATUSES = ("parse_failed", "unsupported")
+MAINTENANCE_TABLES = ("events", "raw_envelopes", "signatures", "candidates")
+
+
+def _coerce_raw_envelope_id(value: Any) -> int | None:
+    if isinstance(value, bool):
+        return None
+    if isinstance(value, int):
+        return value
+    if isinstance(value, str):
+        value = value.strip()
+        if not value.isdigit():
+            return None
+        return int(value)
+    return None
+
+
+def _coerce_raw_envelope_id_list(raw_value: Any) -> List[int]:
+    if raw_value is None:
+        return []
+    if isinstance(raw_value, list):
+        ids = [_coerce_raw_envelope_id(item) for item in raw_value]
+        return [item for item in ids if item is not None]
+    if isinstance(raw_value, tuple):
+        ids = [_coerce_raw_envelope_id(item) for item in raw_value]
+        return [item for item in ids if item is not None]
+    return []
+
+
+def extract_source_raw_envelope_ids(metadata_json: str | None) -> List[int]:
+    if not metadata_json:
+        return []
+    try:
+        metadata = json.loads(metadata_json)
+    except json.JSONDecodeError:
+        return []
+    if not isinstance(metadata, dict):
+        return []
+    return _coerce_raw_envelope_id_list(metadata.get("source_raw_envelope_ids"))
+
+
+def resolve_maintenance_tables(table_names: Iterable[str] | None = None) -> list[str]:
+    if table_names is None:
+        return list(MAINTENANCE_TABLES)
+
+    requested: list[str] = []
+    for table_name in table_names:
+        table_name = str(table_name).strip().lower()
+        if not table_name:
+            continue
+        if table_name == "all":
+            return list(MAINTENANCE_TABLES)
+        if table_name not in MAINTENANCE_TABLES:
+            raise ValueError(f"unsupported table for maintenance clear: {table_name}")
+        if table_name not in requested:
+            requested.append(table_name)
+
+    return requested or list(MAINTENANCE_TABLES)
 
 
 def _validate_canonicalization_status(status: str) -> None:
@@ -101,6 +194,35 @@ def init_db(db_path: str | None = None) -> None:
         conn.close()
 
 
+def get_storage_counts(db_path: str | None = None) -> dict[str, int]:
+    init_db(db_path)
+    conn = connect(db_path)
+    try:
+        return {
+            table: _query_count(conn, f"SELECT COUNT(*) FROM {table}")
+            for table in MAINTENANCE_TABLES
+        }
+    finally:
+        conn.close()
+
+
+def clear_storage_tables(db_path: str | None = None, *, tables: Iterable[str] | None = None) -> dict[str, int]:
+    resolved_tables = resolve_maintenance_tables(tables)
+    init_db(db_path)
+    conn = connect(db_path)
+    try:
+        counts: dict[str, int] = {}
+        for table in resolved_tables:
+            count = _query_count(conn, f"SELECT COUNT(*) FROM {table}")
+            conn.execute(f"DELETE FROM {table}")
+            conn.execute("DELETE FROM sqlite_sequence WHERE name = ?", (table,))
+            counts[table] = count
+        conn.commit()
+        return counts
+    finally:
+        conn.close()
+
+
 def store_raw_envelopes(raw_envelopes: Iterable[RawInputEnvelope], db_path: str | None = None) -> List[int]:
     conn = connect(db_path)
     raw_ids: List[int] = []
@@ -176,17 +298,62 @@ def get_raw_envelope_by_id(raw_envelope_id: int, db_path: str | None = None) ->
         conn.close()
 
 
+def get_raw_envelopes_by_ids(
+    raw_envelope_ids: Iterable[int | str | object],
+    db_path: str | None = None,
+) -> List[sqlite3.Row]:
+    ids = list(dict.fromkeys(_coerce_raw_envelope_id_list(raw_envelope_ids)))
+    if not ids:
+        return []
+
+    conn = connect(db_path)
+    try:
+        placeholders = ",".join(["?"] * len(ids))
+        return conn.execute(
+            f"SELECT * FROM raw_envelopes WHERE id IN ({placeholders})",
+            ids,
+        ).fetchall()
+    finally:
+        conn.close()
+
+
 def _recent_raw_envelopes_query(
     canonicalization_status: str | None,
     *,
     failures_only: bool,
-) -> tuple[str, tuple[str, ...], bool]:
+    tenant_id: str | None = None,
+    source_type: str | None = None,
+    source_id: str | None = None,
+    source_path: str | None = None,
+) -> tuple[str, tuple[str, ...]]:
+    where_clauses: list[str] = []
+    args: list[str] = []
+
     if canonicalization_status is None and failures_only:
-        return "WHERE canonicalization_status IN (?, ?)", RAW_ENVELOPE_FAILURE_STATUSES, True
-    if canonicalization_status is None and not failures_only:
-        return "", (), False
-    _validate_canonicalization_status(canonicalization_status)
-    return "WHERE canonicalization_status = ?", (canonicalization_status,), False
+        where_clauses.append("canonicalization_status IN (?, ?)")
+        args.extend(RAW_ENVELOPE_FAILURE_STATUSES)
+    elif canonicalization_status is None and not failures_only:
+        pass
+    elif canonicalization_status is not None:
+        _validate_canonicalization_status(canonicalization_status)
+        where_clauses.append("canonicalization_status = ?")
+        args.append(canonicalization_status)
+
+    if tenant_id is not None:
+        where_clauses.append("tenant_id = ?")
+        args.append(tenant_id)
+    if source_type is not None:
+        where_clauses.append("source_type = ?")
+        args.append(source_type)
+    if source_id is not None:
+        where_clauses.append("source_id = ?")
+        args.append(source_id)
+    if source_path is not None:
+        where_clauses.append("source_path = ?")
+        args.append(source_path)
+
+    where_clause = f"WHERE {' AND '.join(where_clauses)}" if where_clauses else ""
+    return where_clause, tuple(args)
 
 
 def list_recent_raw_envelopes(
@@ -195,10 +362,21 @@ def list_recent_raw_envelopes(
     limit: int = 20,
     *,
     failures_only: bool = False,
+    tenant_id: str | None = None,
+    source_type: str | None = None,
+    source_id: str | None = None,
+    source_path: str | None = None,
 ) -> List[sqlite3.Row]:
     conn = connect(db_path)
     try:
-        where_clause, status_args, _ = _recent_raw_envelopes_query(status, failures_only=failures_only)
+        where_clause, status_args = _recent_raw_envelopes_query(
+            status,
+            failures_only=failures_only,
+            tenant_id=tenant_id,
+            source_type=source_type,
+            source_id=source_id,
+            source_path=source_path,
+        )
         prefix = f"{where_clause} " if where_clause else ""
         rows = conn.execute(
             f"""
@@ -214,6 +392,51 @@ def list_recent_raw_envelopes(
         conn.close()
 
 
+def list_canonical_events(
+    db_path: str | None = None,
+    limit: int = 20,
+    *,
+    tenant_id: str | None = None,
+    source_type: str | None = None,
+    host: str | None = None,
+    service: str | None = None,
+    severity: str | None = None,
+) -> List[sqlite3.Row]:
+    conn = connect(db_path)
+    try:
+        where_clauses = ["canonicalization_status = ?"]
+        args: List[str] = ["canonicalized"]
+
+        if tenant_id is not None:
+            where_clauses.append("tenant_id = ?")
+            args.append(tenant_id)
+        if source_type is not None:
+            where_clauses.append("source_type = ?")
+            args.append(source_type)
+        if host is not None:
+            where_clauses.append("host = ?")
+            args.append(host)
+        if service is not None:
+            where_clauses.append("service = ?")
+            args.append(service)
+        if severity is not None:
+            where_clauses.append("severity = ?")
+            args.append(severity)
+
+        where_clause = " WHERE " + " AND ".join(where_clauses)
+        return conn.execute(
+            f"""
+            SELECT * FROM raw_envelopes
+            {where_clause}
+            ORDER BY id DESC
+            LIMIT ?
+            """,
+            (*args, max(1, limit)),
+        ).fetchall()
+    finally:
+        conn.close()
+
+
 def list_recent_failed_raw_envelopes(
     db_path: str | None = None,
     *,
@@ -309,6 +532,72 @@ def get_source_dimension_summaries(
         conn.close()
 
 
+def get_source_status_summaries(
+    db_path: str | None = None,
+    *,
+    limit: int = 20,
+    tenant_id: str | None = None,
+    source_type: str | None = None,
+    source_id: str | None = None,
+    source_path: str | None = None,
+) -> List[dict[str, Any]]:
+    init_db(db_path)
+    conn = connect(db_path)
+    try:
+        query = """
+            SELECT
+                tenant_id,
+                source_type,
+                source_id,
+                source_path,
+                COUNT(*) AS raw_count,
+                SUM(CASE WHEN canonicalization_status = 'canonicalized' THEN 1 ELSE 0 END) AS canonicalized_count,
+                SUM(CASE WHEN canonicalization_status = 'parse_failed' THEN 1 ELSE 0 END) AS parse_failed_count,
+                SUM(CASE WHEN canonicalization_status = 'unsupported' THEN 1 ELSE 0 END) AS unsupported_count,
+                MIN(timestamp) AS first_seen_at,
+                MAX(timestamp) AS last_seen_at
+            FROM raw_envelopes
+            WHERE 1 = 1
+        """
+        args: list[Any] = []
+        if tenant_id is not None:
+            query += " AND tenant_id = ?"
+            args.append(tenant_id)
+        if source_type is not None:
+            query += " AND source_type = ?"
+            args.append(source_type)
+        if source_id is not None:
+            query += " AND source_id = ?"
+            args.append(source_id)
+        if source_path is not None:
+            query += " AND source_path = ?"
+            args.append(source_path)
+
+        query += """
+            GROUP BY tenant_id, source_type, source_id, source_path
+            ORDER BY last_seen_at DESC, raw_count DESC
+            LIMIT ?
+        """
+        args.append(max(1, limit))
+        return [
+            {
+                "tenant_id": row["tenant_id"],
+                "source_type": row["source_type"] or "",
+                "source_id": row["source_id"] or "",
+                "source_path": row["source_path"] or "",
+                "raw_count": int(row["raw_count"]),
+                "canonicalized_count": int(row["canonicalized_count"] or 0),
+                "parse_failed_count": int(row["parse_failed_count"] or 0),
+                "unsupported_count": int(row["unsupported_count"] or 0),
+                "first_seen_at": row["first_seen_at"],
+                "last_seen_at": row["last_seen_at"],
+            }
+            for row in conn.execute(query, args).fetchall()
+        ]
+    finally:
+        conn.close()
+
+
 def _get_source_dimension_summaries_from_conn(
     conn: sqlite3.Connection,
     *,
@@ -358,6 +647,27 @@ def store_signatures(signatures: Iterable[Signature], db_path: str | None = None
     count = 0
     try:
         for signature in signatures:
+            row = conn.execute(
+                "SELECT metadata_json FROM signatures WHERE signature_key = ?",
+                (signature.signature_key,),
+            ).fetchone()
+
+            metadata = dict(signature.metadata)
+            raw_ids = _coerce_raw_envelope_id_list(metadata.get("source_raw_envelope_ids"))
+            if not raw_ids:
+                raw_id = _coerce_raw_envelope_id(metadata.get("source_raw_envelope_id"))
+                if raw_id is not None:
+                    raw_ids = [raw_id]
+            metadata.pop("source_raw_envelope_id", None)
+
+            if row is not None:
+                existing_metadata = json.loads(row["metadata_json"] or "{}")
+                if not isinstance(existing_metadata, dict):
+                    existing_metadata = {}
+                existing_raw_ids = _coerce_raw_envelope_id_list(existing_metadata.get("source_raw_envelope_ids"))
+                metadata = dict(existing_metadata) | dict(metadata)
+                metadata["source_raw_envelope_ids"] = sorted(set(existing_raw_ids + raw_ids))
+
             conn.execute(
                 '''
                 INSERT INTO signatures (
@@ -373,7 +683,7 @@ def store_signatures(signatures: Iterable[Signature], db_path: str | None = None
                     signature.event_family,
                     signature.normalized_pattern,
                     signature.service,
-                    json.dumps(signature.metadata, ensure_ascii=False),
+                    json.dumps(metadata, ensure_ascii=False),
                 ),
             )
             count += 1
@@ -416,12 +726,79 @@ def store_candidates(candidates: Iterable[Candidate], db_path: str | None = None
         conn.close()
 
 
-def list_candidates(db_path: str | None = None, limit: int = 20) -> List[sqlite3.Row]:
+def list_candidates(
+    db_path: str | None = None,
+    limit: int = 20,
+    *,
+    candidate_type: str | None = None,
+    decision_band: str | None = None,
+    min_score_total: float | None = None,
+) -> List[sqlite3.Row]:
     conn = connect(db_path)
     try:
+        where_clauses: List[str] = []
+        args: List[Any] = []
+
+        if candidate_type is not None:
+            where_clauses.append("candidate_type = ?")
+            args.append(candidate_type)
+        if decision_band is not None:
+            where_clauses.append("decision_band = ?")
+            args.append(decision_band)
+        if min_score_total is not None:
+            where_clauses.append("score_total >= ?")
+            args.append(min_score_total)
+
+        where_clause = ""
+        if where_clauses:
+            where_clause = " WHERE " + " AND ".join(where_clauses)
+
         rows = conn.execute(
-            'SELECT * FROM candidates ORDER BY score_total DESC, id DESC LIMIT ?',
-            (max(1, limit),),
+            f'SELECT * FROM candidates{where_clause} ORDER BY score_total DESC, id DESC LIMIT ?',
+            (*args, max(1, limit)),
+        ).fetchall()
+        return rows
+    finally:
+        conn.close()
+
+
+def list_signatures(
+    db_path: str | None = None,
+    limit: int = 20,
+    *,
+    event_family: str | None = None,
+    service: str | None = None,
+    min_occurrence_count: int | None = None,
+) -> List[sqlite3.Row]:
+    conn = connect(db_path)
+    try:
+        where_clauses: List[str] = []
+        args: List[Any] = []
+
+        if event_family is not None:
+            where_clauses.append("event_family = ?")
+            args.append(event_family)
+        if service is not None:
+            where_clauses.append("service = ?")
+            args.append(service)
+        if min_occurrence_count is not None:
+            where_clauses.append("occurrence_count >= ?")
+            args.append(min_occurrence_count)
+
+        where_clause = ""
+        if where_clauses:
+            where_clause = " WHERE " + " AND ".join(where_clauses)
+
+        rows = conn.execute(
+            f"""
+            SELECT
+                id, signature_key, event_family, normalized_pattern, service,
+                metadata_json, occurrence_count
+            FROM signatures{where_clause}
+            ORDER BY occurrence_count DESC, id DESC
+            LIMIT ?
+            """,
+            (*args, max(1, limit)),
         ).fetchall()
         return rows
     finally:

package/docs/README.md

@@ -0,0 +1,103 @@
+# Runtime Examples
+
+Use this compact surface for the implemented runtime API, listener, file-ingest, and LogicMonitor-shaped webhook path.
+
+## 0) Shared runtime settings
+
+```bash
+export BRAINSTEM_API_TOKEN=my-local-token  # optional: set only if you want auth required
+export BRAINSTEM_DB_PATH=/tmp/brainstem.sqlite3
+```
+
+`BRAINSTEM_API_TOKEN` is optional. If you do not set it, omit all `X-API-Token` headers in the API examples.
+
+## 1) API entry point
+
+```bash
+# Starts the runtime API
+python -m uvicorn brainstem.api:app --host 127.0.0.1 --port 8000
+```
+
+```bash
+curl -s http://127.0.0.1:8000/healthz
+```
+
+## 2) UDP listener entry point
+
+```bash
+# Prints canonicalized events for each received datagram
+python -m brainstem.listener --tenant demo-tenant --host 127.0.0.1 --port 5514 --source-path /var/log/syslog
+```
+
+```bash
+printf 'Mar 22 03:10:00 fw-01 charon: IPsec SA rekey succeeded\n' | nc -u 127.0.0.1 5514
+```
+
+## 3) API ingest (syslog payload style)
+
+```bash
+curl -s -X POST http://127.0.0.1:8000/ingest/event \
+  -H "Content-Type: application/json" \
+  -H "X-API-Token: $BRAINSTEM_API_TOKEN" \
+  -d '{"tenant_id":"demo-tenant","source_type":"syslog","source_path":"/var/log/syslog","message_raw":"Mar 22 03:11:00 fw-01 charon: child SA rekey started"}'
+```
+
+## 4) API ingest for file source events
+
+```bash
+curl -s -X POST http://127.0.0.1:8000/ingest/batch \
+  -H "Content-Type: application/json" \
+  -H "X-API-Token: $BRAINSTEM_API_TOKEN" \
+  -d '{"threshold":2,"db_path":"/tmp/brainstem.sqlite3","events":[{"tenant_id":"demo-tenant","source_type":"file","source_path":"/tmp/manual.log","message_raw":"vpn tunnel dropped and recovered"}]}'
+```
+
+## 5) LogicMonitor-shaped ingest (connector payloads)
+
+```bash
+curl -s -X POST http://127.0.0.1:8000/ingest/logicmonitor \
+  -H "Content-Type: application/json" \
+  -H "X-API-Token: $BRAINSTEM_API_TOKEN" \
+  -d '{"tenant_id":"demo-tenant","threshold":2,"source_path":"/logicmonitor/ingest","db_path":"/tmp/brainstem.sqlite3","events":[{"resource_id":123,"resource_name":"edge-fw-01","message_raw":"VPN tunnel dropped and recovered","severity":"warning","metadata":{"datasource":"IPSec Tunnel"}}]}'
+```
+
+## 6) Runtime inspection endpoints (same db path)
+
+```bash
+curl -s "http://127.0.0.1:8000/ingest/recent?db_path=/tmp/brainstem.sqlite3&limit=5" \
+  -H "X-API-Token: $BRAINSTEM_API_TOKEN"
+curl -s "http://127.0.0.1:8000/candidates?db_path=/tmp/brainstem.sqlite3&limit=5" \
+  -H "X-API-Token: $BRAINSTEM_API_TOKEN"
+curl -s "http://127.0.0.1:8000/signatures?db_path=/tmp/brainstem.sqlite3&limit=5" \
+  -H "X-API-Token: $BRAINSTEM_API_TOKEN"
+curl -s "http://127.0.0.1:8000/raw_envelopes?db_path=/tmp/brainstem.sqlite3&limit=5" \
+  -H "X-API-Token: $BRAINSTEM_API_TOKEN"
+curl -s "http://127.0.0.1:8000/stats?db_path=/tmp/brainstem.sqlite3" \
+  -H "X-API-Token: $BRAINSTEM_API_TOKEN"
+curl -s "http://127.0.0.1:8000/failures?db_path=/tmp/brainstem.sqlite3&limit=5" \
+  -H "X-API-Token: $BRAINSTEM_API_TOKEN"
+curl -s "http://127.0.0.1:8000/sources?db_path=/tmp/brainstem.sqlite3&limit=5" \
+  -H "X-API-Token: $BRAINSTEM_API_TOKEN"
+curl -s "http://127.0.0.1:8000/sources/status?db_path=/tmp/brainstem.sqlite3&limit=5" \
+  -H "X-API-Token: $BRAINSTEM_API_TOKEN"
+```
+
+## 7) Direct file ingest helper path
+
+```bash
+python - <<'PY'
+from brainstem.ingest import run_ingest_file
+
+result = run_ingest_file(
+    "tests/fixtures/sample_syslog.log",
+    tenant_id="demo-tenant",
+    threshold=2,
+    db_path="/tmp/brainstem.sqlite3",
+)
+print({
+    "events": len(result.events),
+    "signatures": len(result.signatures),
+    "candidates": len(result.candidates),
+    "parse_failed": result.parse_failed,
+})
+PY
+```