@simbimbo/brainstem 0.0.2 → 0.0.3

package/CHANGELOG.md

@@ -1,5 +1,29 @@
 # Changelog
 
+## 0.0.3 — 2026-03-22
+
+Intake Foundation follow-up release for **brAInstem**.
+
+### Highlights
+- persists `RawInputEnvelope` intake records to SQLite before canonicalization
+- records canonicalization outcomes explicitly (`received`, `canonicalized`, `parse_failed`, `unsupported`)
+- adds ingest accounting for:
+  - received
+  - canonicalized
+  - parse_failed
+  - candidates_generated
+- adds runtime inspection endpoints for intake trust and observability:
+  - `GET /stats`
+  - `GET /failures`
+  - `GET /failures/{id}`
+  - `GET /ingest/recent`
+  - `GET /sources`
+- adds storage/query helpers for recent raw envelopes, recent failures, and per-source summaries
+- expands tests around raw-envelope persistence, failure inspection, source summaries, and stats
+
+### Validation
+- local test suite passed (`26 passed`)
+
 ## 0.0.2 — 2026-03-22
 
 First fully aligned public foundation release of **brAInstem**.

package/brainstem/__init__.py

@@ -1,3 +1,3 @@
 """brAInstem — operational memory for weak signals."""
 
-__version__ = "0.0.2"
+__version__ = "0.0.3"

package/brainstem/api.py

@@ -8,11 +8,25 @@ from fastapi import FastAPI, HTTPException, Query
 from fastapi.responses import JSONResponse
 from pydantic import BaseModel, Field
 
-from .ingest import canonicalize_raw_input_envelopes
+from .ingest import canonicalize_raw_input_envelope
 from .interesting import interesting_items
 from .models import Candidate, RawInputEnvelope
 from .recurrence import build_recurrence_candidates
-from .storage import init_db, list_candidates, store_candidates, store_events, store_signatures
+from .storage import (
+    RAW_ENVELOPE_STATUSES,
+    get_ingest_stats,
+    init_db,
+    list_candidates,
+    get_raw_envelope_by_id,
+    get_source_dimension_summaries,
+    list_recent_failed_raw_envelopes,
+    list_recent_raw_envelopes,
+    set_raw_envelope_status,
+    store_candidates,
+    store_events,
+    store_raw_envelopes,
+    store_signatures,
+)
 from .ingest import signatures_for_events
 
 
@@ -22,6 +36,8 @@ app = FastAPI(title="brAInstem Runtime")
 class RawEnvelopeRequest(BaseModel):
     tenant_id: str
     source_type: str
+    source_id: str = ""
+    source_name: str = ""
     message_raw: str
     timestamp: Optional[str] = None
     host: str = ""
@@ -49,6 +65,8 @@ def _raw_envelope_from_request(payload: RawEnvelopeRequest) -> RawInputEnvelope:
     return RawInputEnvelope(
         tenant_id=payload.tenant_id,
         source_type=payload.source_type,
+        source_id=payload.source_id,
+        source_name=payload.source_name,
         timestamp=payload.timestamp or datetime.utcnow().isoformat() + "Z",
         message_raw=payload.message_raw,
         host=payload.host,
@@ -78,15 +96,68 @@ def _candidate_from_row(row) -> Candidate:
     )
 
 
+def _raw_envelope_from_row(row) -> Dict[str, Any]:
+    return {
+        "id": row["id"],
+        "tenant_id": row["tenant_id"],
+        "source_type": row["source_type"],
+        "source_id": row["source_id"],
+        "source_name": row["source_name"],
+        "timestamp": row["timestamp"],
+        "host": row["host"],
+        "service": row["service"],
+        "severity": row["severity"],
+        "asset_id": row["asset_id"],
+        "source_path": row["source_path"],
+        "facility": row["facility"],
+        "message_raw": row["message_raw"],
+        "structured_fields": json.loads(row["structured_fields_json"] or "{}"),
+        "correlation_keys": json.loads(row["correlation_keys_json"] or "{}"),
+        "metadata": json.loads(row["metadata_json"] or "{}"),
+        "canonicalization_status": row["canonicalization_status"],
+        "failure_reason": row["failure_reason"],
+    }
+
+
 def _run_ingest_batch(raw_events: List[RawInputEnvelope], *, threshold: int, db_path: Optional[str]) -> Dict[str, Any]:
-    events = canonicalize_raw_input_envelopes(raw_events)
+    raw_envelope_ids: List[int] = []
+    if db_path:
+        init_db(db_path)
+        raw_envelope_ids = store_raw_envelopes(raw_events, db_path)
+
+    events = []
+    parse_failed = 0
+    for idx, raw_event in enumerate(raw_events):
+        raw_envelope_id = raw_envelope_ids[idx] if idx < len(raw_envelope_ids) else None
+        try:
+            canonical_event = canonicalize_raw_input_envelope(raw_event)
+        except Exception as exc:
+            parse_failed += 1
+            if raw_envelope_id is not None:
+                set_raw_envelope_status(
+                    raw_envelope_id,
+                    "parse_failed",
+                    db_path=db_path,
+                    failure_reason=str(exc),
+                )
+            continue
+        events.append(canonical_event)
+        if raw_envelope_id is not None:
+            set_raw_envelope_status(raw_envelope_id, "canonicalized", db_path=db_path)
+
     if not events:
-        return {"ok": True, "event_count": 0, "signature_count": 0, "candidate_count": 0, "interesting_items": []}
+        return {
+            "ok": True,
+            "event_count": 0,
+            "signature_count": 0,
+            "candidate_count": 0,
+            "parse_failed": parse_failed,
+            "interesting_items": [],
+        }
 
     signatures = signatures_for_events(events)
     candidates = build_recurrence_candidates(events, signatures, threshold=threshold)
     if db_path:
-        init_db(db_path)
         store_events(events, db_path)
         store_signatures(signatures, db_path)
         store_candidates(candidates, db_path)
@@ -97,6 +168,7 @@ def _run_ingest_batch(raw_events: List[RawInputEnvelope], *, threshold: int, db_
         "event_count": len(events),
         "signature_count": len({sig.signature_key for sig in signatures}),
         "candidate_count": len(candidates),
+        "parse_failed": parse_failed,
         "interesting_items": interesting_items(candidates, limit=max(1, 5)),
     }
 
@@ -126,6 +198,60 @@ def get_interesting(
     return {"ok": True, "items": interesting_items(candidates, limit=limit)}
 
 
+@app.get("/stats")
+def get_stats(db_path: Optional[str] = None) -> Dict[str, Any]:
+    return {"ok": True, **get_ingest_stats(db_path)}
+
+
+@app.get("/failures")
+def get_failures(
+    limit: int = Query(default=20, ge=1),
+    status: Optional[str] = None,
+    db_path: Optional[str] = None,
+) -> Dict[str, Any]:
+    if status is not None and status not in RAW_ENVELOPE_STATUSES:
+        raise HTTPException(
+            status_code=422,
+            detail=f"invalid status '{status}'; expected one of: {', '.join(RAW_ENVELOPE_STATUSES)}",
+        )
+
+    rows = list_recent_failed_raw_envelopes(db_path=db_path, status=status, limit=limit)
+    items = [_raw_envelope_from_row(row) for row in rows]
+    return {"ok": True, "items": items, "count": len(items), "status": status}
+
+
+@app.get("/ingest/recent")
+def get_ingest_recent(
+    limit: int = Query(default=20, ge=1),
+    status: Optional[str] = None,
+    db_path: Optional[str] = None,
+) -> Dict[str, Any]:
+    if status is not None and status not in RAW_ENVELOPE_STATUSES:
+        raise HTTPException(
+            status_code=422,
+            detail=f"invalid status '{status}'; expected one of: {', '.join(RAW_ENVELOPE_STATUSES)}",
+        )
+    rows = list_recent_raw_envelopes(db_path=db_path, status=status, limit=limit, failures_only=False)
+    items = [_raw_envelope_from_row(row) for row in rows]
+    return {"ok": True, "items": items, "count": len(items), "status": status}
+
+
+@app.get("/sources")
+def get_sources(
+    limit: int = Query(default=10, ge=1),
+    db_path: Optional[str] = None,
+) -> Dict[str, Any]:
+    return {"ok": True, "items": get_source_dimension_summaries(db_path=db_path, limit=limit)}
+
+
+@app.get("/failures/{raw_envelope_id}")
+def get_failure(raw_envelope_id: int, db_path: Optional[str] = None) -> Dict[str, Any]:
+    row = get_raw_envelope_by_id(raw_envelope_id, db_path=db_path)
+    if row is None:
+        raise HTTPException(status_code=404, detail="raw envelope not found")
+    return {"ok": True, "item": _raw_envelope_from_row(row)}
+
+
 @app.get("/healthz")
 def healthz() -> Dict[str, str]:
     return JSONResponse(content={"ok": True, "status": "ok"})

package/brainstem/ingest.py

@@ -50,6 +50,13 @@ def parse_syslog_envelopes(lines: Iterable[str], *, tenant_id: str, source_path:
 
 
 def canonicalize_raw_input_envelope(raw: RawInputEnvelope) -> CanonicalEvent:
+    parse_error = (raw.metadata or {}).get("parse_error")
+    if parse_error:
+        raise ValueError(f"parse_error: {parse_error}")
+
+    if not (raw.message_raw or "").strip():
+        raise ValueError("message_raw is empty and cannot be canonicalized")
+
     message_normalized = normalize_message(raw.message_raw)
     metadata = dict(raw.metadata or {})
     metadata.setdefault("canonicalization_source", raw.source_type)

package/brainstem/models.py

@@ -10,6 +10,8 @@ class RawInputEnvelope:
     source_type: str
     timestamp: str
     message_raw: str
+    source_id: str = ""
+    source_name: str = ""
     host: str = ""
     service: str = ""
     severity: str = "info"

package/brainstem/storage.py

@@ -2,17 +2,25 @@ from __future__ import annotations
 
 import json
 import sqlite3
-from dataclasses import asdict
 from pathlib import Path
-from typing import Iterable, List
+from typing import Any, Iterable, List
 
-from .models import Candidate, Event, Signature
+from .models import Candidate, Event, RawInputEnvelope, Signature
 
 
 def default_db_path() -> Path:
     return Path('.brainstem-state') / 'brainstem.sqlite3'
 
 
+RAW_ENVELOPE_STATUSES = ("received", "canonicalized", "parse_failed", "unsupported")
+RAW_ENVELOPE_FAILURE_STATUSES = ("parse_failed", "unsupported")
+
+
+def _validate_canonicalization_status(status: str) -> None:
+    if status not in RAW_ENVELOPE_STATUSES:
+        raise ValueError(f"unsupported canonicalization_status: {status}")
+
+
 def connect(db_path: str | None = None) -> sqlite3.Connection:
     path = Path(db_path) if db_path else default_db_path()
     path.parent.mkdir(parents=True, exist_ok=True)
@@ -42,6 +50,27 @@ def init_db(db_path: str | None = None) -> None:
                 correlation_keys_json TEXT NOT NULL
             );
 
+            CREATE TABLE IF NOT EXISTS raw_envelopes (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                tenant_id TEXT NOT NULL,
+                source_type TEXT NOT NULL,
+                source_id TEXT,
+                source_name TEXT,
+                timestamp TEXT NOT NULL,
+                host TEXT,
+                service TEXT,
+                severity TEXT,
+                asset_id TEXT,
+                source_path TEXT,
+                facility TEXT,
+                message_raw TEXT NOT NULL,
+                structured_fields_json TEXT NOT NULL,
+                correlation_keys_json TEXT NOT NULL,
+                metadata_json TEXT NOT NULL,
+                canonicalization_status TEXT NOT NULL DEFAULT 'received',
+                failure_reason TEXT
+            );
+
             CREATE TABLE IF NOT EXISTS signatures (
                 id INTEGER PRIMARY KEY AUTOINCREMENT,
                 signature_key TEXT NOT NULL UNIQUE,
@@ -72,6 +101,133 @@ def init_db(db_path: str | None = None) -> None:
         conn.close()
 
 
+def store_raw_envelopes(raw_envelopes: Iterable[RawInputEnvelope], db_path: str | None = None) -> List[int]:
+    conn = connect(db_path)
+    raw_ids: List[int] = []
+    try:
+        for raw in raw_envelopes:
+            cursor = conn.execute(
+                '''
+                INSERT INTO raw_envelopes (
+                    tenant_id, source_type, source_id, source_name, timestamp, host, service, severity,
+                    asset_id, source_path, facility, message_raw,
+                    structured_fields_json, correlation_keys_json, metadata_json,
+                    canonicalization_status, failure_reason
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                ''',
+                (
+                    raw.tenant_id,
+                    raw.source_type,
+                    raw.source_id,
+                    raw.source_name,
+                    raw.timestamp,
+                    raw.host,
+                    raw.service,
+                    raw.severity,
+                    raw.asset_id,
+                    raw.source_path,
+                    raw.facility,
+                    raw.message_raw,
+                    json.dumps(raw.structured_fields, ensure_ascii=False),
+                    json.dumps(raw.correlation_keys, ensure_ascii=False),
+                    json.dumps(raw.metadata, ensure_ascii=False),
+                    "received",
+                    None,
+                ),
+            )
+            raw_ids.append(int(cursor.lastrowid))
+        conn.commit()
+        return raw_ids
+    finally:
+        conn.close()
+
+
+def set_raw_envelope_status(
+    raw_envelope_id: int,
+    status: str,
+    db_path: str | None = None,
+    *,
+    failure_reason: str | None = None,
+) -> None:
+    _validate_canonicalization_status(status)
+    conn = connect(db_path)
+    try:
+        conn.execute(
+            '''
+            UPDATE raw_envelopes
+            SET canonicalization_status = ?, failure_reason = ?
+            WHERE id = ?
+            ''',
+            (status, failure_reason, raw_envelope_id),
+        )
+        conn.commit()
+    finally:
+        conn.close()
+
+
+def get_raw_envelope_by_id(raw_envelope_id: int, db_path: str | None = None) -> sqlite3.Row | None:
+    conn = connect(db_path)
+    try:
+        return conn.execute(
+            "SELECT * FROM raw_envelopes WHERE id = ?",
+            (raw_envelope_id,),
+        ).fetchone()
+    finally:
+        conn.close()
+
+
+def _recent_raw_envelopes_query(
+    canonicalization_status: str | None,
+    *,
+    failures_only: bool,
+) -> tuple[str, tuple[str, ...], bool]:
+    if canonicalization_status is None and failures_only:
+        return "WHERE canonicalization_status IN (?, ?)", RAW_ENVELOPE_FAILURE_STATUSES, True
+    if canonicalization_status is None and not failures_only:
+        return "", (), False
+    _validate_canonicalization_status(canonicalization_status)
+    return "WHERE canonicalization_status = ?", (canonicalization_status,), False
+
+
+def list_recent_raw_envelopes(
+    db_path: str | None = None,
+    status: str | None = None,
+    limit: int = 20,
+    *,
+    failures_only: bool = False,
+) -> List[sqlite3.Row]:
+    conn = connect(db_path)
+    try:
+        where_clause, status_args, _ = _recent_raw_envelopes_query(status, failures_only=failures_only)
+        prefix = f"{where_clause} " if where_clause else ""
+        rows = conn.execute(
+            f"""
+            SELECT * FROM raw_envelopes
+            {prefix}
+            ORDER BY id DESC
+            LIMIT ?
+            """,
+            (*status_args, max(1, limit)),
+        ).fetchall()
+        return rows
+    finally:
+        conn.close()
+
+
+def list_recent_failed_raw_envelopes(
+    db_path: str | None = None,
+    *,
+    status: str | None = None,
+    limit: int = 20,
+) -> List[sqlite3.Row]:
+    return list_recent_raw_envelopes(
+        db_path=db_path,
+        status=status,
+        limit=limit,
+        failures_only=status is None,
+    )
+
+
 def store_events(events: Iterable[Event], db_path: str | None = None) -> int:
     conn = connect(db_path)
     count = 0
@@ -107,6 +263,96 @@ def store_events(events: Iterable[Event], db_path: str | None = None) -> int:
         conn.close()
 
 
+SOURCE_SUMMARY_DIMENSIONS = (
+    "source_type",
+    "source_path",
+    "source_id",
+    "source_name",
+    "host",
+    "service",
+)
+
+
+def _summarize_raw_envelopes_by_dimension(
+    conn: sqlite3.Connection,
+    dimension: str,
+    limit: int = 20,
+) -> List[dict[str, int | str]]:
+    if dimension not in SOURCE_SUMMARY_DIMENSIONS:
+        raise ValueError(f"unsupported dimension: {dimension}")
+    return [
+        {"value": row["value"], "count": int(row["count"])}
+        for row in conn.execute(
+            f"""
+            SELECT {dimension} AS value, COUNT(*) AS count
+            FROM raw_envelopes
+            WHERE COALESCE(TRIM({dimension}), '') <> ''
+            GROUP BY {dimension}
+            ORDER BY COUNT(*) DESC, value ASC
+            LIMIT ?
+            """,
+            (max(1, limit),),
+        ).fetchall()
+    ]
+
+
+def get_source_dimension_summaries(
+    db_path: str | None = None,
+    *,
+    limit: int = 20,
+) -> dict[str, List[dict[str, int | str]]]:
+    init_db(db_path)
+    conn = connect(db_path)
+    try:
+        return _get_source_dimension_summaries_from_conn(conn, limit=limit)
+    finally:
+        conn.close()
+
+
+def _get_source_dimension_summaries_from_conn(
+    conn: sqlite3.Connection,
+    *,
+    limit: int = 20,
+) -> dict[str, List[dict[str, int | str]]]:
+    return {
+        "source_type": _summarize_raw_envelopes_by_dimension(conn, "source_type", limit=limit),
+        "source_path": _summarize_raw_envelopes_by_dimension(conn, "source_path", limit=limit),
+        "source_id": _summarize_raw_envelopes_by_dimension(conn, "source_id", limit=limit),
+        "source_name": _summarize_raw_envelopes_by_dimension(conn, "source_name", limit=limit),
+        "host": _summarize_raw_envelopes_by_dimension(conn, "host", limit=limit),
+        "service": _summarize_raw_envelopes_by_dimension(conn, "service", limit=limit),
+    }
+
+
+def _query_count(conn: sqlite3.Connection, query: str) -> int:
+    return int(conn.execute(query).fetchone()[0])
+
+
+def get_ingest_stats(db_path: str | None = None) -> dict[str, Any]:
+    init_db(db_path)
+    conn = connect(db_path)
+    try:
+        return {
+            "received": _query_count(conn, "SELECT COUNT(*) FROM raw_envelopes"),
+            "canonicalized": _query_count(
+                conn,
+                "SELECT COUNT(*) FROM raw_envelopes WHERE canonicalization_status = 'canonicalized'",
+            ),
+            "parse_failed": _query_count(
+                conn,
+                "SELECT COUNT(*) FROM raw_envelopes WHERE canonicalization_status = 'parse_failed'",
+            ),
+            "unsupported": _query_count(
+                conn,
+                "SELECT COUNT(*) FROM raw_envelopes WHERE canonicalization_status = 'unsupported'",
+            ),
+            "candidates_generated": _query_count(conn, "SELECT COUNT(*) FROM candidates"),
+            "source_summaries": _get_source_dimension_summaries_from_conn(conn),
+        }
+    finally:
+        conn.close()
+
+
 def store_signatures(signatures: Iterable[Signature], db_path: str | None = None) -> int:
     conn = connect(db_path)
     count = 0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@simbimbo/brainstem",
-  "version": "0.0.2",
+  "version": "0.0.3",
   "description": "brAInstem — operational memory for weak signals.",
   "license": "MIT",
   "type": "module",

package/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "brainstem"
-version = "0.0.2"
+version = "0.0.3"
 description = "brAInstem — operational memory for weak signals."
 readme = "README.md"
 requires-python = ">=3.9"

package/tests/test_api.py

@@ -3,6 +3,12 @@ from pathlib import Path
 from fastapi.testclient import TestClient
 
 from brainstem.api import app
+from brainstem.models import RawInputEnvelope
+from brainstem.storage import (
+    init_db,
+    set_raw_envelope_status,
+    store_raw_envelopes,
+)
 
 
 def test_ingest_event_endpoint_round_trip(tmp_path: Path) -> None:
@@ -65,8 +71,249 @@ def test_ingest_batch_and_interesting(tmp_path: Path) -> None:
     assert interesting_payload["items"]
 
 
+def test_stats_after_successful_and_failed_ingest(tmp_path: Path) -> None:
+    client = TestClient(app)
+    db_path = tmp_path / "brainstem.sqlite3"
+    batch_response = client.post(
+        "/ingest/batch",
+        json={
+            "threshold": 2,
+            "db_path": str(db_path),
+            "events": [
+                {
+                    "tenant_id": "client-a",
+                    "source_type": "syslog",
+                    "message_raw": "Failed password for admin from 10.1.2.3",
+                    "host": "fw-01",
+                    "service": "sshd",
+                },
+                {
+                    "tenant_id": "client-a",
+                    "source_type": "syslog",
+                    "message_raw": "Failed password for admin from 10.1.2.3",
+                    "host": "fw-01",
+                    "service": "sshd",
+                },
+                {
+                    "tenant_id": "client-a",
+                    "source_type": "syslog",
+                    "message_raw": "",
+                    "host": "fw-01",
+                    "service": "sshd",
+                },
+            ],
+        },
+    )
+    assert batch_response.status_code == 200
+    batch_payload = batch_response.json()
+    assert batch_payload["ok"] is True
+    assert batch_payload["event_count"] == 2
+    assert batch_payload["parse_failed"] == 1
+
+    stats = client.get(f"/stats?db_path={db_path}")
+    assert stats.status_code == 200
+    stats_payload = stats.json()
+    assert stats_payload["ok"] is True
+    assert stats_payload["received"] == 3
+    assert stats_payload["canonicalized"] == 2
+    assert stats_payload["parse_failed"] == 1
+    assert stats_payload["candidates_generated"] >= 1
+
+
 def test_healthz_is_ready() -> None:
     client = TestClient(app)
     response = client.get("/healthz")
     assert response.status_code == 200
     assert response.json()["ok"] is True
+
+
+def test_failures_endpoint_lists_recent_parse_failures(tmp_path: Path) -> None:
+    client = TestClient(app)
+    db_path = tmp_path / "brainstem.sqlite3"
+    client.post(
+        "/ingest/batch",
+        json={
+            "threshold": 2,
+            "db_path": str(db_path),
+            "events": [
+                {
+                    "tenant_id": "client-a",
+                    "source_type": "syslog",
+                    "message_raw": "",
+                    "host": "fw-01",
+                    "service": "sshd",
+                },
+                {
+                    "tenant_id": "client-a",
+                    "source_type": "syslog",
+                    "message_raw": "VPN tunnel dropped and recovered",
+                    "host": "fw-01",
+                    "service": "charon",
+                },
+            ],
+        },
+    )
+
+    response = client.get(f"/failures?db_path={db_path}&limit=10")
+    assert response.status_code == 200
+    payload = response.json()
+    assert payload["ok"] is True
+    assert payload["count"] == 1
+    assert payload["items"][0]["canonicalization_status"] == "parse_failed"
+
+
+def test_failures_endpoint_filters_by_status_and_fetches_single_record(tmp_path: Path) -> None:
+    client = TestClient(app)
+    db_path = tmp_path / "brainstem.sqlite3"
+    init_db(str(db_path))
+    raw_ids = store_raw_envelopes(
+        [
+            RawInputEnvelope(
+                tenant_id="client-a",
+                source_type="syslog",
+                timestamp="2026-03-22T00:00:01Z",
+                message_raw="first",
+                host="fw-01",
+                service="sshd",
+            ),
+            RawInputEnvelope(
+                tenant_id="client-a",
+                source_type="syslog",
+                timestamp="2026-03-22T00:00:02Z",
+                message_raw="second",
+                host="fw-01",
+                service="sshd",
+            ),
+        ],
+        db_path=str(db_path),
+    )
+    set_raw_envelope_status(raw_ids[0], "parse_failed", db_path=str(db_path), failure_reason="bad parse")
+    set_raw_envelope_status(raw_ids[1], "unsupported", db_path=str(db_path), failure_reason="unsupported source")
+
+    failed_only = client.get(f"/failures?db_path={db_path}&status=parse_failed&limit=10")
+    assert failed_only.status_code == 200
+    failed_payload = failed_only.json()
+    assert failed_payload["count"] == 1
+    assert failed_payload["items"][0]["id"] == raw_ids[0]
+
+    unsupported = client.get(f"/failures?db_path={db_path}&status=unsupported&limit=10")
+    assert unsupported.status_code == 200
+    unsupported_payload = unsupported.json()
+    assert unsupported_payload["count"] == 1
+    assert unsupported_payload["items"][0]["id"] == raw_ids[1]
+
+    single = client.get(f"/failures/{raw_ids[1]}?db_path={db_path}")
+    assert single.status_code == 200
+    single_payload = single.json()
+    assert single_payload["ok"] is True
+    assert single_payload["item"]["id"] == raw_ids[1]
+    assert single_payload["item"]["failure_reason"] == "unsupported source"
+
+    invalid = client.get(f"/failures?db_path={db_path}&status=bogus")
+    assert invalid.status_code == 422
+
+
+def test_sources_endpoint_summarizes_ingest_dimensions(tmp_path: Path) -> None:
+    client = TestClient(app)
+    db_path = tmp_path / "brainstem.sqlite3"
+    batch_response = client.post(
+        "/ingest/batch",
+        json={
+            "threshold": 1,
+            "db_path": str(db_path),
+            "events": [
+                {
+                    "tenant_id": "client-a",
+                    "source_type": "syslog",
+                    "source_id": "fw-01",
+                    "source_name": "edge-fw-01",
+                    "source_path": "/var/log/syslog",
+                    "message_raw": "Failed password for admin from 10.1.2.3",
+                    "host": "fw-01",
+                    "service": "sshd",
+                    "severity": "info",
+                },
+                {
+                    "tenant_id": "client-a",
+                    "source_type": "syslog",
+                    "source_id": "fw-01",
+                    "source_name": "edge-fw-01",
+                    "source_path": "/var/log/syslog",
+                    "message_raw": "Failed password for admin from 10.1.2.3",
+                    "host": "fw-01",
+                    "service": "sshd",
+                    "severity": "info",
+                },
+                {
+                    "tenant_id": "client-a",
+                    "source_type": "logicmonitor",
+                    "source_id": "lm-01",
+                    "source_name": "edge-lm-01",
+                    "source_path": "/alerts",
+                    "message_raw": "Disk space low",
+                    "host": "lm-01",
+                    "service": "logicmonitor",
+                    "severity": "warning",
+                },
+            ],
+        },
+    )
+    assert batch_response.status_code == 200
+
+    response = client.get(f"/sources?db_path={db_path}&limit=10")
+    assert response.status_code == 200
+    payload = response.json()
+    assert payload["ok"] is True
+    assert payload["items"]["source_type"] == [
+        {"value": "syslog", "count": 2},
+        {"value": "logicmonitor", "count": 1},
+    ]
+    assert dict((entry["value"], entry["count"]) for entry in payload["items"]["source_name"]) == {
+        "edge-fw-01": 2,
+        "edge-lm-01": 1,
+    }
+
+
+def test_ingest_recent_endpoint_returns_recent_intake_and_allows_status_filter(tmp_path: Path) -> None:
+    client = TestClient(app)
+    db_path = tmp_path / "brainstem.sqlite3"
+    client.post(
+        "/ingest/batch",
+        json={
+            "threshold": 1,
+            "db_path": str(db_path),
+            "events": [
+                {
+                    "tenant_id": "client-a",
+                    "source_type": "syslog",
+                    "source_id": "fw-01",
+                    "source_name": "edge-fw-01",
+                    "message_raw": "service restarted",
+                    "host": "fw-01",
+                    "service": "systemd",
+                },
+                {
+                    "tenant_id": "client-a",
+                    "source_type": "syslog",
+                    "source_id": "fw-01",
+                    "source_name": "edge-fw-01",
+                    "message_raw": "",
+                    "host": "fw-01",
+                    "service": "systemd",
+                },
+            ],
+        },
+    )
+
+    response = client.get(f"/ingest/recent?db_path={db_path}&limit=10")
+    assert response.status_code == 200
+    payload = response.json()
+    assert payload["ok"] is True
+    assert payload["count"] == 2
+    assert len({item["canonicalization_status"] for item in payload["items"]}) == 2
+
+    failed = client.get(f"/ingest/recent?db_path={db_path}&status=parse_failed&limit=10")
+    assert failed.status_code == 200
+    failed_payload = failed.json()
+    assert failed_payload["count"] == 1
+    assert failed_payload["items"][0]["canonicalization_status"] == "parse_failed"

package/tests/test_storage.py

@@ -1,8 +1,23 @@
+import sqlite3
 from pathlib import Path
 
 from brainstem.ingest import ingest_syslog_lines, signatures_for_events
+from brainstem.models import RawInputEnvelope
 from brainstem.recurrence import build_recurrence_candidates
-from brainstem.storage import init_db, list_candidates, store_candidates, store_events, store_signatures
+from brainstem.storage import (
+    get_raw_envelope_by_id,
+    get_ingest_stats,
+    init_db,
+    list_candidates,
+    get_source_dimension_summaries,
+    store_candidates,
+    store_events,
+    list_recent_failed_raw_envelopes,
+    list_recent_raw_envelopes,
+    store_raw_envelopes,
+    set_raw_envelope_status,
+    store_signatures,
+)
 
 
 def test_storage_round_trip(tmp_path: Path) -> None:
@@ -24,3 +39,256 @@ def test_storage_round_trip(tmp_path: Path) -> None:
     rows = list_candidates(str(db_path), limit=10)
     assert rows
     assert rows[0]['title']
+
+
+def test_raw_envelope_records_are_persisted(tmp_path: Path) -> None:
+    db_path = tmp_path / 'brainstem.sqlite3'
+    init_db(str(db_path))
+    raw_events = [
+        RawInputEnvelope(
+            tenant_id="client-a",
+            source_type="syslog",
+            timestamp="2026-03-22T00:00:01Z",
+            message_raw="VPN tunnel dropped and recovered",
+            host="fw-01",
+            service="charon",
+        ),
+        RawInputEnvelope(
+            tenant_id="client-a",
+            source_type="syslog",
+            timestamp="2026-03-22T00:00:02Z",
+            message_raw="IPsec SA rekey succeeded",
+            host="fw-01",
+            service="charon",
+        ),
+    ]
+    assert store_raw_envelopes(raw_events, str(db_path)) == [1, 2]
+
+    conn = sqlite3.connect(db_path)
+    try:
+        rows = conn.execute(
+            "SELECT tenant_id, source_type, message_raw, canonicalization_status FROM raw_envelopes ORDER BY id ASC"
+        ).fetchall()
+    finally:
+        conn.close()
+
+    assert len(rows) == 2
+    assert rows[0][0] == "client-a"
+    assert rows[0][1] == "syslog"
+    assert rows[0][2] == "VPN tunnel dropped and recovered"
+    assert rows[0][3] == "received"
+    assert rows[1][3] == "received"
+
+
+def test_ingest_stats_from_raw_envelopes(tmp_path: Path) -> None:
+    db_path = tmp_path / 'brainstem.sqlite3'
+    init_db(str(db_path))
+    conn = sqlite3.connect(db_path)
+    try:
+        conn.execute(
+            """
+            INSERT INTO raw_envelopes (
+                tenant_id, source_type, timestamp, host, service, severity,
+                asset_id, source_path, facility, message_raw,
+                structured_fields_json, correlation_keys_json, metadata_json,
+                canonicalization_status, failure_reason
+            ) VALUES (
+                'client-a', 'syslog', '2026-03-22T00:00:00Z',
+                'fw-01', 'charon', 'info', '', '', '', 'ok', '{}', '{}', '{}',
+                'canonicalized', NULL
+            )
+            """
+        )
+        conn.execute(
+            """
+            INSERT INTO raw_envelopes (
+                tenant_id, source_type, timestamp, host, service, severity,
+                asset_id, source_path, facility, message_raw,
+                structured_fields_json, correlation_keys_json, metadata_json,
+                canonicalization_status, failure_reason
+            ) VALUES (
+                'client-a', 'syslog', '2026-03-22T00:00:00Z',
+                'fw-01', 'charon', 'info', '', '', '', 'bad', '{}', '{}', '{}',
+                'parse_failed', 'message empty'
+            )
+            """
+        )
+        conn.execute(
+            "INSERT INTO candidates (candidate_type, title, summary, score_total, score_breakdown_json, decision_band, source_signature_ids_json, source_event_ids_json, confidence, metadata_json) VALUES ('recurrence', 'x', 'y', 1.0, '{}', 'medium', '[]', '[]', 0.1, '{}')"
+        )
+        conn.commit()
+    finally:
+        conn.close()
+
+    stats = get_ingest_stats(str(db_path))
+    assert stats["received"] == 2
+    assert stats["canonicalized"] == 1
+    assert stats["parse_failed"] == 1
+    assert stats["candidates_generated"] == 1
+
+
+def test_source_dimension_summaries(tmp_path: Path) -> None:
+    db_path = tmp_path / 'brainstem.sqlite3'
+    init_db(str(db_path))
+    store_raw_envelopes(
+        [
+            RawInputEnvelope(
+                tenant_id='client-a',
+                source_type='syslog',
+                source_id='fw-01',
+                source_name='edge-fw-01',
+                timestamp='2026-03-22T00:00:01Z',
+                message_raw='VPN tunnel dropped and recovered',
+                source_path='/var/log/syslog',
+                host='fw-01',
+                service='charon',
+            ),
+            RawInputEnvelope(
+                tenant_id='client-a',
+                source_type='syslog',
+                source_id='fw-01',
+                source_name='edge-fw-01',
+                timestamp='2026-03-22T00:00:02Z',
+                message_raw='IPsec SA rekey succeeded',
+                source_path='/var/log/syslog',
+                host='fw-01',
+                service='charon',
+            ),
+            RawInputEnvelope(
+                tenant_id='client-a',
+                source_type='logicmonitor',
+                source_id='lm-1',
+                source_name='edge-lm-01',
+                timestamp='2026-03-22T00:00:03Z',
+                message_raw='CPU usage high',
+                source_path='/alerts',
+                host='lm-01',
+                service='logicmonitor',
+            ),
+        ],
+        db_path=str(db_path),
+    )
+
+    summary = get_source_dimension_summaries(str(db_path), limit=10)
+    assert summary['source_type'][0]['value'] == "syslog"
+    assert summary['source_type'][0]['count'] == 2
+    assert summary['source_type'][1]['value'] == "logicmonitor"
+    assert summary['source_type'][1]['count'] == 1
+    assert dict((entry['value'], entry['count']) for entry in summary['source_path']) == {
+        '/alerts': 1,
+        '/var/log/syslog': 2,
+    }
+    assert dict((entry['value'], entry['count']) for entry in summary['source_id']) == {
+        'fw-01': 2,
+        'lm-1': 1,
+    }
+
+
+def test_list_recent_raw_envelopes_supports_status_filtering(tmp_path: Path) -> None:
+    db_path = tmp_path / 'brainstem.sqlite3'
+    init_db(str(db_path))
+    raw_ids = store_raw_envelopes(
+        [
+            RawInputEnvelope(
+                tenant_id='client-a',
+                source_type='syslog',
+                timestamp='2026-03-22T00:00:01Z',
+                message_raw='first',
+                host='fw-01',
+                service='sshd',
+            ),
+            RawInputEnvelope(
+                tenant_id='client-a',
+                source_type='syslog',
+                timestamp='2026-03-22T00:00:02Z',
+                message_raw='second',
+                host='fw-01',
+                service='sshd',
+            ),
+            RawInputEnvelope(
+                tenant_id='client-a',
+                source_type='syslog',
+                timestamp='2026-03-22T00:00:03Z',
+                message_raw='third',
+                host='fw-01',
+                service='sshd',
+            ),
+        ],
+        db_path=str(db_path),
+    )
+    set_raw_envelope_status(raw_ids[0], 'parse_failed', db_path=str(db_path), failure_reason='empty message')
+    set_raw_envelope_status(raw_ids[1], 'canonicalized', db_path=str(db_path))
+    set_raw_envelope_status(raw_ids[2], 'unsupported', db_path=str(db_path), failure_reason='unsupported source')
+
+    all_rows = list_recent_raw_envelopes(str(db_path), limit=10)
+    assert [row['id'] for row in all_rows] == [raw_ids[2], raw_ids[1], raw_ids[0]]
+
+    parsed_only = list_recent_raw_envelopes(str(db_path), status='parse_failed', limit=10)
+    assert [row['id'] for row in parsed_only] == [raw_ids[0]]
+
+
+def test_query_recent_failed_raw_envelopes_with_status_filter(tmp_path: Path) -> None:
+    db_path = tmp_path / 'brainstem.sqlite3'
+    init_db(str(db_path))
+    raw_events = [
+        RawInputEnvelope(
+            tenant_id="client-a",
+            source_type="syslog",
+            timestamp="2026-03-22T00:00:01Z",
+            message_raw="first",
+            host="fw-01",
+            service="sshd",
+        ),
+        RawInputEnvelope(
+            tenant_id="client-a",
+            source_type="syslog",
+            timestamp="2026-03-22T00:00:02Z",
+            message_raw="second",
+            host="fw-01",
+            service="sshd",
+        ),
+        RawInputEnvelope(
+            tenant_id="client-a",
+            source_type="syslog",
+            timestamp="2026-03-22T00:00:03Z",
+            message_raw="third",
+            host="fw-01",
+            service="sshd",
+        ),
+    ]
+    raw_ids = store_raw_envelopes(raw_events, str(db_path))
+    set_raw_envelope_status(raw_ids[0], "parse_failed", db_path=str(db_path), failure_reason="empty message")
+    set_raw_envelope_status(raw_ids[1], "canonicalized", db_path=str(db_path))
+    set_raw_envelope_status(raw_ids[2], "unsupported", db_path=str(db_path), failure_reason="unsupported source")
+
+    failures = list_recent_failed_raw_envelopes(str(db_path), limit=10)
+    assert [row["id"] for row in failures] == [raw_ids[2], raw_ids[0]]
+    assert failures[0]["canonicalization_status"] == "unsupported"
+    assert failures[1]["canonicalization_status"] == "parse_failed"
+
+    parsed_only = list_recent_failed_raw_envelopes(str(db_path), status="parse_failed", limit=10)
+    assert len(parsed_only) == 1
+    assert parsed_only[0]["id"] == raw_ids[0]
+
+
+def test_get_raw_envelope_by_id(tmp_path: Path) -> None:
+    db_path = tmp_path / 'brainstem.sqlite3'
+    init_db(str(db_path))
+    raw_events = [
+        RawInputEnvelope(
+            tenant_id="client-a",
+            source_type="syslog",
+            timestamp="2026-03-22T00:00:01Z",
+            message_raw="single",
+            host="fw-01",
+            service="charon",
+        )
+    ]
+    (raw_id,) = store_raw_envelopes(raw_events, str(db_path))
+    set_raw_envelope_status(raw_id, "parse_failed", db_path=str(db_path), failure_reason="empty message")
+
+    row = get_raw_envelope_by_id(raw_id, db_path=str(db_path))
+    assert row is not None
+    assert row["id"] == raw_id
+    assert row["canonicalization_status"] == "parse_failed"
+    assert row["failure_reason"] == "empty message"