@simbimbo/brainstem 0.0.2 → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -0
- package/README.md +25 -0
- package/brainstem/__init__.py +1 -1
- package/brainstem/adapters.py +120 -0
- package/brainstem/api.py +483 -23
- package/brainstem/config.py +70 -0
- package/brainstem/ingest.py +418 -33
- package/brainstem/interesting.py +56 -1
- package/brainstem/listener.py +175 -0
- package/brainstem/models.py +3 -0
- package/brainstem/recurrence.py +38 -1
- package/brainstem/source_drivers.py +150 -0
- package/brainstem/storage.py +547 -8
- package/docs/README.md +94 -0
- package/docs/adapters.md +97 -401
- package/docs/api.md +223 -278
- package/package.json +1 -1
- package/pyproject.toml +1 -1
- package/tests/test_adapters.py +94 -0
- package/tests/test_api.py +973 -0
- package/tests/test_canonicalization.py +8 -0
- package/tests/test_config.py +24 -0
- package/tests/test_file_ingest.py +77 -0
- package/tests/test_interesting.py +10 -0
- package/tests/test_listener.py +253 -0
- package/tests/test_recurrence.py +2 -0
- package/tests/test_source_drivers.py +95 -0
- package/tests/test_storage.py +370 -2
package/brainstem/api.py
CHANGED
|
@@ -1,27 +1,111 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
from datetime import datetime
|
|
4
|
+
import os
|
|
5
|
+
import secrets
|
|
6
|
+
import sys
|
|
7
|
+
from dataclasses import asdict
|
|
4
8
|
from typing import Any, Dict, List, Optional
|
|
5
9
|
|
|
6
10
|
import json
|
|
7
|
-
from fastapi import FastAPI, HTTPException, Query
|
|
8
|
-
from fastapi.responses import JSONResponse
|
|
11
|
+
from fastapi import Depends, FastAPI, Header, HTTPException, Query
|
|
9
12
|
from pydantic import BaseModel, Field
|
|
10
13
|
|
|
11
|
-
from .ingest import
|
|
14
|
+
from .ingest import ReplayAttempt, replay_raw_envelopes_by_ids, run_ingest_pipeline
|
|
12
15
|
from .interesting import interesting_items
|
|
16
|
+
from .config import get_runtime_config
|
|
17
|
+
from .fingerprint import normalize_message
|
|
18
|
+
from . import __version__
|
|
13
19
|
from .models import Candidate, RawInputEnvelope
|
|
14
|
-
from .
|
|
15
|
-
|
|
16
|
-
|
|
20
|
+
from .storage import (
|
|
21
|
+
RAW_ENVELOPE_STATUSES,
|
|
22
|
+
extract_source_raw_envelope_ids,
|
|
23
|
+
get_ingest_stats,
|
|
24
|
+
list_candidates,
|
|
25
|
+
list_signatures,
|
|
26
|
+
get_raw_envelope_by_id,
|
|
27
|
+
get_raw_envelopes_by_ids,
|
|
28
|
+
get_source_dimension_summaries,
|
|
29
|
+
get_source_status_summaries,
|
|
30
|
+
list_canonical_events,
|
|
31
|
+
list_recent_failed_raw_envelopes,
|
|
32
|
+
list_recent_raw_envelopes,
|
|
33
|
+
)
|
|
17
34
|
|
|
18
35
|
|
|
19
36
|
app = FastAPI(title="brAInstem Runtime")
|
|
37
|
+
RUNTIME_CONFIG = get_runtime_config()
|
|
38
|
+
API_TOKEN_ENV_VAR = RUNTIME_CONFIG.api_token_env_var
|
|
39
|
+
API_TOKEN_HEADER = "X-API-Token"
|
|
40
|
+
RUNTIME_DEFAULTS = asdict(RUNTIME_CONFIG.defaults)
|
|
41
|
+
RUNTIME_LIMITS = {k: v for k, v in asdict(RUNTIME_CONFIG.limits).items() if k != "replay_allowed_statuses"}
|
|
42
|
+
MAX_REPLAY_IDS = RUNTIME_LIMITS["replay_raw_max_ids"]
|
|
43
|
+
DEFAULT_INGEST_THRESHOLD = RUNTIME_DEFAULTS["ingest_threshold"]
|
|
44
|
+
DEFAULT_BATCH_THRESHOLD = RUNTIME_DEFAULTS["batch_threshold"]
|
|
45
|
+
DEFAULT_INTERESTING_LIMIT = RUNTIME_DEFAULTS["interesting_limit"]
|
|
46
|
+
DEFAULT_FAILURE_LIMIT = RUNTIME_DEFAULTS["failure_limit"]
|
|
47
|
+
DEFAULT_INGEST_RECENT_LIMIT = RUNTIME_DEFAULTS["ingest_recent_limit"]
|
|
48
|
+
DEFAULT_SOURCES_LIMIT = RUNTIME_DEFAULTS["sources_limit"]
|
|
49
|
+
DEFAULT_SOURCES_STATUS_LIMIT = RUNTIME_DEFAULTS["sources_status_limit"]
|
|
50
|
+
DEFAULT_REPLAY_THRESHOLD = RUNTIME_DEFAULTS["replay_threshold"]
|
|
51
|
+
DEFAULT_REPLAY_ALLOWED_STATUSES = tuple(RUNTIME_CONFIG.limits.replay_allowed_statuses)
|
|
52
|
+
CAPABILITIES = {
|
|
53
|
+
"ingest_endpoints": {
|
|
54
|
+
"single_event": True,
|
|
55
|
+
"batch_events": True,
|
|
56
|
+
"replay_raw": True,
|
|
57
|
+
},
|
|
58
|
+
"inspection_endpoints": {
|
|
59
|
+
"interesting": True,
|
|
60
|
+
"candidates": True,
|
|
61
|
+
"signatures": True,
|
|
62
|
+
"canonical_events": True,
|
|
63
|
+
"stats": True,
|
|
64
|
+
"raw_envelopes": True,
|
|
65
|
+
"failures": True,
|
|
66
|
+
"ingest_recent": True,
|
|
67
|
+
"sources": True,
|
|
68
|
+
},
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _configured_api_token() -> str:
|
|
73
|
+
return os.getenv(API_TOKEN_ENV_VAR, "").strip()
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def is_api_token_auth_enabled() -> bool:
|
|
77
|
+
return bool(_configured_api_token())
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _extract_bearer_token(authorization: Optional[str]) -> Optional[str]:
|
|
81
|
+
if not authorization:
|
|
82
|
+
return None
|
|
83
|
+
scheme, separator, token = authorization.partition(" ")
|
|
84
|
+
if separator != " ":
|
|
85
|
+
return None
|
|
86
|
+
if scheme.lower() != "bearer":
|
|
87
|
+
return None
|
|
88
|
+
return token.strip() or None
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _require_api_token(
|
|
92
|
+
x_api_token: Optional[str] = Header(default=None, alias=API_TOKEN_HEADER),
|
|
93
|
+
authorization: Optional[str] = Header(default=None),
|
|
94
|
+
) -> None:
|
|
95
|
+
configured_token = _configured_api_token()
|
|
96
|
+
if not configured_token:
|
|
97
|
+
return
|
|
98
|
+
|
|
99
|
+
provided_token = x_api_token or _extract_bearer_token(authorization)
|
|
100
|
+
if not provided_token or not secrets.compare_digest(provided_token, configured_token):
|
|
101
|
+
raise HTTPException(status_code=401, detail="missing or invalid API token")
|
|
20
102
|
|
|
21
103
|
|
|
22
104
|
class RawEnvelopeRequest(BaseModel):
|
|
23
105
|
tenant_id: str
|
|
24
106
|
source_type: str
|
|
107
|
+
source_id: str = ""
|
|
108
|
+
source_name: str = ""
|
|
25
109
|
message_raw: str
|
|
26
110
|
timestamp: Optional[str] = None
|
|
27
111
|
host: str = ""
|
|
@@ -41,14 +125,24 @@ class IngestEventRequest(RawEnvelopeRequest):
|
|
|
41
125
|
|
|
42
126
|
class IngestBatchRequest(BaseModel):
|
|
43
127
|
events: List[RawEnvelopeRequest]
|
|
44
|
-
threshold: int = Field(default=
|
|
128
|
+
threshold: int = Field(default=DEFAULT_BATCH_THRESHOLD, ge=1)
|
|
45
129
|
db_path: Optional[str] = None
|
|
46
130
|
|
|
47
131
|
|
|
132
|
+
class ReplayRawRequest(BaseModel):
|
|
133
|
+
db_path: str
|
|
134
|
+
raw_envelope_ids: List[int] = Field(default_factory=list, min_length=1, max_length=MAX_REPLAY_IDS)
|
|
135
|
+
threshold: int = Field(default=DEFAULT_REPLAY_THRESHOLD, ge=1)
|
|
136
|
+
force: bool = False
|
|
137
|
+
allowed_statuses: List[str] = Field(default_factory=lambda: list(DEFAULT_REPLAY_ALLOWED_STATUSES))
|
|
138
|
+
|
|
139
|
+
|
|
48
140
|
def _raw_envelope_from_request(payload: RawEnvelopeRequest) -> RawInputEnvelope:
|
|
49
141
|
return RawInputEnvelope(
|
|
50
142
|
tenant_id=payload.tenant_id,
|
|
51
143
|
source_type=payload.source_type,
|
|
144
|
+
source_id=payload.source_id,
|
|
145
|
+
source_name=payload.source_name,
|
|
52
146
|
timestamp=payload.timestamp or datetime.utcnow().isoformat() + "Z",
|
|
53
147
|
message_raw=payload.message_raw,
|
|
54
148
|
host=payload.host,
|
|
@@ -78,18 +172,166 @@ def _candidate_from_row(row) -> Candidate:
|
|
|
78
172
|
)
|
|
79
173
|
|
|
80
174
|
|
|
175
|
+
def _candidate_inspection_from_row(row, db_path: str | None) -> Dict[str, Any]:
|
|
176
|
+
candidate = _candidate_from_row(row)
|
|
177
|
+
inspected = interesting_items([candidate], limit=1)
|
|
178
|
+
summary = inspected[0] if inspected else {
|
|
179
|
+
"title": candidate.title,
|
|
180
|
+
"summary": candidate.summary,
|
|
181
|
+
"decision_band": candidate.decision_band,
|
|
182
|
+
"attention_band": None,
|
|
183
|
+
"attention_score": candidate.score_total,
|
|
184
|
+
"score_total": candidate.score_total,
|
|
185
|
+
"attention_explanation": [],
|
|
186
|
+
+ "score_breakdown": candidate.score_breakdown,
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
raw_envelope_ids = extract_source_raw_envelope_ids(row["metadata_json"])
|
|
190
|
+
raw_envelopes = []
|
|
191
|
+
if raw_envelope_ids:
|
|
192
|
+
raw_envelope_rows = get_raw_envelopes_by_ids(raw_envelope_ids, db_path=db_path)
|
|
193
|
+
raw_envelope_index = {raw_row["id"]: raw_row for raw_row in raw_envelope_rows}
|
|
194
|
+
raw_envelopes = [
|
|
195
|
+
_raw_envelope_from_row(raw_envelope_index[row_id])
|
|
196
|
+
for row_id in raw_envelope_ids
|
|
197
|
+
if row_id in raw_envelope_index
|
|
198
|
+
]
|
|
199
|
+
|
|
200
|
+
return {
|
|
201
|
+
**summary,
|
|
202
|
+
"id": row["id"],
|
|
203
|
+
"candidate_type": candidate.candidate_type,
|
|
204
|
+
"confidence": candidate.confidence,
|
|
205
|
+
"source_signature_ids": candidate.source_signature_ids,
|
|
206
|
+
"source_event_ids": candidate.source_event_ids,
|
|
207
|
+
"score_breakdown": candidate.score_breakdown,
|
|
208
|
+
"raw_envelope_ids": raw_envelope_ids,
|
|
209
|
+
"raw_envelopes": raw_envelopes,
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _signature_inspection_from_row(row, db_path: str | None) -> Dict[str, Any]:
|
|
214
|
+
metadata = json.loads(row["metadata_json"] or "{}")
|
|
215
|
+
if not isinstance(metadata, dict):
|
|
216
|
+
metadata = {}
|
|
217
|
+
raw_envelope_ids = extract_source_raw_envelope_ids(row["metadata_json"])
|
|
218
|
+
occurrence_count = int(row["occurrence_count"]) if row["occurrence_count"] is not None else 0
|
|
219
|
+
raw_envelope_count = len(raw_envelope_ids)
|
|
220
|
+
|
|
221
|
+
return {
|
|
222
|
+
"signature_key": row["signature_key"],
|
|
223
|
+
"event_family": row["event_family"],
|
|
224
|
+
"normalized_pattern": row["normalized_pattern"],
|
|
225
|
+
"service": row["service"],
|
|
226
|
+
"occurrence_count": occurrence_count,
|
|
227
|
+
"raw_envelope_ids": raw_envelope_ids,
|
|
228
|
+
"raw_envelope_count": raw_envelope_count,
|
|
229
|
+
"recurrence": {
|
|
230
|
+
"occurrence_count": occurrence_count,
|
|
231
|
+
"raw_envelope_count": raw_envelope_count,
|
|
232
|
+
"signature_id": row["id"],
|
|
233
|
+
},
|
|
234
|
+
"metadata": metadata,
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _canonical_event_from_row(row) -> Dict[str, Any]:
|
|
239
|
+
message_raw = row["message_raw"] or ""
|
|
240
|
+
return {
|
|
241
|
+
"raw_envelope_id": row["id"],
|
|
242
|
+
"tenant_id": row["tenant_id"],
|
|
243
|
+
"source": row["source_type"],
|
|
244
|
+
"source_type": row["source_type"],
|
|
245
|
+
"message_raw": message_raw,
|
|
246
|
+
"timestamp": row["timestamp"],
|
|
247
|
+
"host": row["host"] or "",
|
|
248
|
+
"service": row["service"] or "",
|
|
249
|
+
"severity": row["severity"] or "info",
|
|
250
|
+
"message_normalized": normalize_message(message_raw),
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def _raw_envelope_from_row(row) -> Dict[str, Any]:
|
|
255
|
+
return {
|
|
256
|
+
"id": row["id"],
|
|
257
|
+
"tenant_id": row["tenant_id"],
|
|
258
|
+
"source_type": row["source_type"],
|
|
259
|
+
"source_id": row["source_id"],
|
|
260
|
+
"source_name": row["source_name"],
|
|
261
|
+
"timestamp": row["timestamp"],
|
|
262
|
+
"host": row["host"],
|
|
263
|
+
"service": row["service"],
|
|
264
|
+
"severity": row["severity"],
|
|
265
|
+
"asset_id": row["asset_id"],
|
|
266
|
+
"source_path": row["source_path"],
|
|
267
|
+
"facility": row["facility"],
|
|
268
|
+
"message_raw": row["message_raw"],
|
|
269
|
+
"structured_fields": json.loads(row["structured_fields_json"] or "{}"),
|
|
270
|
+
"correlation_keys": json.loads(row["correlation_keys_json"] or "{}"),
|
|
271
|
+
"metadata": json.loads(row["metadata_json"] or "{}"),
|
|
272
|
+
"canonicalization_status": row["canonicalization_status"],
|
|
273
|
+
"failure_reason": row["failure_reason"],
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def _replay_attempt_from_item(item: ReplayAttempt) -> Dict[str, Any]:
|
|
278
|
+
return {
|
|
279
|
+
"raw_envelope_id": item.raw_envelope_id,
|
|
280
|
+
"reason": item.reason,
|
|
281
|
+
"status": item.status,
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def _runtime_summary() -> Dict[str, Any]:
|
|
286
|
+
runtime_config = get_runtime_config()
|
|
287
|
+
return {
|
|
288
|
+
"version": __version__,
|
|
289
|
+
"runtime": {
|
|
290
|
+
"python_version": ".".join(str(item) for item in sys.version_info[:3]),
|
|
291
|
+
"api_token_env": runtime_config.api_token_env_var,
|
|
292
|
+
"config": runtime_config.as_dict(),
|
|
293
|
+
},
|
|
294
|
+
"capability_flags": CAPABILITIES,
|
|
295
|
+
"auth_state": {
|
|
296
|
+
"api_token_configured": is_api_token_auth_enabled(),
|
|
297
|
+
"supports_x_api_token_header": True,
|
|
298
|
+
"supports_bearer_header": True,
|
|
299
|
+
},
|
|
300
|
+
"defaults": dict(RUNTIME_DEFAULTS),
|
|
301
|
+
"limits": dict(RUNTIME_LIMITS),
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def _status_payload() -> Dict[str, Any]:
|
|
306
|
+
runtime_summary = _runtime_summary()
|
|
307
|
+
return {
|
|
308
|
+
"ok": True,
|
|
309
|
+
"status": "ok",
|
|
310
|
+
"runtime": runtime_summary,
|
|
311
|
+
"api_token_enabled": runtime_summary["auth_state"]["api_token_configured"],
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
|
|
81
315
|
def _run_ingest_batch(raw_events: List[RawInputEnvelope], *, threshold: int, db_path: Optional[str]) -> Dict[str, Any]:
|
|
82
|
-
|
|
316
|
+
result = run_ingest_pipeline(raw_events, threshold=threshold, db_path=db_path)
|
|
317
|
+
events = result.events
|
|
318
|
+
parse_failed = result.parse_failed
|
|
319
|
+
item_results = [asdict(item) for item in result.item_results]
|
|
320
|
+
|
|
83
321
|
if not events:
|
|
84
|
-
return {
|
|
322
|
+
return {
|
|
323
|
+
"ok": True,
|
|
324
|
+
"event_count": 0,
|
|
325
|
+
"signature_count": 0,
|
|
326
|
+
"candidate_count": 0,
|
|
327
|
+
"item_count": len(raw_events),
|
|
328
|
+
"item_results": item_results,
|
|
329
|
+
"parse_failed": parse_failed,
|
|
330
|
+
"interesting_items": [],
|
|
331
|
+
}
|
|
85
332
|
|
|
86
|
-
signatures =
|
|
87
|
-
candidates =
|
|
88
|
-
if db_path:
|
|
89
|
-
init_db(db_path)
|
|
90
|
-
store_events(events, db_path)
|
|
91
|
-
store_signatures(signatures, db_path)
|
|
92
|
-
store_candidates(candidates, db_path)
|
|
333
|
+
signatures = result.signatures
|
|
334
|
+
candidates = result.candidates
|
|
93
335
|
|
|
94
336
|
return {
|
|
95
337
|
"ok": True,
|
|
@@ -97,26 +339,61 @@ def _run_ingest_batch(raw_events: List[RawInputEnvelope], *, threshold: int, db_
|
|
|
97
339
|
"event_count": len(events),
|
|
98
340
|
"signature_count": len({sig.signature_key for sig in signatures}),
|
|
99
341
|
"candidate_count": len(candidates),
|
|
342
|
+
"item_count": len(raw_events),
|
|
343
|
+
"item_results": item_results,
|
|
344
|
+
"parse_failed": parse_failed,
|
|
100
345
|
"interesting_items": interesting_items(candidates, limit=max(1, 5)),
|
|
101
346
|
}
|
|
102
347
|
|
|
103
348
|
|
|
104
|
-
@app.post("/
|
|
105
|
-
def
|
|
349
|
+
@app.post("/replay/raw", dependencies=[Depends(_require_api_token)])
|
|
350
|
+
def replay_raw(payload: ReplayRawRequest) -> Dict[str, Any]:
|
|
351
|
+
if not payload.raw_envelope_ids:
|
|
352
|
+
raise HTTPException(status_code=422, detail="raw_envelope_ids is required")
|
|
353
|
+
if payload.allowed_statuses:
|
|
354
|
+
invalid = [status for status in payload.allowed_statuses if status not in RAW_ENVELOPE_STATUSES]
|
|
355
|
+
if invalid:
|
|
356
|
+
raise HTTPException(
|
|
357
|
+
status_code=422,
|
|
358
|
+
detail=f"invalid status in allowed_statuses: {invalid}; expected one of: {', '.join(RAW_ENVELOPE_STATUSES)}",
|
|
359
|
+
)
|
|
360
|
+
|
|
361
|
+
result = replay_raw_envelopes_by_ids(
|
|
362
|
+
payload.raw_envelope_ids,
|
|
363
|
+
db_path=payload.db_path,
|
|
364
|
+
threshold=payload.threshold,
|
|
365
|
+
force=payload.force,
|
|
366
|
+
allowed_statuses=payload.allowed_statuses,
|
|
367
|
+
)
|
|
368
|
+
return {
|
|
369
|
+
"ok": True,
|
|
370
|
+
"db_path": payload.db_path,
|
|
371
|
+
"requested_raw_envelope_ids": result.requested_raw_envelope_ids,
|
|
372
|
+
"attempted_raw_envelope_ids": result.attempted_raw_envelope_ids,
|
|
373
|
+
"skipped": [_replay_attempt_from_item(item) for item in result.skipped],
|
|
374
|
+
"event_count": len(result.events),
|
|
375
|
+
"signature_count": len(result.signatures),
|
|
376
|
+
"candidate_count": len(result.candidates),
|
|
377
|
+
"parse_failed": result.parse_failed,
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
@app.post("/ingest/event", dependencies=[Depends(_require_api_token)])
|
|
382
|
+
def ingest_event(payload: IngestEventRequest, threshold: int = DEFAULT_INGEST_THRESHOLD, db_path: Optional[str] = None) -> Dict[str, Any]:
|
|
106
383
|
if threshold < 1:
|
|
107
384
|
raise HTTPException(status_code=422, detail="threshold must be >= 1")
|
|
108
385
|
return _run_ingest_batch([_raw_envelope_from_request(payload)], threshold=threshold, db_path=db_path)
|
|
109
386
|
|
|
110
387
|
|
|
111
|
-
@app.post("/ingest/batch")
|
|
388
|
+
@app.post("/ingest/batch", dependencies=[Depends(_require_api_token)])
|
|
112
389
|
def ingest_batch(payload: IngestBatchRequest) -> Dict[str, Any]:
|
|
113
390
|
raw_events = [_raw_envelope_from_request(event) for event in payload.events]
|
|
114
391
|
return _run_ingest_batch(raw_events, threshold=payload.threshold, db_path=payload.db_path)
|
|
115
392
|
|
|
116
393
|
|
|
117
|
-
@app.get("/interesting")
|
|
394
|
+
@app.get("/interesting", dependencies=[Depends(_require_api_token)])
|
|
118
395
|
def get_interesting(
|
|
119
|
-
limit: int = Query(default=
|
|
396
|
+
limit: int = Query(default=DEFAULT_INTERESTING_LIMIT, ge=1),
|
|
120
397
|
db_path: Optional[str] = None,
|
|
121
398
|
) -> Dict[str, Any]:
|
|
122
399
|
if not db_path:
|
|
@@ -126,6 +403,189 @@ def get_interesting(
|
|
|
126
403
|
return {"ok": True, "items": interesting_items(candidates, limit=limit)}
|
|
127
404
|
|
|
128
405
|
|
|
406
|
+
@app.get("/candidates", dependencies=[Depends(_require_api_token)])
|
|
407
|
+
def get_candidates(
|
|
408
|
+
limit: int = Query(default=DEFAULT_INTERESTING_LIMIT, ge=1),
|
|
409
|
+
db_path: Optional[str] = None,
|
|
410
|
+
candidate_type: Optional[str] = None,
|
|
411
|
+
decision_band: Optional[str] = None,
|
|
412
|
+
min_score_total: Optional[float] = Query(default=None, ge=0),
|
|
413
|
+
) -> Dict[str, Any]:
|
|
414
|
+
if not db_path:
|
|
415
|
+
return {"ok": True, "count": 0, "items": []}
|
|
416
|
+
|
|
417
|
+
rows = list_candidates(
|
|
418
|
+
db_path=db_path,
|
|
419
|
+
limit=limit,
|
|
420
|
+
candidate_type=candidate_type,
|
|
421
|
+
decision_band=decision_band,
|
|
422
|
+
min_score_total=min_score_total,
|
|
423
|
+
)
|
|
424
|
+
items = [_candidate_inspection_from_row(row, db_path=db_path) for row in rows]
|
|
425
|
+
return {"ok": True, "count": len(items), "items": items}
|
|
426
|
+
|
|
427
|
+
|
|
428
|
+
@app.get("/signatures", dependencies=[Depends(_require_api_token)])
|
|
429
|
+
def get_signatures(
|
|
430
|
+
limit: int = Query(default=DEFAULT_INTERESTING_LIMIT, ge=1),
|
|
431
|
+
db_path: Optional[str] = None,
|
|
432
|
+
event_family: Optional[str] = None,
|
|
433
|
+
service: Optional[str] = None,
|
|
434
|
+
min_occurrence_count: Optional[int] = Query(default=None, ge=1),
|
|
435
|
+
) -> Dict[str, Any]:
|
|
436
|
+
if not db_path:
|
|
437
|
+
return {"ok": True, "count": 0, "items": []}
|
|
438
|
+
|
|
439
|
+
rows = list_signatures(
|
|
440
|
+
db_path=db_path,
|
|
441
|
+
limit=limit,
|
|
442
|
+
event_family=event_family,
|
|
443
|
+
service=service,
|
|
444
|
+
min_occurrence_count=min_occurrence_count,
|
|
445
|
+
)
|
|
446
|
+
items = [_signature_inspection_from_row(row, db_path=db_path) for row in rows]
|
|
447
|
+
return {"ok": True, "count": len(items), "items": items}
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
@app.get("/stats", dependencies=[Depends(_require_api_token)])
|
|
451
|
+
def get_stats(db_path: Optional[str] = None) -> Dict[str, Any]:
|
|
452
|
+
return {"ok": True, **get_ingest_stats(db_path)}
|
|
453
|
+
|
|
454
|
+
|
|
455
|
+
@app.get("/failures", dependencies=[Depends(_require_api_token)])
|
|
456
|
+
def get_failures(
|
|
457
|
+
limit: int = Query(default=DEFAULT_FAILURE_LIMIT, ge=1),
|
|
458
|
+
status: Optional[str] = None,
|
|
459
|
+
db_path: Optional[str] = None,
|
|
460
|
+
) -> Dict[str, Any]:
|
|
461
|
+
if status is not None and status not in RAW_ENVELOPE_STATUSES:
|
|
462
|
+
raise HTTPException(
|
|
463
|
+
status_code=422,
|
|
464
|
+
detail=f"invalid status '{status}'; expected one of: {', '.join(RAW_ENVELOPE_STATUSES)}",
|
|
465
|
+
)
|
|
466
|
+
|
|
467
|
+
rows = list_recent_failed_raw_envelopes(db_path=db_path, status=status, limit=limit)
|
|
468
|
+
items = [_raw_envelope_from_row(row) for row in rows]
|
|
469
|
+
return {"ok": True, "items": items, "count": len(items), "status": status}
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
@app.get("/raw_envelopes", dependencies=[Depends(_require_api_token)])
|
|
473
|
+
def get_raw_envelopes(
|
|
474
|
+
limit: int = Query(default=DEFAULT_INGEST_RECENT_LIMIT, ge=1),
|
|
475
|
+
status: Optional[str] = None,
|
|
476
|
+
tenant_id: Optional[str] = None,
|
|
477
|
+
source_type: Optional[str] = None,
|
|
478
|
+
source_id: Optional[str] = None,
|
|
479
|
+
source_path: Optional[str] = None,
|
|
480
|
+
db_path: Optional[str] = None,
|
|
481
|
+
) -> Dict[str, Any]:
|
|
482
|
+
if status is not None and status not in RAW_ENVELOPE_STATUSES:
|
|
483
|
+
raise HTTPException(
|
|
484
|
+
status_code=422,
|
|
485
|
+
detail=f"invalid status '{status}'; expected one of: {', '.join(RAW_ENVELOPE_STATUSES)}",
|
|
486
|
+
)
|
|
487
|
+
rows = list_recent_raw_envelopes(
|
|
488
|
+
db_path=db_path,
|
|
489
|
+
status=status,
|
|
490
|
+
limit=limit,
|
|
491
|
+
tenant_id=tenant_id,
|
|
492
|
+
source_type=source_type,
|
|
493
|
+
source_id=source_id,
|
|
494
|
+
source_path=source_path,
|
|
495
|
+
failures_only=False,
|
|
496
|
+
)
|
|
497
|
+
items = [_raw_envelope_from_row(row) for row in rows]
|
|
498
|
+
return {"ok": True, "items": items, "count": len(items)}
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
@app.get("/ingest/recent", dependencies=[Depends(_require_api_token)])
|
|
502
|
+
def get_ingest_recent(
|
|
503
|
+
limit: int = Query(default=DEFAULT_INGEST_RECENT_LIMIT, ge=1),
|
|
504
|
+
status: Optional[str] = None,
|
|
505
|
+
db_path: Optional[str] = None,
|
|
506
|
+
) -> Dict[str, Any]:
|
|
507
|
+
if status is not None and status not in RAW_ENVELOPE_STATUSES:
|
|
508
|
+
raise HTTPException(
|
|
509
|
+
status_code=422,
|
|
510
|
+
detail=f"invalid status '{status}'; expected one of: {', '.join(RAW_ENVELOPE_STATUSES)}",
|
|
511
|
+
)
|
|
512
|
+
rows = list_recent_raw_envelopes(db_path=db_path, status=status, limit=limit, failures_only=False)
|
|
513
|
+
items = [_raw_envelope_from_row(row) for row in rows]
|
|
514
|
+
return {"ok": True, "items": items, "count": len(items), "status": status}
|
|
515
|
+
|
|
516
|
+
|
|
517
|
+
@app.get("/canonical_events", dependencies=[Depends(_require_api_token)])
|
|
518
|
+
def get_canonical_events(
|
|
519
|
+
limit: int = Query(default=DEFAULT_INGEST_RECENT_LIMIT, ge=1),
|
|
520
|
+
tenant_id: Optional[str] = None,
|
|
521
|
+
source: Optional[str] = None,
|
|
522
|
+
host: Optional[str] = None,
|
|
523
|
+
service: Optional[str] = None,
|
|
524
|
+
severity: Optional[str] = None,
|
|
525
|
+
db_path: Optional[str] = None,
|
|
526
|
+
) -> Dict[str, Any]:
|
|
527
|
+
if not db_path:
|
|
528
|
+
return {"ok": True, "items": [], "count": 0}
|
|
529
|
+
|
|
530
|
+
rows = list_canonical_events(
|
|
531
|
+
db_path=db_path,
|
|
532
|
+
limit=limit,
|
|
533
|
+
tenant_id=tenant_id,
|
|
534
|
+
source_type=source,
|
|
535
|
+
host=host,
|
|
536
|
+
service=service,
|
|
537
|
+
severity=severity,
|
|
538
|
+
)
|
|
539
|
+
items = [_canonical_event_from_row(row) for row in rows]
|
|
540
|
+
return {"ok": True, "items": items, "count": len(items)}
|
|
541
|
+
|
|
542
|
+
|
|
543
|
+
@app.get("/sources", dependencies=[Depends(_require_api_token)])
|
|
544
|
+
def get_sources(
|
|
545
|
+
limit: int = Query(default=DEFAULT_SOURCES_LIMIT, ge=1),
|
|
546
|
+
db_path: Optional[str] = None,
|
|
547
|
+
) -> Dict[str, Any]:
|
|
548
|
+
return {"ok": True, "items": get_source_dimension_summaries(db_path=db_path, limit=limit)}
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
@app.get("/sources/status", dependencies=[Depends(_require_api_token)])
|
|
552
|
+
def get_sources_status(
|
|
553
|
+
limit: int = Query(default=DEFAULT_SOURCES_STATUS_LIMIT, ge=1),
|
|
554
|
+
tenant_id: Optional[str] = None,
|
|
555
|
+
source_type: Optional[str] = None,
|
|
556
|
+
source_id: Optional[str] = None,
|
|
557
|
+
source_path: Optional[str] = None,
|
|
558
|
+
db_path: Optional[str] = None,
|
|
559
|
+
) -> Dict[str, Any]:
|
|
560
|
+
items = get_source_status_summaries(
|
|
561
|
+
db_path=db_path,
|
|
562
|
+
limit=limit,
|
|
563
|
+
tenant_id=tenant_id,
|
|
564
|
+
source_type=source_type,
|
|
565
|
+
source_id=source_id,
|
|
566
|
+
source_path=source_path,
|
|
567
|
+
)
|
|
568
|
+
return {"ok": True, "items": items, "count": len(items)}
|
|
569
|
+
|
|
570
|
+
|
|
571
|
+
@app.get("/failures/{raw_envelope_id}", dependencies=[Depends(_require_api_token)])
|
|
572
|
+
def get_failure(raw_envelope_id: int, db_path: Optional[str] = None) -> Dict[str, Any]:
|
|
573
|
+
row = get_raw_envelope_by_id(raw_envelope_id, db_path=db_path)
|
|
574
|
+
if row is None:
|
|
575
|
+
raise HTTPException(status_code=404, detail="raw envelope not found")
|
|
576
|
+
return {"ok": True, "item": _raw_envelope_from_row(row)}
|
|
577
|
+
|
|
578
|
+
|
|
129
579
|
@app.get("/healthz")
|
|
130
|
-
def healthz() -> Dict[str,
|
|
131
|
-
return
|
|
580
|
+
def healthz() -> Dict[str, Any]:
|
|
581
|
+
return _status_payload()
|
|
582
|
+
|
|
583
|
+
|
|
584
|
+
@app.get("/runtime")
|
|
585
|
+
def runtime() -> Dict[str, Any]:
|
|
586
|
+
return {"ok": True, "runtime": _runtime_summary()}
|
|
587
|
+
|
|
588
|
+
|
|
589
|
+
@app.get("/status")
|
|
590
|
+
def status() -> Dict[str, Any]:
|
|
591
|
+
return _status_payload()
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from dataclasses import asdict, dataclass
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Any, Dict
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def resolve_default_db_path() -> str:
|
|
10
|
+
configured_db_path = os.getenv("BRAINSTEM_DB_PATH", "").strip()
|
|
11
|
+
if configured_db_path:
|
|
12
|
+
return configured_db_path
|
|
13
|
+
return str(Path(".brainstem-state") / "brainstem.sqlite3")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass(frozen=True)
|
|
17
|
+
class ListenerConfig:
|
|
18
|
+
syslog_host: str = "127.0.0.1"
|
|
19
|
+
syslog_port: int = 5514
|
|
20
|
+
syslog_source_path: str = "/dev/udp"
|
|
21
|
+
syslog_socket_timeout: float = 0.5
|
|
22
|
+
ingest_threshold: int = 2
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(frozen=True)
|
|
26
|
+
class RuntimeDefaults:
|
|
27
|
+
ingest_threshold: int = 2
|
|
28
|
+
batch_threshold: int = 2
|
|
29
|
+
interesting_limit: int = 5
|
|
30
|
+
failure_limit: int = 20
|
|
31
|
+
ingest_recent_limit: int = 20
|
|
32
|
+
sources_limit: int = 10
|
|
33
|
+
sources_status_limit: int = 20
|
|
34
|
+
replay_threshold: int = 2
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass(frozen=True)
|
|
38
|
+
class RuntimeLimits:
|
|
39
|
+
replay_raw_max_ids: int = 32
|
|
40
|
+
status_filter_limit: int = 20
|
|
41
|
+
replay_allowed_statuses: tuple[str, ...] = ("received", "parse_failed")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass(frozen=True)
|
|
45
|
+
class DBConfig:
|
|
46
|
+
default_path: str
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@dataclass(frozen=True)
|
|
50
|
+
class RuntimeConfig:
|
|
51
|
+
api_token_env_var: str = "BRAINSTEM_API_TOKEN"
|
|
52
|
+
listener: ListenerConfig = ListenerConfig()
|
|
53
|
+
defaults: RuntimeDefaults = RuntimeDefaults()
|
|
54
|
+
limits: RuntimeLimits = RuntimeLimits()
|
|
55
|
+
db: DBConfig = DBConfig(default_path=resolve_default_db_path())
|
|
56
|
+
|
|
57
|
+
def as_dict(self) -> Dict[str, Any]:
|
|
58
|
+
return {
|
|
59
|
+
"api_token_env_var": self.api_token_env_var,
|
|
60
|
+
"listener": asdict(self.listener),
|
|
61
|
+
"defaults": asdict(self.defaults),
|
|
62
|
+
"limits": asdict(self.limits),
|
|
63
|
+
"db": asdict(self.db),
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def get_runtime_config() -> RuntimeConfig:
|
|
68
|
+
return RuntimeConfig(
|
|
69
|
+
db=DBConfig(default_path=resolve_default_db_path()),
|
|
70
|
+
)
|