@simbimbo/brainstem 0.0.3 → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/README.md +25 -0
- package/brainstem/__init__.py +1 -1
- package/brainstem/adapters.py +120 -0
- package/brainstem/api.py +391 -57
- package/brainstem/config.py +70 -0
- package/brainstem/ingest.py +411 -33
- package/brainstem/interesting.py +56 -1
- package/brainstem/listener.py +175 -0
- package/brainstem/models.py +1 -0
- package/brainstem/recurrence.py +38 -1
- package/brainstem/source_drivers.py +150 -0
- package/brainstem/storage.py +305 -12
- package/docs/README.md +94 -0
- package/docs/adapters.md +97 -401
- package/docs/api.md +223 -278
- package/package.json +1 -1
- package/pyproject.toml +1 -1
- package/tests/test_adapters.py +94 -0
- package/tests/test_api.py +726 -0
- package/tests/test_canonicalization.py +8 -0
- package/tests/test_config.py +24 -0
- package/tests/test_file_ingest.py +77 -0
- package/tests/test_interesting.py +10 -0
- package/tests/test_listener.py +253 -0
- package/tests/test_recurrence.py +2 -0
- package/tests/test_source_drivers.py +95 -0
- package/tests/test_storage.py +101 -1
package/brainstem/api.py
CHANGED
|
@@ -1,36 +1,104 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
from datetime import datetime
|
|
4
|
+
import os
|
|
5
|
+
import secrets
|
|
6
|
+
import sys
|
|
7
|
+
from dataclasses import asdict
|
|
4
8
|
from typing import Any, Dict, List, Optional
|
|
5
9
|
|
|
6
10
|
import json
|
|
7
|
-
from fastapi import FastAPI, HTTPException, Query
|
|
8
|
-
from fastapi.responses import JSONResponse
|
|
11
|
+
from fastapi import Depends, FastAPI, Header, HTTPException, Query
|
|
9
12
|
from pydantic import BaseModel, Field
|
|
10
13
|
|
|
11
|
-
from .ingest import
|
|
14
|
+
from .ingest import ReplayAttempt, replay_raw_envelopes_by_ids, run_ingest_pipeline
|
|
12
15
|
from .interesting import interesting_items
|
|
16
|
+
from .config import get_runtime_config
|
|
17
|
+
from .fingerprint import normalize_message
|
|
18
|
+
from . import __version__
|
|
13
19
|
from .models import Candidate, RawInputEnvelope
|
|
14
|
-
from .recurrence import build_recurrence_candidates
|
|
15
20
|
from .storage import (
|
|
16
21
|
RAW_ENVELOPE_STATUSES,
|
|
22
|
+
extract_source_raw_envelope_ids,
|
|
17
23
|
get_ingest_stats,
|
|
18
|
-
init_db,
|
|
19
24
|
list_candidates,
|
|
25
|
+
list_signatures,
|
|
20
26
|
get_raw_envelope_by_id,
|
|
27
|
+
get_raw_envelopes_by_ids,
|
|
21
28
|
get_source_dimension_summaries,
|
|
29
|
+
get_source_status_summaries,
|
|
30
|
+
list_canonical_events,
|
|
22
31
|
list_recent_failed_raw_envelopes,
|
|
23
32
|
list_recent_raw_envelopes,
|
|
24
|
-
set_raw_envelope_status,
|
|
25
|
-
store_candidates,
|
|
26
|
-
store_events,
|
|
27
|
-
store_raw_envelopes,
|
|
28
|
-
store_signatures,
|
|
29
33
|
)
|
|
30
|
-
from .ingest import signatures_for_events
|
|
31
34
|
|
|
32
35
|
|
|
33
36
|
app = FastAPI(title="brAInstem Runtime")
|
|
37
|
+
RUNTIME_CONFIG = get_runtime_config()
|
|
38
|
+
API_TOKEN_ENV_VAR = RUNTIME_CONFIG.api_token_env_var
|
|
39
|
+
API_TOKEN_HEADER = "X-API-Token"
|
|
40
|
+
RUNTIME_DEFAULTS = asdict(RUNTIME_CONFIG.defaults)
|
|
41
|
+
RUNTIME_LIMITS = {k: v for k, v in asdict(RUNTIME_CONFIG.limits).items() if k != "replay_allowed_statuses"}
|
|
42
|
+
MAX_REPLAY_IDS = RUNTIME_LIMITS["replay_raw_max_ids"]
|
|
43
|
+
DEFAULT_INGEST_THRESHOLD = RUNTIME_DEFAULTS["ingest_threshold"]
|
|
44
|
+
DEFAULT_BATCH_THRESHOLD = RUNTIME_DEFAULTS["batch_threshold"]
|
|
45
|
+
DEFAULT_INTERESTING_LIMIT = RUNTIME_DEFAULTS["interesting_limit"]
|
|
46
|
+
DEFAULT_FAILURE_LIMIT = RUNTIME_DEFAULTS["failure_limit"]
|
|
47
|
+
DEFAULT_INGEST_RECENT_LIMIT = RUNTIME_DEFAULTS["ingest_recent_limit"]
|
|
48
|
+
DEFAULT_SOURCES_LIMIT = RUNTIME_DEFAULTS["sources_limit"]
|
|
49
|
+
DEFAULT_SOURCES_STATUS_LIMIT = RUNTIME_DEFAULTS["sources_status_limit"]
|
|
50
|
+
DEFAULT_REPLAY_THRESHOLD = RUNTIME_DEFAULTS["replay_threshold"]
|
|
51
|
+
DEFAULT_REPLAY_ALLOWED_STATUSES = tuple(RUNTIME_CONFIG.limits.replay_allowed_statuses)
|
|
52
|
+
CAPABILITIES = {
|
|
53
|
+
"ingest_endpoints": {
|
|
54
|
+
"single_event": True,
|
|
55
|
+
"batch_events": True,
|
|
56
|
+
"replay_raw": True,
|
|
57
|
+
},
|
|
58
|
+
"inspection_endpoints": {
|
|
59
|
+
"interesting": True,
|
|
60
|
+
"candidates": True,
|
|
61
|
+
"signatures": True,
|
|
62
|
+
"canonical_events": True,
|
|
63
|
+
"stats": True,
|
|
64
|
+
"raw_envelopes": True,
|
|
65
|
+
"failures": True,
|
|
66
|
+
"ingest_recent": True,
|
|
67
|
+
"sources": True,
|
|
68
|
+
},
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _configured_api_token() -> str:
|
|
73
|
+
return os.getenv(API_TOKEN_ENV_VAR, "").strip()
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def is_api_token_auth_enabled() -> bool:
|
|
77
|
+
return bool(_configured_api_token())
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _extract_bearer_token(authorization: Optional[str]) -> Optional[str]:
|
|
81
|
+
if not authorization:
|
|
82
|
+
return None
|
|
83
|
+
scheme, separator, token = authorization.partition(" ")
|
|
84
|
+
if separator != " ":
|
|
85
|
+
return None
|
|
86
|
+
if scheme.lower() != "bearer":
|
|
87
|
+
return None
|
|
88
|
+
return token.strip() or None
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _require_api_token(
|
|
92
|
+
x_api_token: Optional[str] = Header(default=None, alias=API_TOKEN_HEADER),
|
|
93
|
+
authorization: Optional[str] = Header(default=None),
|
|
94
|
+
) -> None:
|
|
95
|
+
configured_token = _configured_api_token()
|
|
96
|
+
if not configured_token:
|
|
97
|
+
return
|
|
98
|
+
|
|
99
|
+
provided_token = x_api_token or _extract_bearer_token(authorization)
|
|
100
|
+
if not provided_token or not secrets.compare_digest(provided_token, configured_token):
|
|
101
|
+
raise HTTPException(status_code=401, detail="missing or invalid API token")
|
|
34
102
|
|
|
35
103
|
|
|
36
104
|
class RawEnvelopeRequest(BaseModel):
|
|
@@ -57,10 +125,18 @@ class IngestEventRequest(RawEnvelopeRequest):
|
|
|
57
125
|
|
|
58
126
|
class IngestBatchRequest(BaseModel):
|
|
59
127
|
events: List[RawEnvelopeRequest]
|
|
60
|
-
threshold: int = Field(default=
|
|
128
|
+
threshold: int = Field(default=DEFAULT_BATCH_THRESHOLD, ge=1)
|
|
61
129
|
db_path: Optional[str] = None
|
|
62
130
|
|
|
63
131
|
|
|
132
|
+
class ReplayRawRequest(BaseModel):
|
|
133
|
+
db_path: str
|
|
134
|
+
raw_envelope_ids: List[int] = Field(default_factory=list, min_length=1, max_length=MAX_REPLAY_IDS)
|
|
135
|
+
threshold: int = Field(default=DEFAULT_REPLAY_THRESHOLD, ge=1)
|
|
136
|
+
force: bool = False
|
|
137
|
+
allowed_statuses: List[str] = Field(default_factory=lambda: list(DEFAULT_REPLAY_ALLOWED_STATUSES))
|
|
138
|
+
|
|
139
|
+
|
|
64
140
|
def _raw_envelope_from_request(payload: RawEnvelopeRequest) -> RawInputEnvelope:
|
|
65
141
|
return RawInputEnvelope(
|
|
66
142
|
tenant_id=payload.tenant_id,
|
|
@@ -96,6 +172,85 @@ def _candidate_from_row(row) -> Candidate:
|
|
|
96
172
|
)
|
|
97
173
|
|
|
98
174
|
|
|
175
|
+
def _candidate_inspection_from_row(row, db_path: str | None) -> Dict[str, Any]:
|
|
176
|
+
candidate = _candidate_from_row(row)
|
|
177
|
+
inspected = interesting_items([candidate], limit=1)
|
|
178
|
+
summary = inspected[0] if inspected else {
|
|
179
|
+
"title": candidate.title,
|
|
180
|
+
"summary": candidate.summary,
|
|
181
|
+
"decision_band": candidate.decision_band,
|
|
182
|
+
"attention_band": None,
|
|
183
|
+
"attention_score": candidate.score_total,
|
|
184
|
+
"score_total": candidate.score_total,
|
|
185
|
+
"attention_explanation": [],
|
|
186
|
+
+ "score_breakdown": candidate.score_breakdown,
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
raw_envelope_ids = extract_source_raw_envelope_ids(row["metadata_json"])
|
|
190
|
+
raw_envelopes = []
|
|
191
|
+
if raw_envelope_ids:
|
|
192
|
+
raw_envelope_rows = get_raw_envelopes_by_ids(raw_envelope_ids, db_path=db_path)
|
|
193
|
+
raw_envelope_index = {raw_row["id"]: raw_row for raw_row in raw_envelope_rows}
|
|
194
|
+
raw_envelopes = [
|
|
195
|
+
_raw_envelope_from_row(raw_envelope_index[row_id])
|
|
196
|
+
for row_id in raw_envelope_ids
|
|
197
|
+
if row_id in raw_envelope_index
|
|
198
|
+
]
|
|
199
|
+
|
|
200
|
+
return {
|
|
201
|
+
**summary,
|
|
202
|
+
"id": row["id"],
|
|
203
|
+
"candidate_type": candidate.candidate_type,
|
|
204
|
+
"confidence": candidate.confidence,
|
|
205
|
+
"source_signature_ids": candidate.source_signature_ids,
|
|
206
|
+
"source_event_ids": candidate.source_event_ids,
|
|
207
|
+
"score_breakdown": candidate.score_breakdown,
|
|
208
|
+
"raw_envelope_ids": raw_envelope_ids,
|
|
209
|
+
"raw_envelopes": raw_envelopes,
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _signature_inspection_from_row(row, db_path: str | None) -> Dict[str, Any]:
|
|
214
|
+
metadata = json.loads(row["metadata_json"] or "{}")
|
|
215
|
+
if not isinstance(metadata, dict):
|
|
216
|
+
metadata = {}
|
|
217
|
+
raw_envelope_ids = extract_source_raw_envelope_ids(row["metadata_json"])
|
|
218
|
+
occurrence_count = int(row["occurrence_count"]) if row["occurrence_count"] is not None else 0
|
|
219
|
+
raw_envelope_count = len(raw_envelope_ids)
|
|
220
|
+
|
|
221
|
+
return {
|
|
222
|
+
"signature_key": row["signature_key"],
|
|
223
|
+
"event_family": row["event_family"],
|
|
224
|
+
"normalized_pattern": row["normalized_pattern"],
|
|
225
|
+
"service": row["service"],
|
|
226
|
+
"occurrence_count": occurrence_count,
|
|
227
|
+
"raw_envelope_ids": raw_envelope_ids,
|
|
228
|
+
"raw_envelope_count": raw_envelope_count,
|
|
229
|
+
"recurrence": {
|
|
230
|
+
"occurrence_count": occurrence_count,
|
|
231
|
+
"raw_envelope_count": raw_envelope_count,
|
|
232
|
+
"signature_id": row["id"],
|
|
233
|
+
},
|
|
234
|
+
"metadata": metadata,
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _canonical_event_from_row(row) -> Dict[str, Any]:
|
|
239
|
+
message_raw = row["message_raw"] or ""
|
|
240
|
+
return {
|
|
241
|
+
"raw_envelope_id": row["id"],
|
|
242
|
+
"tenant_id": row["tenant_id"],
|
|
243
|
+
"source": row["source_type"],
|
|
244
|
+
"source_type": row["source_type"],
|
|
245
|
+
"message_raw": message_raw,
|
|
246
|
+
"timestamp": row["timestamp"],
|
|
247
|
+
"host": row["host"] or "",
|
|
248
|
+
"service": row["service"] or "",
|
|
249
|
+
"severity": row["severity"] or "info",
|
|
250
|
+
"message_normalized": normalize_message(message_raw),
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
|
|
99
254
|
def _raw_envelope_from_row(row) -> Dict[str, Any]:
|
|
100
255
|
return {
|
|
101
256
|
"id": row["id"],
|
|
@@ -119,31 +274,49 @@ def _raw_envelope_from_row(row) -> Dict[str, Any]:
|
|
|
119
274
|
}
|
|
120
275
|
|
|
121
276
|
|
|
277
|
+
def _replay_attempt_from_item(item: ReplayAttempt) -> Dict[str, Any]:
|
|
278
|
+
return {
|
|
279
|
+
"raw_envelope_id": item.raw_envelope_id,
|
|
280
|
+
"reason": item.reason,
|
|
281
|
+
"status": item.status,
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def _runtime_summary() -> Dict[str, Any]:
|
|
286
|
+
runtime_config = get_runtime_config()
|
|
287
|
+
return {
|
|
288
|
+
"version": __version__,
|
|
289
|
+
"runtime": {
|
|
290
|
+
"python_version": ".".join(str(item) for item in sys.version_info[:3]),
|
|
291
|
+
"api_token_env": runtime_config.api_token_env_var,
|
|
292
|
+
"config": runtime_config.as_dict(),
|
|
293
|
+
},
|
|
294
|
+
"capability_flags": CAPABILITIES,
|
|
295
|
+
"auth_state": {
|
|
296
|
+
"api_token_configured": is_api_token_auth_enabled(),
|
|
297
|
+
"supports_x_api_token_header": True,
|
|
298
|
+
"supports_bearer_header": True,
|
|
299
|
+
},
|
|
300
|
+
"defaults": dict(RUNTIME_DEFAULTS),
|
|
301
|
+
"limits": dict(RUNTIME_LIMITS),
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def _status_payload() -> Dict[str, Any]:
|
|
306
|
+
runtime_summary = _runtime_summary()
|
|
307
|
+
return {
|
|
308
|
+
"ok": True,
|
|
309
|
+
"status": "ok",
|
|
310
|
+
"runtime": runtime_summary,
|
|
311
|
+
"api_token_enabled": runtime_summary["auth_state"]["api_token_configured"],
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
|
|
122
315
|
def _run_ingest_batch(raw_events: List[RawInputEnvelope], *, threshold: int, db_path: Optional[str]) -> Dict[str, Any]:
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
events = []
|
|
129
|
-
parse_failed = 0
|
|
130
|
-
for idx, raw_event in enumerate(raw_events):
|
|
131
|
-
raw_envelope_id = raw_envelope_ids[idx] if idx < len(raw_envelope_ids) else None
|
|
132
|
-
try:
|
|
133
|
-
canonical_event = canonicalize_raw_input_envelope(raw_event)
|
|
134
|
-
except Exception as exc:
|
|
135
|
-
parse_failed += 1
|
|
136
|
-
if raw_envelope_id is not None:
|
|
137
|
-
set_raw_envelope_status(
|
|
138
|
-
raw_envelope_id,
|
|
139
|
-
"parse_failed",
|
|
140
|
-
db_path=db_path,
|
|
141
|
-
failure_reason=str(exc),
|
|
142
|
-
)
|
|
143
|
-
continue
|
|
144
|
-
events.append(canonical_event)
|
|
145
|
-
if raw_envelope_id is not None:
|
|
146
|
-
set_raw_envelope_status(raw_envelope_id, "canonicalized", db_path=db_path)
|
|
316
|
+
result = run_ingest_pipeline(raw_events, threshold=threshold, db_path=db_path)
|
|
317
|
+
events = result.events
|
|
318
|
+
parse_failed = result.parse_failed
|
|
319
|
+
item_results = [asdict(item) for item in result.item_results]
|
|
147
320
|
|
|
148
321
|
if not events:
|
|
149
322
|
return {
|
|
@@ -151,16 +324,14 @@ def _run_ingest_batch(raw_events: List[RawInputEnvelope], *, threshold: int, db_
|
|
|
151
324
|
"event_count": 0,
|
|
152
325
|
"signature_count": 0,
|
|
153
326
|
"candidate_count": 0,
|
|
327
|
+
"item_count": len(raw_events),
|
|
328
|
+
"item_results": item_results,
|
|
154
329
|
"parse_failed": parse_failed,
|
|
155
330
|
"interesting_items": [],
|
|
156
331
|
}
|
|
157
332
|
|
|
158
|
-
signatures =
|
|
159
|
-
candidates =
|
|
160
|
-
if db_path:
|
|
161
|
-
store_events(events, db_path)
|
|
162
|
-
store_signatures(signatures, db_path)
|
|
163
|
-
store_candidates(candidates, db_path)
|
|
333
|
+
signatures = result.signatures
|
|
334
|
+
candidates = result.candidates
|
|
164
335
|
|
|
165
336
|
return {
|
|
166
337
|
"ok": True,
|
|
@@ -168,27 +339,61 @@ def _run_ingest_batch(raw_events: List[RawInputEnvelope], *, threshold: int, db_
|
|
|
168
339
|
"event_count": len(events),
|
|
169
340
|
"signature_count": len({sig.signature_key for sig in signatures}),
|
|
170
341
|
"candidate_count": len(candidates),
|
|
342
|
+
"item_count": len(raw_events),
|
|
343
|
+
"item_results": item_results,
|
|
171
344
|
"parse_failed": parse_failed,
|
|
172
345
|
"interesting_items": interesting_items(candidates, limit=max(1, 5)),
|
|
173
346
|
}
|
|
174
347
|
|
|
175
348
|
|
|
176
|
-
@app.post("/
|
|
177
|
-
def
|
|
349
|
+
@app.post("/replay/raw", dependencies=[Depends(_require_api_token)])
|
|
350
|
+
def replay_raw(payload: ReplayRawRequest) -> Dict[str, Any]:
|
|
351
|
+
if not payload.raw_envelope_ids:
|
|
352
|
+
raise HTTPException(status_code=422, detail="raw_envelope_ids is required")
|
|
353
|
+
if payload.allowed_statuses:
|
|
354
|
+
invalid = [status for status in payload.allowed_statuses if status not in RAW_ENVELOPE_STATUSES]
|
|
355
|
+
if invalid:
|
|
356
|
+
raise HTTPException(
|
|
357
|
+
status_code=422,
|
|
358
|
+
detail=f"invalid status in allowed_statuses: {invalid}; expected one of: {', '.join(RAW_ENVELOPE_STATUSES)}",
|
|
359
|
+
)
|
|
360
|
+
|
|
361
|
+
result = replay_raw_envelopes_by_ids(
|
|
362
|
+
payload.raw_envelope_ids,
|
|
363
|
+
db_path=payload.db_path,
|
|
364
|
+
threshold=payload.threshold,
|
|
365
|
+
force=payload.force,
|
|
366
|
+
allowed_statuses=payload.allowed_statuses,
|
|
367
|
+
)
|
|
368
|
+
return {
|
|
369
|
+
"ok": True,
|
|
370
|
+
"db_path": payload.db_path,
|
|
371
|
+
"requested_raw_envelope_ids": result.requested_raw_envelope_ids,
|
|
372
|
+
"attempted_raw_envelope_ids": result.attempted_raw_envelope_ids,
|
|
373
|
+
"skipped": [_replay_attempt_from_item(item) for item in result.skipped],
|
|
374
|
+
"event_count": len(result.events),
|
|
375
|
+
"signature_count": len(result.signatures),
|
|
376
|
+
"candidate_count": len(result.candidates),
|
|
377
|
+
"parse_failed": result.parse_failed,
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
@app.post("/ingest/event", dependencies=[Depends(_require_api_token)])
|
|
382
|
+
def ingest_event(payload: IngestEventRequest, threshold: int = DEFAULT_INGEST_THRESHOLD, db_path: Optional[str] = None) -> Dict[str, Any]:
|
|
178
383
|
if threshold < 1:
|
|
179
384
|
raise HTTPException(status_code=422, detail="threshold must be >= 1")
|
|
180
385
|
return _run_ingest_batch([_raw_envelope_from_request(payload)], threshold=threshold, db_path=db_path)
|
|
181
386
|
|
|
182
387
|
|
|
183
|
-
@app.post("/ingest/batch")
|
|
388
|
+
@app.post("/ingest/batch", dependencies=[Depends(_require_api_token)])
|
|
184
389
|
def ingest_batch(payload: IngestBatchRequest) -> Dict[str, Any]:
|
|
185
390
|
raw_events = [_raw_envelope_from_request(event) for event in payload.events]
|
|
186
391
|
return _run_ingest_batch(raw_events, threshold=payload.threshold, db_path=payload.db_path)
|
|
187
392
|
|
|
188
393
|
|
|
189
|
-
@app.get("/interesting")
|
|
394
|
+
@app.get("/interesting", dependencies=[Depends(_require_api_token)])
|
|
190
395
|
def get_interesting(
|
|
191
|
-
limit: int = Query(default=
|
|
396
|
+
limit: int = Query(default=DEFAULT_INTERESTING_LIMIT, ge=1),
|
|
192
397
|
db_path: Optional[str] = None,
|
|
193
398
|
) -> Dict[str, Any]:
|
|
194
399
|
if not db_path:
|
|
@@ -198,14 +403,58 @@ def get_interesting(
|
|
|
198
403
|
return {"ok": True, "items": interesting_items(candidates, limit=limit)}
|
|
199
404
|
|
|
200
405
|
|
|
201
|
-
@app.get("/
|
|
406
|
+
@app.get("/candidates", dependencies=[Depends(_require_api_token)])
|
|
407
|
+
def get_candidates(
|
|
408
|
+
limit: int = Query(default=DEFAULT_INTERESTING_LIMIT, ge=1),
|
|
409
|
+
db_path: Optional[str] = None,
|
|
410
|
+
candidate_type: Optional[str] = None,
|
|
411
|
+
decision_band: Optional[str] = None,
|
|
412
|
+
min_score_total: Optional[float] = Query(default=None, ge=0),
|
|
413
|
+
) -> Dict[str, Any]:
|
|
414
|
+
if not db_path:
|
|
415
|
+
return {"ok": True, "count": 0, "items": []}
|
|
416
|
+
|
|
417
|
+
rows = list_candidates(
|
|
418
|
+
db_path=db_path,
|
|
419
|
+
limit=limit,
|
|
420
|
+
candidate_type=candidate_type,
|
|
421
|
+
decision_band=decision_band,
|
|
422
|
+
min_score_total=min_score_total,
|
|
423
|
+
)
|
|
424
|
+
items = [_candidate_inspection_from_row(row, db_path=db_path) for row in rows]
|
|
425
|
+
return {"ok": True, "count": len(items), "items": items}
|
|
426
|
+
|
|
427
|
+
|
|
428
|
+
@app.get("/signatures", dependencies=[Depends(_require_api_token)])
|
|
429
|
+
def get_signatures(
|
|
430
|
+
limit: int = Query(default=DEFAULT_INTERESTING_LIMIT, ge=1),
|
|
431
|
+
db_path: Optional[str] = None,
|
|
432
|
+
event_family: Optional[str] = None,
|
|
433
|
+
service: Optional[str] = None,
|
|
434
|
+
min_occurrence_count: Optional[int] = Query(default=None, ge=1),
|
|
435
|
+
) -> Dict[str, Any]:
|
|
436
|
+
if not db_path:
|
|
437
|
+
return {"ok": True, "count": 0, "items": []}
|
|
438
|
+
|
|
439
|
+
rows = list_signatures(
|
|
440
|
+
db_path=db_path,
|
|
441
|
+
limit=limit,
|
|
442
|
+
event_family=event_family,
|
|
443
|
+
service=service,
|
|
444
|
+
min_occurrence_count=min_occurrence_count,
|
|
445
|
+
)
|
|
446
|
+
items = [_signature_inspection_from_row(row, db_path=db_path) for row in rows]
|
|
447
|
+
return {"ok": True, "count": len(items), "items": items}
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
@app.get("/stats", dependencies=[Depends(_require_api_token)])
|
|
202
451
|
def get_stats(db_path: Optional[str] = None) -> Dict[str, Any]:
|
|
203
452
|
return {"ok": True, **get_ingest_stats(db_path)}
|
|
204
453
|
|
|
205
454
|
|
|
206
|
-
@app.get("/failures")
|
|
455
|
+
@app.get("/failures", dependencies=[Depends(_require_api_token)])
|
|
207
456
|
def get_failures(
|
|
208
|
-
limit: int = Query(default=
|
|
457
|
+
limit: int = Query(default=DEFAULT_FAILURE_LIMIT, ge=1),
|
|
209
458
|
status: Optional[str] = None,
|
|
210
459
|
db_path: Optional[str] = None,
|
|
211
460
|
) -> Dict[str, Any]:
|
|
@@ -220,9 +469,38 @@ def get_failures(
|
|
|
220
469
|
return {"ok": True, "items": items, "count": len(items), "status": status}
|
|
221
470
|
|
|
222
471
|
|
|
223
|
-
@app.get("/
|
|
472
|
+
@app.get("/raw_envelopes", dependencies=[Depends(_require_api_token)])
|
|
473
|
+
def get_raw_envelopes(
|
|
474
|
+
limit: int = Query(default=DEFAULT_INGEST_RECENT_LIMIT, ge=1),
|
|
475
|
+
status: Optional[str] = None,
|
|
476
|
+
tenant_id: Optional[str] = None,
|
|
477
|
+
source_type: Optional[str] = None,
|
|
478
|
+
source_id: Optional[str] = None,
|
|
479
|
+
source_path: Optional[str] = None,
|
|
480
|
+
db_path: Optional[str] = None,
|
|
481
|
+
) -> Dict[str, Any]:
|
|
482
|
+
if status is not None and status not in RAW_ENVELOPE_STATUSES:
|
|
483
|
+
raise HTTPException(
|
|
484
|
+
status_code=422,
|
|
485
|
+
detail=f"invalid status '{status}'; expected one of: {', '.join(RAW_ENVELOPE_STATUSES)}",
|
|
486
|
+
)
|
|
487
|
+
rows = list_recent_raw_envelopes(
|
|
488
|
+
db_path=db_path,
|
|
489
|
+
status=status,
|
|
490
|
+
limit=limit,
|
|
491
|
+
tenant_id=tenant_id,
|
|
492
|
+
source_type=source_type,
|
|
493
|
+
source_id=source_id,
|
|
494
|
+
source_path=source_path,
|
|
495
|
+
failures_only=False,
|
|
496
|
+
)
|
|
497
|
+
items = [_raw_envelope_from_row(row) for row in rows]
|
|
498
|
+
return {"ok": True, "items": items, "count": len(items)}
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
@app.get("/ingest/recent", dependencies=[Depends(_require_api_token)])
|
|
224
502
|
def get_ingest_recent(
|
|
225
|
-
limit: int = Query(default=
|
|
503
|
+
limit: int = Query(default=DEFAULT_INGEST_RECENT_LIMIT, ge=1),
|
|
226
504
|
status: Optional[str] = None,
|
|
227
505
|
db_path: Optional[str] = None,
|
|
228
506
|
) -> Dict[str, Any]:
|
|
@@ -236,15 +514,61 @@ def get_ingest_recent(
|
|
|
236
514
|
return {"ok": True, "items": items, "count": len(items), "status": status}
|
|
237
515
|
|
|
238
516
|
|
|
239
|
-
@app.get("/
|
|
517
|
+
@app.get("/canonical_events", dependencies=[Depends(_require_api_token)])
|
|
518
|
+
def get_canonical_events(
|
|
519
|
+
limit: int = Query(default=DEFAULT_INGEST_RECENT_LIMIT, ge=1),
|
|
520
|
+
tenant_id: Optional[str] = None,
|
|
521
|
+
source: Optional[str] = None,
|
|
522
|
+
host: Optional[str] = None,
|
|
523
|
+
service: Optional[str] = None,
|
|
524
|
+
severity: Optional[str] = None,
|
|
525
|
+
db_path: Optional[str] = None,
|
|
526
|
+
) -> Dict[str, Any]:
|
|
527
|
+
if not db_path:
|
|
528
|
+
return {"ok": True, "items": [], "count": 0}
|
|
529
|
+
|
|
530
|
+
rows = list_canonical_events(
|
|
531
|
+
db_path=db_path,
|
|
532
|
+
limit=limit,
|
|
533
|
+
tenant_id=tenant_id,
|
|
534
|
+
source_type=source,
|
|
535
|
+
host=host,
|
|
536
|
+
service=service,
|
|
537
|
+
severity=severity,
|
|
538
|
+
)
|
|
539
|
+
items = [_canonical_event_from_row(row) for row in rows]
|
|
540
|
+
return {"ok": True, "items": items, "count": len(items)}
|
|
541
|
+
|
|
542
|
+
|
|
543
|
+
@app.get("/sources", dependencies=[Depends(_require_api_token)])
|
|
240
544
|
def get_sources(
|
|
241
|
-
limit: int = Query(default=
|
|
545
|
+
limit: int = Query(default=DEFAULT_SOURCES_LIMIT, ge=1),
|
|
242
546
|
db_path: Optional[str] = None,
|
|
243
547
|
) -> Dict[str, Any]:
|
|
244
548
|
return {"ok": True, "items": get_source_dimension_summaries(db_path=db_path, limit=limit)}
|
|
245
549
|
|
|
246
550
|
|
|
247
|
-
@app.get("/
|
|
551
|
+
@app.get("/sources/status", dependencies=[Depends(_require_api_token)])
|
|
552
|
+
def get_sources_status(
|
|
553
|
+
limit: int = Query(default=DEFAULT_SOURCES_STATUS_LIMIT, ge=1),
|
|
554
|
+
tenant_id: Optional[str] = None,
|
|
555
|
+
source_type: Optional[str] = None,
|
|
556
|
+
source_id: Optional[str] = None,
|
|
557
|
+
source_path: Optional[str] = None,
|
|
558
|
+
db_path: Optional[str] = None,
|
|
559
|
+
) -> Dict[str, Any]:
|
|
560
|
+
items = get_source_status_summaries(
|
|
561
|
+
db_path=db_path,
|
|
562
|
+
limit=limit,
|
|
563
|
+
tenant_id=tenant_id,
|
|
564
|
+
source_type=source_type,
|
|
565
|
+
source_id=source_id,
|
|
566
|
+
source_path=source_path,
|
|
567
|
+
)
|
|
568
|
+
return {"ok": True, "items": items, "count": len(items)}
|
|
569
|
+
|
|
570
|
+
|
|
571
|
+
@app.get("/failures/{raw_envelope_id}", dependencies=[Depends(_require_api_token)])
|
|
248
572
|
def get_failure(raw_envelope_id: int, db_path: Optional[str] = None) -> Dict[str, Any]:
|
|
249
573
|
row = get_raw_envelope_by_id(raw_envelope_id, db_path=db_path)
|
|
250
574
|
if row is None:
|
|
@@ -253,5 +577,15 @@ def get_failure(raw_envelope_id: int, db_path: Optional[str] = None) -> Dict[str
|
|
|
253
577
|
|
|
254
578
|
|
|
255
579
|
@app.get("/healthz")
|
|
256
|
-
def healthz() -> Dict[str,
|
|
257
|
-
return
|
|
580
|
+
def healthz() -> Dict[str, Any]:
|
|
581
|
+
return _status_payload()
|
|
582
|
+
|
|
583
|
+
|
|
584
|
+
@app.get("/runtime")
|
|
585
|
+
def runtime() -> Dict[str, Any]:
|
|
586
|
+
return {"ok": True, "runtime": _runtime_summary()}
|
|
587
|
+
|
|
588
|
+
|
|
589
|
+
@app.get("/status")
|
|
590
|
+
def status() -> Dict[str, Any]:
|
|
591
|
+
return _status_payload()
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from dataclasses import asdict, dataclass
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Any, Dict
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def resolve_default_db_path() -> str:
|
|
10
|
+
configured_db_path = os.getenv("BRAINSTEM_DB_PATH", "").strip()
|
|
11
|
+
if configured_db_path:
|
|
12
|
+
return configured_db_path
|
|
13
|
+
return str(Path(".brainstem-state") / "brainstem.sqlite3")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass(frozen=True)
|
|
17
|
+
class ListenerConfig:
|
|
18
|
+
syslog_host: str = "127.0.0.1"
|
|
19
|
+
syslog_port: int = 5514
|
|
20
|
+
syslog_source_path: str = "/dev/udp"
|
|
21
|
+
syslog_socket_timeout: float = 0.5
|
|
22
|
+
ingest_threshold: int = 2
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(frozen=True)
|
|
26
|
+
class RuntimeDefaults:
|
|
27
|
+
ingest_threshold: int = 2
|
|
28
|
+
batch_threshold: int = 2
|
|
29
|
+
interesting_limit: int = 5
|
|
30
|
+
failure_limit: int = 20
|
|
31
|
+
ingest_recent_limit: int = 20
|
|
32
|
+
sources_limit: int = 10
|
|
33
|
+
sources_status_limit: int = 20
|
|
34
|
+
replay_threshold: int = 2
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass(frozen=True)
|
|
38
|
+
class RuntimeLimits:
|
|
39
|
+
replay_raw_max_ids: int = 32
|
|
40
|
+
status_filter_limit: int = 20
|
|
41
|
+
replay_allowed_statuses: tuple[str, ...] = ("received", "parse_failed")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass(frozen=True)
|
|
45
|
+
class DBConfig:
|
|
46
|
+
default_path: str
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@dataclass(frozen=True)
|
|
50
|
+
class RuntimeConfig:
|
|
51
|
+
api_token_env_var: str = "BRAINSTEM_API_TOKEN"
|
|
52
|
+
listener: ListenerConfig = ListenerConfig()
|
|
53
|
+
defaults: RuntimeDefaults = RuntimeDefaults()
|
|
54
|
+
limits: RuntimeLimits = RuntimeLimits()
|
|
55
|
+
db: DBConfig = DBConfig(default_path=resolve_default_db_path())
|
|
56
|
+
|
|
57
|
+
def as_dict(self) -> Dict[str, Any]:
|
|
58
|
+
return {
|
|
59
|
+
"api_token_env_var": self.api_token_env_var,
|
|
60
|
+
"listener": asdict(self.listener),
|
|
61
|
+
"defaults": asdict(self.defaults),
|
|
62
|
+
"limits": asdict(self.limits),
|
|
63
|
+
"db": asdict(self.db),
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def get_runtime_config() -> RuntimeConfig:
|
|
68
|
+
return RuntimeConfig(
|
|
69
|
+
db=DBConfig(default_path=resolve_default_db_path()),
|
|
70
|
+
)
|