@simbimbo/brainstem 0.0.3 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/README.md +26 -0
- package/brainstem/__init__.py +1 -1
- package/brainstem/adapters.py +120 -0
- package/brainstem/api.py +468 -57
- package/brainstem/config.py +136 -0
- package/brainstem/connectors/logicmonitor.py +57 -0
- package/brainstem/demo.py +16 -2
- package/brainstem/fingerprint.py +54 -0
- package/brainstem/ingest.py +440 -33
- package/brainstem/interesting.py +56 -1
- package/brainstem/listener.py +181 -0
- package/brainstem/models.py +1 -0
- package/brainstem/recurrence.py +63 -9
- package/brainstem/scoring.py +6 -4
- package/brainstem/source_drivers.py +179 -0
- package/brainstem/storage.py +389 -12
- package/docs/README.md +103 -0
- package/docs/api.md +260 -280
- package/package.json +1 -1
- package/pyproject.toml +1 -1
- package/tests/test_adapters.py +95 -0
- package/tests/test_api.py +812 -0
- package/tests/test_canonicalization.py +8 -0
- package/tests/test_config.py +39 -0
- package/tests/test_file_ingest.py +77 -0
- package/tests/test_fingerprint.py +51 -1
- package/tests/test_interesting.py +10 -0
- package/tests/test_listener.py +253 -0
- package/tests/test_logicmonitor.py +54 -1
- package/tests/test_recurrence.py +16 -0
- package/tests/test_source_drivers.py +95 -0
- package/tests/test_storage.py +178 -1
package/brainstem/api.py
CHANGED
|
@@ -1,36 +1,106 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
from datetime import datetime
|
|
4
|
+
import os
|
|
5
|
+
import secrets
|
|
6
|
+
import sys
|
|
7
|
+
from dataclasses import asdict
|
|
4
8
|
from typing import Any, Dict, List, Optional
|
|
5
9
|
|
|
6
10
|
import json
|
|
7
|
-
from fastapi import FastAPI, HTTPException, Query
|
|
8
|
-
from fastapi.responses import JSONResponse
|
|
11
|
+
from fastapi import Depends, FastAPI, Header, HTTPException, Query
|
|
9
12
|
from pydantic import BaseModel, Field
|
|
10
13
|
|
|
11
|
-
from .ingest import
|
|
14
|
+
from .ingest import ReplayAttempt, run_ingest_logicmonitor_events, replay_raw_envelopes_by_ids, run_ingest_pipeline
|
|
12
15
|
from .interesting import interesting_items
|
|
16
|
+
from .config import get_runtime_config
|
|
17
|
+
from .fingerprint import normalize_message
|
|
18
|
+
from . import __version__
|
|
13
19
|
from .models import Candidate, RawInputEnvelope
|
|
14
|
-
from .
|
|
20
|
+
from .source_drivers import list_source_driver_types
|
|
15
21
|
from .storage import (
|
|
16
22
|
RAW_ENVELOPE_STATUSES,
|
|
23
|
+
extract_source_raw_envelope_ids,
|
|
17
24
|
get_ingest_stats,
|
|
18
|
-
init_db,
|
|
19
25
|
list_candidates,
|
|
26
|
+
list_signatures,
|
|
20
27
|
get_raw_envelope_by_id,
|
|
28
|
+
get_raw_envelopes_by_ids,
|
|
21
29
|
get_source_dimension_summaries,
|
|
30
|
+
get_source_status_summaries,
|
|
31
|
+
list_canonical_events,
|
|
22
32
|
list_recent_failed_raw_envelopes,
|
|
23
33
|
list_recent_raw_envelopes,
|
|
24
|
-
set_raw_envelope_status,
|
|
25
|
-
store_candidates,
|
|
26
|
-
store_events,
|
|
27
|
-
store_raw_envelopes,
|
|
28
|
-
store_signatures,
|
|
29
34
|
)
|
|
30
|
-
from .ingest import signatures_for_events
|
|
31
35
|
|
|
32
36
|
|
|
33
37
|
app = FastAPI(title="brAInstem Runtime")
|
|
38
|
+
RUNTIME_CONFIG = get_runtime_config()
|
|
39
|
+
API_TOKEN_ENV_VAR = RUNTIME_CONFIG.api_token_env_var
|
|
40
|
+
API_TOKEN_HEADER = "X-API-Token"
|
|
41
|
+
RUNTIME_DEFAULTS = asdict(RUNTIME_CONFIG.defaults)
|
|
42
|
+
RUNTIME_LIMITS = {k: v for k, v in asdict(RUNTIME_CONFIG.limits).items() if k != "replay_allowed_statuses"}
|
|
43
|
+
MAX_REPLAY_IDS = RUNTIME_LIMITS["replay_raw_max_ids"]
|
|
44
|
+
DEFAULT_INGEST_THRESHOLD = RUNTIME_DEFAULTS["ingest_threshold"]
|
|
45
|
+
DEFAULT_BATCH_THRESHOLD = RUNTIME_DEFAULTS["batch_threshold"]
|
|
46
|
+
DEFAULT_INTERESTING_LIMIT = RUNTIME_DEFAULTS["interesting_limit"]
|
|
47
|
+
DEFAULT_FAILURE_LIMIT = RUNTIME_DEFAULTS["failure_limit"]
|
|
48
|
+
DEFAULT_INGEST_RECENT_LIMIT = RUNTIME_DEFAULTS["ingest_recent_limit"]
|
|
49
|
+
DEFAULT_SOURCES_LIMIT = RUNTIME_DEFAULTS["sources_limit"]
|
|
50
|
+
DEFAULT_SOURCES_STATUS_LIMIT = RUNTIME_DEFAULTS["sources_status_limit"]
|
|
51
|
+
DEFAULT_REPLAY_THRESHOLD = RUNTIME_DEFAULTS["replay_threshold"]
|
|
52
|
+
DEFAULT_REPLAY_ALLOWED_STATUSES = tuple(RUNTIME_CONFIG.limits.replay_allowed_statuses)
|
|
53
|
+
CAPABILITIES = {
|
|
54
|
+
"ingest_endpoints": {
|
|
55
|
+
"single_event": True,
|
|
56
|
+
"batch_events": True,
|
|
57
|
+
"logicmonitor_events": True,
|
|
58
|
+
"replay_raw": True,
|
|
59
|
+
},
|
|
60
|
+
"inspection_endpoints": {
|
|
61
|
+
"interesting": True,
|
|
62
|
+
"candidates": True,
|
|
63
|
+
"signatures": True,
|
|
64
|
+
"canonical_events": True,
|
|
65
|
+
"stats": True,
|
|
66
|
+
"raw_envelopes": True,
|
|
67
|
+
"failures": True,
|
|
68
|
+
"ingest_recent": True,
|
|
69
|
+
"sources": True,
|
|
70
|
+
},
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _configured_api_token() -> str:
|
|
75
|
+
return os.getenv(API_TOKEN_ENV_VAR, "").strip()
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def is_api_token_auth_enabled() -> bool:
|
|
79
|
+
return bool(_configured_api_token())
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _extract_bearer_token(authorization: Optional[str]) -> Optional[str]:
|
|
83
|
+
if not authorization:
|
|
84
|
+
return None
|
|
85
|
+
scheme, separator, token = authorization.partition(" ")
|
|
86
|
+
if separator != " ":
|
|
87
|
+
return None
|
|
88
|
+
if scheme.lower() != "bearer":
|
|
89
|
+
return None
|
|
90
|
+
return token.strip() or None
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _require_api_token(
|
|
94
|
+
x_api_token: Optional[str] = Header(default=None, alias=API_TOKEN_HEADER),
|
|
95
|
+
authorization: Optional[str] = Header(default=None),
|
|
96
|
+
) -> None:
|
|
97
|
+
configured_token = _configured_api_token()
|
|
98
|
+
if not configured_token:
|
|
99
|
+
return
|
|
100
|
+
|
|
101
|
+
provided_token = x_api_token or _extract_bearer_token(authorization)
|
|
102
|
+
if not provided_token or not secrets.compare_digest(provided_token, configured_token):
|
|
103
|
+
raise HTTPException(status_code=401, detail="missing or invalid API token")
|
|
34
104
|
|
|
35
105
|
|
|
36
106
|
class RawEnvelopeRequest(BaseModel):
|
|
@@ -57,7 +127,23 @@ class IngestEventRequest(RawEnvelopeRequest):
|
|
|
57
127
|
|
|
58
128
|
class IngestBatchRequest(BaseModel):
|
|
59
129
|
events: List[RawEnvelopeRequest]
|
|
60
|
-
threshold: int = Field(default=
|
|
130
|
+
threshold: int = Field(default=DEFAULT_BATCH_THRESHOLD, ge=1)
|
|
131
|
+
db_path: Optional[str] = None
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
class ReplayRawRequest(BaseModel):
|
|
135
|
+
db_path: str
|
|
136
|
+
raw_envelope_ids: List[int] = Field(default_factory=list, min_length=1, max_length=MAX_REPLAY_IDS)
|
|
137
|
+
threshold: int = Field(default=DEFAULT_REPLAY_THRESHOLD, ge=1)
|
|
138
|
+
force: bool = False
|
|
139
|
+
allowed_statuses: List[str] = Field(default_factory=lambda: list(DEFAULT_REPLAY_ALLOWED_STATUSES))
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
class IngestLogicMonitorRequest(BaseModel):
|
|
143
|
+
tenant_id: str
|
|
144
|
+
events: List[Dict[str, Any]] = Field(default_factory=list, min_length=1)
|
|
145
|
+
source_path: str = "/logicmonitor/ingest"
|
|
146
|
+
threshold: int = Field(default=DEFAULT_BATCH_THRESHOLD, ge=1)
|
|
61
147
|
db_path: Optional[str] = None
|
|
62
148
|
|
|
63
149
|
|
|
@@ -96,6 +182,85 @@ def _candidate_from_row(row) -> Candidate:
|
|
|
96
182
|
)
|
|
97
183
|
|
|
98
184
|
|
|
185
|
+
def _candidate_inspection_from_row(row, db_path: str | None) -> Dict[str, Any]:
|
|
186
|
+
candidate = _candidate_from_row(row)
|
|
187
|
+
inspected = interesting_items([candidate], limit=1)
|
|
188
|
+
summary = inspected[0] if inspected else {
|
|
189
|
+
"title": candidate.title,
|
|
190
|
+
"summary": candidate.summary,
|
|
191
|
+
"decision_band": candidate.decision_band,
|
|
192
|
+
"attention_band": None,
|
|
193
|
+
"attention_score": candidate.score_total,
|
|
194
|
+
"score_total": candidate.score_total,
|
|
195
|
+
"attention_explanation": [],
|
|
196
|
+
"score_breakdown": candidate.score_breakdown,
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
raw_envelope_ids = extract_source_raw_envelope_ids(row["metadata_json"])
|
|
200
|
+
raw_envelopes = []
|
|
201
|
+
if raw_envelope_ids:
|
|
202
|
+
raw_envelope_rows = get_raw_envelopes_by_ids(raw_envelope_ids, db_path=db_path)
|
|
203
|
+
raw_envelope_index = {raw_row["id"]: raw_row for raw_row in raw_envelope_rows}
|
|
204
|
+
raw_envelopes = [
|
|
205
|
+
_raw_envelope_from_row(raw_envelope_index[row_id])
|
|
206
|
+
for row_id in raw_envelope_ids
|
|
207
|
+
if row_id in raw_envelope_index
|
|
208
|
+
]
|
|
209
|
+
|
|
210
|
+
return {
|
|
211
|
+
**summary,
|
|
212
|
+
"id": row["id"],
|
|
213
|
+
"candidate_type": candidate.candidate_type,
|
|
214
|
+
"confidence": candidate.confidence,
|
|
215
|
+
"source_signature_ids": candidate.source_signature_ids,
|
|
216
|
+
"source_event_ids": candidate.source_event_ids,
|
|
217
|
+
"score_breakdown": candidate.score_breakdown,
|
|
218
|
+
"raw_envelope_ids": raw_envelope_ids,
|
|
219
|
+
"raw_envelopes": raw_envelopes,
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _signature_inspection_from_row(row, db_path: str | None) -> Dict[str, Any]:
|
|
224
|
+
metadata = json.loads(row["metadata_json"] or "{}")
|
|
225
|
+
if not isinstance(metadata, dict):
|
|
226
|
+
metadata = {}
|
|
227
|
+
raw_envelope_ids = extract_source_raw_envelope_ids(row["metadata_json"])
|
|
228
|
+
occurrence_count = int(row["occurrence_count"]) if row["occurrence_count"] is not None else 0
|
|
229
|
+
raw_envelope_count = len(raw_envelope_ids)
|
|
230
|
+
|
|
231
|
+
return {
|
|
232
|
+
"signature_key": row["signature_key"],
|
|
233
|
+
"event_family": row["event_family"],
|
|
234
|
+
"normalized_pattern": row["normalized_pattern"],
|
|
235
|
+
"service": row["service"],
|
|
236
|
+
"occurrence_count": occurrence_count,
|
|
237
|
+
"raw_envelope_ids": raw_envelope_ids,
|
|
238
|
+
"raw_envelope_count": raw_envelope_count,
|
|
239
|
+
"recurrence": {
|
|
240
|
+
"occurrence_count": occurrence_count,
|
|
241
|
+
"raw_envelope_count": raw_envelope_count,
|
|
242
|
+
"signature_id": row["id"],
|
|
243
|
+
},
|
|
244
|
+
"metadata": metadata,
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _canonical_event_from_row(row) -> Dict[str, Any]:
|
|
249
|
+
message_raw = row["message_raw"] or ""
|
|
250
|
+
return {
|
|
251
|
+
"raw_envelope_id": row["id"],
|
|
252
|
+
"tenant_id": row["tenant_id"],
|
|
253
|
+
"source": row["source_type"],
|
|
254
|
+
"source_type": row["source_type"],
|
|
255
|
+
"message_raw": message_raw,
|
|
256
|
+
"timestamp": row["timestamp"],
|
|
257
|
+
"host": row["host"] or "",
|
|
258
|
+
"service": row["service"] or "",
|
|
259
|
+
"severity": row["severity"] or "info",
|
|
260
|
+
"message_normalized": normalize_message(message_raw),
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
|
|
99
264
|
def _raw_envelope_from_row(row) -> Dict[str, Any]:
|
|
100
265
|
return {
|
|
101
266
|
"id": row["id"],
|
|
@@ -119,31 +284,76 @@ def _raw_envelope_from_row(row) -> Dict[str, Any]:
|
|
|
119
284
|
}
|
|
120
285
|
|
|
121
286
|
|
|
287
|
+
def _replay_attempt_from_item(item: ReplayAttempt) -> Dict[str, Any]:
|
|
288
|
+
return {
|
|
289
|
+
"raw_envelope_id": item.raw_envelope_id,
|
|
290
|
+
"reason": item.reason,
|
|
291
|
+
"status": item.status,
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def _source_capability_matrix() -> Dict[str, Any]:
|
|
296
|
+
source_types = list_source_driver_types()
|
|
297
|
+
modes_by_source_type: List[Dict[str, Any]] = []
|
|
298
|
+
for source_type in source_types:
|
|
299
|
+
if source_type == "logicmonitor":
|
|
300
|
+
modes = [
|
|
301
|
+
{"mode": "logicmonitor_webhook", "endpoint": "/ingest/logicmonitor"},
|
|
302
|
+
]
|
|
303
|
+
else:
|
|
304
|
+
modes = [
|
|
305
|
+
{"mode": "single_event_api", "endpoint": "/ingest/event"},
|
|
306
|
+
{"mode": "batch_api", "endpoint": "/ingest/batch"},
|
|
307
|
+
]
|
|
308
|
+
if source_type == "syslog":
|
|
309
|
+
modes.append({"mode": "udp_listener", "endpoint": "brainstem.listener"})
|
|
310
|
+
|
|
311
|
+
modes_by_source_type.append({"source_type": source_type, "modes": modes})
|
|
312
|
+
|
|
313
|
+
return {
|
|
314
|
+
"source_types": source_types,
|
|
315
|
+
"ingest_modes_by_source_type": modes_by_source_type,
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def _runtime_summary() -> Dict[str, Any]:
|
|
320
|
+
runtime_config = get_runtime_config()
|
|
321
|
+
return {
|
|
322
|
+
"version": __version__,
|
|
323
|
+
"runtime": {
|
|
324
|
+
"python_version": ".".join(str(item) for item in sys.version_info[:3]),
|
|
325
|
+
"api_token_env": runtime_config.api_token_env_var,
|
|
326
|
+
"config": runtime_config.as_dict(),
|
|
327
|
+
},
|
|
328
|
+
"capability_flags": {
|
|
329
|
+
**CAPABILITIES,
|
|
330
|
+
"source_capabilities": _source_capability_matrix(),
|
|
331
|
+
},
|
|
332
|
+
"auth_state": {
|
|
333
|
+
"api_token_configured": is_api_token_auth_enabled(),
|
|
334
|
+
"supports_x_api_token_header": True,
|
|
335
|
+
"supports_bearer_header": True,
|
|
336
|
+
},
|
|
337
|
+
"defaults": dict(RUNTIME_DEFAULTS),
|
|
338
|
+
"limits": dict(RUNTIME_LIMITS),
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def _status_payload() -> Dict[str, Any]:
|
|
343
|
+
runtime_summary = _runtime_summary()
|
|
344
|
+
return {
|
|
345
|
+
"ok": True,
|
|
346
|
+
"status": "ok",
|
|
347
|
+
"runtime": runtime_summary,
|
|
348
|
+
"api_token_enabled": runtime_summary["auth_state"]["api_token_configured"],
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
|
|
122
352
|
def _run_ingest_batch(raw_events: List[RawInputEnvelope], *, threshold: int, db_path: Optional[str]) -> Dict[str, Any]:
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
events = []
|
|
129
|
-
parse_failed = 0
|
|
130
|
-
for idx, raw_event in enumerate(raw_events):
|
|
131
|
-
raw_envelope_id = raw_envelope_ids[idx] if idx < len(raw_envelope_ids) else None
|
|
132
|
-
try:
|
|
133
|
-
canonical_event = canonicalize_raw_input_envelope(raw_event)
|
|
134
|
-
except Exception as exc:
|
|
135
|
-
parse_failed += 1
|
|
136
|
-
if raw_envelope_id is not None:
|
|
137
|
-
set_raw_envelope_status(
|
|
138
|
-
raw_envelope_id,
|
|
139
|
-
"parse_failed",
|
|
140
|
-
db_path=db_path,
|
|
141
|
-
failure_reason=str(exc),
|
|
142
|
-
)
|
|
143
|
-
continue
|
|
144
|
-
events.append(canonical_event)
|
|
145
|
-
if raw_envelope_id is not None:
|
|
146
|
-
set_raw_envelope_status(raw_envelope_id, "canonicalized", db_path=db_path)
|
|
353
|
+
result = run_ingest_pipeline(raw_events, threshold=threshold, db_path=db_path)
|
|
354
|
+
events = result.events
|
|
355
|
+
parse_failed = result.parse_failed
|
|
356
|
+
item_results = [asdict(item) for item in result.item_results]
|
|
147
357
|
|
|
148
358
|
if not events:
|
|
149
359
|
return {
|
|
@@ -151,16 +361,14 @@ def _run_ingest_batch(raw_events: List[RawInputEnvelope], *, threshold: int, db_
|
|
|
151
361
|
"event_count": 0,
|
|
152
362
|
"signature_count": 0,
|
|
153
363
|
"candidate_count": 0,
|
|
364
|
+
"item_count": len(raw_events),
|
|
365
|
+
"item_results": item_results,
|
|
154
366
|
"parse_failed": parse_failed,
|
|
155
367
|
"interesting_items": [],
|
|
156
368
|
}
|
|
157
369
|
|
|
158
|
-
signatures =
|
|
159
|
-
candidates =
|
|
160
|
-
if db_path:
|
|
161
|
-
store_events(events, db_path)
|
|
162
|
-
store_signatures(signatures, db_path)
|
|
163
|
-
store_candidates(candidates, db_path)
|
|
370
|
+
signatures = result.signatures
|
|
371
|
+
candidates = result.candidates
|
|
164
372
|
|
|
165
373
|
return {
|
|
166
374
|
"ok": True,
|
|
@@ -168,27 +376,101 @@ def _run_ingest_batch(raw_events: List[RawInputEnvelope], *, threshold: int, db_
|
|
|
168
376
|
"event_count": len(events),
|
|
169
377
|
"signature_count": len({sig.signature_key for sig in signatures}),
|
|
170
378
|
"candidate_count": len(candidates),
|
|
379
|
+
"item_count": len(raw_events),
|
|
380
|
+
"item_results": item_results,
|
|
381
|
+
"parse_failed": parse_failed,
|
|
382
|
+
"interesting_items": interesting_items(candidates, limit=max(1, 5)),
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
@app.post("/ingest/logicmonitor", dependencies=[Depends(_require_api_token)])
|
|
387
|
+
def ingest_logicmonitor(payload: IngestLogicMonitorRequest) -> Dict[str, Any]:
|
|
388
|
+
result = run_ingest_logicmonitor_events(
|
|
389
|
+
payload.events,
|
|
390
|
+
tenant_id=payload.tenant_id,
|
|
391
|
+
source_path=payload.source_path,
|
|
392
|
+
threshold=payload.threshold,
|
|
393
|
+
db_path=payload.db_path,
|
|
394
|
+
)
|
|
395
|
+
events = result.events
|
|
396
|
+
parse_failed = result.parse_failed
|
|
397
|
+
item_results = [asdict(item) for item in result.item_results]
|
|
398
|
+
|
|
399
|
+
if not events:
|
|
400
|
+
return {
|
|
401
|
+
"ok": True,
|
|
402
|
+
"event_count": 0,
|
|
403
|
+
"signature_count": 0,
|
|
404
|
+
"candidate_count": 0,
|
|
405
|
+
"item_count": len(result.raw_envelopes),
|
|
406
|
+
"item_results": item_results,
|
|
407
|
+
"parse_failed": parse_failed,
|
|
408
|
+
"interesting_items": [],
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
signatures = result.signatures
|
|
412
|
+
candidates = result.candidates
|
|
413
|
+
return {
|
|
414
|
+
"ok": True,
|
|
415
|
+
"tenant_id": events[0].tenant_id if events else payload.tenant_id,
|
|
416
|
+
"event_count": len(events),
|
|
417
|
+
"signature_count": len({sig.signature_key for sig in signatures}),
|
|
418
|
+
"candidate_count": len(candidates),
|
|
419
|
+
"item_count": len(result.raw_envelopes),
|
|
420
|
+
"item_results": item_results,
|
|
171
421
|
"parse_failed": parse_failed,
|
|
172
422
|
"interesting_items": interesting_items(candidates, limit=max(1, 5)),
|
|
173
423
|
}
|
|
174
424
|
|
|
175
425
|
|
|
176
|
-
@app.post("/
|
|
177
|
-
def
|
|
426
|
+
@app.post("/replay/raw", dependencies=[Depends(_require_api_token)])
|
|
427
|
+
def replay_raw(payload: ReplayRawRequest) -> Dict[str, Any]:
|
|
428
|
+
if not payload.raw_envelope_ids:
|
|
429
|
+
raise HTTPException(status_code=422, detail="raw_envelope_ids is required")
|
|
430
|
+
if payload.allowed_statuses:
|
|
431
|
+
invalid = [status for status in payload.allowed_statuses if status not in RAW_ENVELOPE_STATUSES]
|
|
432
|
+
if invalid:
|
|
433
|
+
raise HTTPException(
|
|
434
|
+
status_code=422,
|
|
435
|
+
detail=f"invalid status in allowed_statuses: {invalid}; expected one of: {', '.join(RAW_ENVELOPE_STATUSES)}",
|
|
436
|
+
)
|
|
437
|
+
|
|
438
|
+
result = replay_raw_envelopes_by_ids(
|
|
439
|
+
payload.raw_envelope_ids,
|
|
440
|
+
db_path=payload.db_path,
|
|
441
|
+
threshold=payload.threshold,
|
|
442
|
+
force=payload.force,
|
|
443
|
+
allowed_statuses=payload.allowed_statuses,
|
|
444
|
+
)
|
|
445
|
+
return {
|
|
446
|
+
"ok": True,
|
|
447
|
+
"db_path": payload.db_path,
|
|
448
|
+
"requested_raw_envelope_ids": result.requested_raw_envelope_ids,
|
|
449
|
+
"attempted_raw_envelope_ids": result.attempted_raw_envelope_ids,
|
|
450
|
+
"skipped": [_replay_attempt_from_item(item) for item in result.skipped],
|
|
451
|
+
"event_count": len(result.events),
|
|
452
|
+
"signature_count": len(result.signatures),
|
|
453
|
+
"candidate_count": len(result.candidates),
|
|
454
|
+
"parse_failed": result.parse_failed,
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
|
|
458
|
+
@app.post("/ingest/event", dependencies=[Depends(_require_api_token)])
|
|
459
|
+
def ingest_event(payload: IngestEventRequest, threshold: int = DEFAULT_INGEST_THRESHOLD, db_path: Optional[str] = None) -> Dict[str, Any]:
|
|
178
460
|
if threshold < 1:
|
|
179
461
|
raise HTTPException(status_code=422, detail="threshold must be >= 1")
|
|
180
462
|
return _run_ingest_batch([_raw_envelope_from_request(payload)], threshold=threshold, db_path=db_path)
|
|
181
463
|
|
|
182
464
|
|
|
183
|
-
@app.post("/ingest/batch")
|
|
465
|
+
@app.post("/ingest/batch", dependencies=[Depends(_require_api_token)])
|
|
184
466
|
def ingest_batch(payload: IngestBatchRequest) -> Dict[str, Any]:
|
|
185
467
|
raw_events = [_raw_envelope_from_request(event) for event in payload.events]
|
|
186
468
|
return _run_ingest_batch(raw_events, threshold=payload.threshold, db_path=payload.db_path)
|
|
187
469
|
|
|
188
470
|
|
|
189
|
-
@app.get("/interesting")
|
|
471
|
+
@app.get("/interesting", dependencies=[Depends(_require_api_token)])
|
|
190
472
|
def get_interesting(
|
|
191
|
-
limit: int = Query(default=
|
|
473
|
+
limit: int = Query(default=DEFAULT_INTERESTING_LIMIT, ge=1),
|
|
192
474
|
db_path: Optional[str] = None,
|
|
193
475
|
) -> Dict[str, Any]:
|
|
194
476
|
if not db_path:
|
|
@@ -198,14 +480,58 @@ def get_interesting(
|
|
|
198
480
|
return {"ok": True, "items": interesting_items(candidates, limit=limit)}
|
|
199
481
|
|
|
200
482
|
|
|
201
|
-
@app.get("/
|
|
483
|
+
@app.get("/candidates", dependencies=[Depends(_require_api_token)])
|
|
484
|
+
def get_candidates(
|
|
485
|
+
limit: int = Query(default=DEFAULT_INTERESTING_LIMIT, ge=1),
|
|
486
|
+
db_path: Optional[str] = None,
|
|
487
|
+
candidate_type: Optional[str] = None,
|
|
488
|
+
decision_band: Optional[str] = None,
|
|
489
|
+
min_score_total: Optional[float] = Query(default=None, ge=0),
|
|
490
|
+
) -> Dict[str, Any]:
|
|
491
|
+
if not db_path:
|
|
492
|
+
return {"ok": True, "count": 0, "items": []}
|
|
493
|
+
|
|
494
|
+
rows = list_candidates(
|
|
495
|
+
db_path=db_path,
|
|
496
|
+
limit=limit,
|
|
497
|
+
candidate_type=candidate_type,
|
|
498
|
+
decision_band=decision_band,
|
|
499
|
+
min_score_total=min_score_total,
|
|
500
|
+
)
|
|
501
|
+
items = [_candidate_inspection_from_row(row, db_path=db_path) for row in rows]
|
|
502
|
+
return {"ok": True, "count": len(items), "items": items}
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
@app.get("/signatures", dependencies=[Depends(_require_api_token)])
|
|
506
|
+
def get_signatures(
|
|
507
|
+
limit: int = Query(default=DEFAULT_INTERESTING_LIMIT, ge=1),
|
|
508
|
+
db_path: Optional[str] = None,
|
|
509
|
+
event_family: Optional[str] = None,
|
|
510
|
+
service: Optional[str] = None,
|
|
511
|
+
min_occurrence_count: Optional[int] = Query(default=None, ge=1),
|
|
512
|
+
) -> Dict[str, Any]:
|
|
513
|
+
if not db_path:
|
|
514
|
+
return {"ok": True, "count": 0, "items": []}
|
|
515
|
+
|
|
516
|
+
rows = list_signatures(
|
|
517
|
+
db_path=db_path,
|
|
518
|
+
limit=limit,
|
|
519
|
+
event_family=event_family,
|
|
520
|
+
service=service,
|
|
521
|
+
min_occurrence_count=min_occurrence_count,
|
|
522
|
+
)
|
|
523
|
+
items = [_signature_inspection_from_row(row, db_path=db_path) for row in rows]
|
|
524
|
+
return {"ok": True, "count": len(items), "items": items}
|
|
525
|
+
|
|
526
|
+
|
|
527
|
+
@app.get("/stats", dependencies=[Depends(_require_api_token)])
|
|
202
528
|
def get_stats(db_path: Optional[str] = None) -> Dict[str, Any]:
|
|
203
529
|
return {"ok": True, **get_ingest_stats(db_path)}
|
|
204
530
|
|
|
205
531
|
|
|
206
|
-
@app.get("/failures")
|
|
532
|
+
@app.get("/failures", dependencies=[Depends(_require_api_token)])
|
|
207
533
|
def get_failures(
|
|
208
|
-
limit: int = Query(default=
|
|
534
|
+
limit: int = Query(default=DEFAULT_FAILURE_LIMIT, ge=1),
|
|
209
535
|
status: Optional[str] = None,
|
|
210
536
|
db_path: Optional[str] = None,
|
|
211
537
|
) -> Dict[str, Any]:
|
|
@@ -220,9 +546,38 @@ def get_failures(
|
|
|
220
546
|
return {"ok": True, "items": items, "count": len(items), "status": status}
|
|
221
547
|
|
|
222
548
|
|
|
223
|
-
@app.get("/
|
|
549
|
+
@app.get("/raw_envelopes", dependencies=[Depends(_require_api_token)])
|
|
550
|
+
def get_raw_envelopes(
|
|
551
|
+
limit: int = Query(default=DEFAULT_INGEST_RECENT_LIMIT, ge=1),
|
|
552
|
+
status: Optional[str] = None,
|
|
553
|
+
tenant_id: Optional[str] = None,
|
|
554
|
+
source_type: Optional[str] = None,
|
|
555
|
+
source_id: Optional[str] = None,
|
|
556
|
+
source_path: Optional[str] = None,
|
|
557
|
+
db_path: Optional[str] = None,
|
|
558
|
+
) -> Dict[str, Any]:
|
|
559
|
+
if status is not None and status not in RAW_ENVELOPE_STATUSES:
|
|
560
|
+
raise HTTPException(
|
|
561
|
+
status_code=422,
|
|
562
|
+
detail=f"invalid status '{status}'; expected one of: {', '.join(RAW_ENVELOPE_STATUSES)}",
|
|
563
|
+
)
|
|
564
|
+
rows = list_recent_raw_envelopes(
|
|
565
|
+
db_path=db_path,
|
|
566
|
+
status=status,
|
|
567
|
+
limit=limit,
|
|
568
|
+
tenant_id=tenant_id,
|
|
569
|
+
source_type=source_type,
|
|
570
|
+
source_id=source_id,
|
|
571
|
+
source_path=source_path,
|
|
572
|
+
failures_only=False,
|
|
573
|
+
)
|
|
574
|
+
items = [_raw_envelope_from_row(row) for row in rows]
|
|
575
|
+
return {"ok": True, "items": items, "count": len(items)}
|
|
576
|
+
|
|
577
|
+
|
|
578
|
+
@app.get("/ingest/recent", dependencies=[Depends(_require_api_token)])
|
|
224
579
|
def get_ingest_recent(
|
|
225
|
-
limit: int = Query(default=
|
|
580
|
+
limit: int = Query(default=DEFAULT_INGEST_RECENT_LIMIT, ge=1),
|
|
226
581
|
status: Optional[str] = None,
|
|
227
582
|
db_path: Optional[str] = None,
|
|
228
583
|
) -> Dict[str, Any]:
|
|
@@ -236,15 +591,61 @@ def get_ingest_recent(
|
|
|
236
591
|
return {"ok": True, "items": items, "count": len(items), "status": status}
|
|
237
592
|
|
|
238
593
|
|
|
239
|
-
@app.get("/
|
|
594
|
+
@app.get("/canonical_events", dependencies=[Depends(_require_api_token)])
|
|
595
|
+
def get_canonical_events(
|
|
596
|
+
limit: int = Query(default=DEFAULT_INGEST_RECENT_LIMIT, ge=1),
|
|
597
|
+
tenant_id: Optional[str] = None,
|
|
598
|
+
source: Optional[str] = None,
|
|
599
|
+
host: Optional[str] = None,
|
|
600
|
+
service: Optional[str] = None,
|
|
601
|
+
severity: Optional[str] = None,
|
|
602
|
+
db_path: Optional[str] = None,
|
|
603
|
+
) -> Dict[str, Any]:
|
|
604
|
+
if not db_path:
|
|
605
|
+
return {"ok": True, "items": [], "count": 0}
|
|
606
|
+
|
|
607
|
+
rows = list_canonical_events(
|
|
608
|
+
db_path=db_path,
|
|
609
|
+
limit=limit,
|
|
610
|
+
tenant_id=tenant_id,
|
|
611
|
+
source_type=source,
|
|
612
|
+
host=host,
|
|
613
|
+
service=service,
|
|
614
|
+
severity=severity,
|
|
615
|
+
)
|
|
616
|
+
items = [_canonical_event_from_row(row) for row in rows]
|
|
617
|
+
return {"ok": True, "items": items, "count": len(items)}
|
|
618
|
+
|
|
619
|
+
|
|
620
|
+
@app.get("/sources", dependencies=[Depends(_require_api_token)])
|
|
240
621
|
def get_sources(
|
|
241
|
-
limit: int = Query(default=
|
|
622
|
+
limit: int = Query(default=DEFAULT_SOURCES_LIMIT, ge=1),
|
|
242
623
|
db_path: Optional[str] = None,
|
|
243
624
|
) -> Dict[str, Any]:
|
|
244
625
|
return {"ok": True, "items": get_source_dimension_summaries(db_path=db_path, limit=limit)}
|
|
245
626
|
|
|
246
627
|
|
|
247
|
-
@app.get("/
|
|
628
|
+
@app.get("/sources/status", dependencies=[Depends(_require_api_token)])
|
|
629
|
+
def get_sources_status(
|
|
630
|
+
limit: int = Query(default=DEFAULT_SOURCES_STATUS_LIMIT, ge=1),
|
|
631
|
+
tenant_id: Optional[str] = None,
|
|
632
|
+
source_type: Optional[str] = None,
|
|
633
|
+
source_id: Optional[str] = None,
|
|
634
|
+
source_path: Optional[str] = None,
|
|
635
|
+
db_path: Optional[str] = None,
|
|
636
|
+
) -> Dict[str, Any]:
|
|
637
|
+
items = get_source_status_summaries(
|
|
638
|
+
db_path=db_path,
|
|
639
|
+
limit=limit,
|
|
640
|
+
tenant_id=tenant_id,
|
|
641
|
+
source_type=source_type,
|
|
642
|
+
source_id=source_id,
|
|
643
|
+
source_path=source_path,
|
|
644
|
+
)
|
|
645
|
+
return {"ok": True, "items": items, "count": len(items)}
|
|
646
|
+
|
|
647
|
+
|
|
648
|
+
@app.get("/failures/{raw_envelope_id}", dependencies=[Depends(_require_api_token)])
|
|
248
649
|
def get_failure(raw_envelope_id: int, db_path: Optional[str] = None) -> Dict[str, Any]:
|
|
249
650
|
row = get_raw_envelope_by_id(raw_envelope_id, db_path=db_path)
|
|
250
651
|
if row is None:
|
|
@@ -253,5 +654,15 @@ def get_failure(raw_envelope_id: int, db_path: Optional[str] = None) -> Dict[str
|
|
|
253
654
|
|
|
254
655
|
|
|
255
656
|
@app.get("/healthz")
|
|
256
|
-
def healthz() -> Dict[str,
|
|
257
|
-
return
|
|
657
|
+
def healthz() -> Dict[str, Any]:
|
|
658
|
+
return _status_payload()
|
|
659
|
+
|
|
660
|
+
|
|
661
|
+
@app.get("/runtime")
|
|
662
|
+
def runtime() -> Dict[str, Any]:
|
|
663
|
+
return {"ok": True, "runtime": _runtime_summary()}
|
|
664
|
+
|
|
665
|
+
|
|
666
|
+
@app.get("/status")
|
|
667
|
+
def status() -> Dict[str, Any]:
|
|
668
|
+
return _status_payload()
|