@simbimbo/brainstem 0.0.4 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/README.md +4 -3
- package/brainstem/__init__.py +1 -1
- package/brainstem/api.py +80 -3
- package/brainstem/config.py +66 -0
- package/brainstem/connectors/logicmonitor.py +57 -0
- package/brainstem/demo.py +16 -2
- package/brainstem/fingerprint.py +54 -0
- package/brainstem/ingest.py +34 -5
- package/brainstem/listener.py +8 -2
- package/brainstem/recurrence.py +25 -8
- package/brainstem/scoring.py +6 -4
- package/brainstem/source_drivers.py +29 -0
- package/brainstem/storage.py +84 -0
- package/docs/README.md +12 -3
- package/docs/adapters.md +401 -97
- package/docs/api.md +36 -1
- package/package.json +1 -1
- package/pyproject.toml +1 -1
- package/tests/test_adapters.py +1 -0
- package/tests/test_api.py +86 -0
- package/tests/test_config.py +15 -0
- package/tests/test_fingerprint.py +51 -1
- package/tests/test_logicmonitor.py +54 -1
- package/tests/test_recurrence.py +14 -0
- package/tests/test_storage.py +77 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,18 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.0.5 — 2026-03-22
|
|
4
|
+
|
|
5
|
+
Source capability reporting and docs/runtime alignment pass.
|
|
6
|
+
|
|
7
|
+
### Highlights
|
|
8
|
+
- exposes compact source-support matrix in `/runtime` (`capability_flags.source_capabilities`)
|
|
9
|
+
- documents LogicMonitor-shaped ingest path as implemented runtime API surface
|
|
10
|
+
- aligns README/docs/examples with runtime API surface and active source drivers
|
|
11
|
+
- keeps behavior intact for existing ingestion/canonicalization paths
|
|
12
|
+
|
|
13
|
+
### Validation
|
|
14
|
+
- local test suite passed (with updated runtime capability coverage)
|
|
15
|
+
|
|
3
16
|
## 0.0.4 — 2026-03-22
|
|
4
17
|
|
|
5
18
|
Release-prep alignment for the current intake foundation state.
|
package/README.md
CHANGED
|
@@ -53,17 +53,18 @@ For a given tenant or environment, brAInstem should answer:
|
|
|
53
53
|
Current implementation is intentionally bounded:
|
|
54
54
|
- FastAPI runtime at `brainstem.api:app`
|
|
55
55
|
- Implemented API endpoints include:
|
|
56
|
-
- `POST /ingest/event`, `POST /ingest/batch`, `POST /replay/raw`
|
|
56
|
+
- `POST /ingest/event`, `POST /ingest/batch`, `POST /ingest/logicmonitor`, `POST /replay/raw`
|
|
57
57
|
- `GET /interesting`, `GET /candidates`, `GET /signatures`, `GET /canonical_events`
|
|
58
58
|
- `GET /stats`, `GET /failures`, `GET /failures/{id}`
|
|
59
59
|
- `GET /raw_envelopes`, `GET /ingest/recent`, `GET /sources`, `GET /sources/status`
|
|
60
60
|
- `GET /runtime`, `GET /status`, `GET /healthz`
|
|
61
61
|
- UDP syslog listener at `brainstem.listener`
|
|
62
|
-
- source-driver intake for `syslog` and `
|
|
62
|
+
- source-driver intake for `syslog`, `file`, and narrow `logicmonitor` connector payload shape
|
|
63
63
|
- SQLite-backed persistence with default DB at `.brainstem-state/brainstem.sqlite3`
|
|
64
64
|
- optional API token in `BRAINSTEM_API_TOKEN`
|
|
65
65
|
|
|
66
|
-
Runtime config can also be inspected at `/runtime
|
|
66
|
+
Runtime config can also be inspected at `/runtime`. The runtime summary now includes explicit `source_capabilities`
|
|
67
|
+
(`source_types` + per-source ingest modes) and auth/capabilities flags.
|
|
67
68
|
|
|
68
69
|
## Quick run path (API + listener + file/syslog intake)
|
|
69
70
|
|
package/brainstem/__init__.py
CHANGED
package/brainstem/api.py
CHANGED
|
@@ -11,12 +11,13 @@ import json
|
|
|
11
11
|
from fastapi import Depends, FastAPI, Header, HTTPException, Query
|
|
12
12
|
from pydantic import BaseModel, Field
|
|
13
13
|
|
|
14
|
-
from .ingest import ReplayAttempt, replay_raw_envelopes_by_ids, run_ingest_pipeline
|
|
14
|
+
from .ingest import ReplayAttempt, run_ingest_logicmonitor_events, replay_raw_envelopes_by_ids, run_ingest_pipeline
|
|
15
15
|
from .interesting import interesting_items
|
|
16
16
|
from .config import get_runtime_config
|
|
17
17
|
from .fingerprint import normalize_message
|
|
18
18
|
from . import __version__
|
|
19
19
|
from .models import Candidate, RawInputEnvelope
|
|
20
|
+
from .source_drivers import list_source_driver_types
|
|
20
21
|
from .storage import (
|
|
21
22
|
RAW_ENVELOPE_STATUSES,
|
|
22
23
|
extract_source_raw_envelope_ids,
|
|
@@ -53,6 +54,7 @@ CAPABILITIES = {
|
|
|
53
54
|
"ingest_endpoints": {
|
|
54
55
|
"single_event": True,
|
|
55
56
|
"batch_events": True,
|
|
57
|
+
"logicmonitor_events": True,
|
|
56
58
|
"replay_raw": True,
|
|
57
59
|
},
|
|
58
60
|
"inspection_endpoints": {
|
|
@@ -137,6 +139,14 @@ class ReplayRawRequest(BaseModel):
|
|
|
137
139
|
allowed_statuses: List[str] = Field(default_factory=lambda: list(DEFAULT_REPLAY_ALLOWED_STATUSES))
|
|
138
140
|
|
|
139
141
|
|
|
142
|
+
class IngestLogicMonitorRequest(BaseModel):
|
|
143
|
+
tenant_id: str
|
|
144
|
+
events: List[Dict[str, Any]] = Field(default_factory=list, min_length=1)
|
|
145
|
+
source_path: str = "/logicmonitor/ingest"
|
|
146
|
+
threshold: int = Field(default=DEFAULT_BATCH_THRESHOLD, ge=1)
|
|
147
|
+
db_path: Optional[str] = None
|
|
148
|
+
|
|
149
|
+
|
|
140
150
|
def _raw_envelope_from_request(payload: RawEnvelopeRequest) -> RawInputEnvelope:
|
|
141
151
|
return RawInputEnvelope(
|
|
142
152
|
tenant_id=payload.tenant_id,
|
|
@@ -183,7 +193,7 @@ def _candidate_inspection_from_row(row, db_path: str | None) -> Dict[str, Any]:
|
|
|
183
193
|
"attention_score": candidate.score_total,
|
|
184
194
|
"score_total": candidate.score_total,
|
|
185
195
|
"attention_explanation": [],
|
|
186
|
-
|
|
196
|
+
"score_breakdown": candidate.score_breakdown,
|
|
187
197
|
}
|
|
188
198
|
|
|
189
199
|
raw_envelope_ids = extract_source_raw_envelope_ids(row["metadata_json"])
|
|
@@ -282,6 +292,30 @@ def _replay_attempt_from_item(item: ReplayAttempt) -> Dict[str, Any]:
|
|
|
282
292
|
}
|
|
283
293
|
|
|
284
294
|
|
|
295
|
+
def _source_capability_matrix() -> Dict[str, Any]:
|
|
296
|
+
source_types = list_source_driver_types()
|
|
297
|
+
modes_by_source_type: List[Dict[str, Any]] = []
|
|
298
|
+
for source_type in source_types:
|
|
299
|
+
if source_type == "logicmonitor":
|
|
300
|
+
modes = [
|
|
301
|
+
{"mode": "logicmonitor_webhook", "endpoint": "/ingest/logicmonitor"},
|
|
302
|
+
]
|
|
303
|
+
else:
|
|
304
|
+
modes = [
|
|
305
|
+
{"mode": "single_event_api", "endpoint": "/ingest/event"},
|
|
306
|
+
{"mode": "batch_api", "endpoint": "/ingest/batch"},
|
|
307
|
+
]
|
|
308
|
+
if source_type == "syslog":
|
|
309
|
+
modes.append({"mode": "udp_listener", "endpoint": "brainstem.listener"})
|
|
310
|
+
|
|
311
|
+
modes_by_source_type.append({"source_type": source_type, "modes": modes})
|
|
312
|
+
|
|
313
|
+
return {
|
|
314
|
+
"source_types": source_types,
|
|
315
|
+
"ingest_modes_by_source_type": modes_by_source_type,
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
|
|
285
319
|
def _runtime_summary() -> Dict[str, Any]:
|
|
286
320
|
runtime_config = get_runtime_config()
|
|
287
321
|
return {
|
|
@@ -291,7 +325,10 @@ def _runtime_summary() -> Dict[str, Any]:
|
|
|
291
325
|
"api_token_env": runtime_config.api_token_env_var,
|
|
292
326
|
"config": runtime_config.as_dict(),
|
|
293
327
|
},
|
|
294
|
-
"capability_flags":
|
|
328
|
+
"capability_flags": {
|
|
329
|
+
**CAPABILITIES,
|
|
330
|
+
"source_capabilities": _source_capability_matrix(),
|
|
331
|
+
},
|
|
295
332
|
"auth_state": {
|
|
296
333
|
"api_token_configured": is_api_token_auth_enabled(),
|
|
297
334
|
"supports_x_api_token_header": True,
|
|
@@ -346,6 +383,46 @@ def _run_ingest_batch(raw_events: List[RawInputEnvelope], *, threshold: int, db_
|
|
|
346
383
|
}
|
|
347
384
|
|
|
348
385
|
|
|
386
|
+
@app.post("/ingest/logicmonitor", dependencies=[Depends(_require_api_token)])
|
|
387
|
+
def ingest_logicmonitor(payload: IngestLogicMonitorRequest) -> Dict[str, Any]:
|
|
388
|
+
result = run_ingest_logicmonitor_events(
|
|
389
|
+
payload.events,
|
|
390
|
+
tenant_id=payload.tenant_id,
|
|
391
|
+
source_path=payload.source_path,
|
|
392
|
+
threshold=payload.threshold,
|
|
393
|
+
db_path=payload.db_path,
|
|
394
|
+
)
|
|
395
|
+
events = result.events
|
|
396
|
+
parse_failed = result.parse_failed
|
|
397
|
+
item_results = [asdict(item) for item in result.item_results]
|
|
398
|
+
|
|
399
|
+
if not events:
|
|
400
|
+
return {
|
|
401
|
+
"ok": True,
|
|
402
|
+
"event_count": 0,
|
|
403
|
+
"signature_count": 0,
|
|
404
|
+
"candidate_count": 0,
|
|
405
|
+
"item_count": len(result.raw_envelopes),
|
|
406
|
+
"item_results": item_results,
|
|
407
|
+
"parse_failed": parse_failed,
|
|
408
|
+
"interesting_items": [],
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
signatures = result.signatures
|
|
412
|
+
candidates = result.candidates
|
|
413
|
+
return {
|
|
414
|
+
"ok": True,
|
|
415
|
+
"tenant_id": events[0].tenant_id if events else payload.tenant_id,
|
|
416
|
+
"event_count": len(events),
|
|
417
|
+
"signature_count": len({sig.signature_key for sig in signatures}),
|
|
418
|
+
"candidate_count": len(candidates),
|
|
419
|
+
"item_count": len(result.raw_envelopes),
|
|
420
|
+
"item_results": item_results,
|
|
421
|
+
"parse_failed": parse_failed,
|
|
422
|
+
"interesting_items": interesting_items(candidates, limit=max(1, 5)),
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
|
|
349
426
|
@app.post("/replay/raw", dependencies=[Depends(_require_api_token)])
|
|
350
427
|
def replay_raw(payload: ReplayRawRequest) -> Dict[str, Any]:
|
|
351
428
|
if not payload.raw_envelope_ids:
|
package/brainstem/config.py
CHANGED
|
@@ -13,6 +13,26 @@ def resolve_default_db_path() -> str:
|
|
|
13
13
|
return str(Path(".brainstem-state") / "brainstem.sqlite3")
|
|
14
14
|
|
|
15
15
|
|
|
16
|
+
def _read_env_int(env_name: str, default: int) -> int:
|
|
17
|
+
value = os.getenv(env_name, "").strip()
|
|
18
|
+
if not value:
|
|
19
|
+
return default
|
|
20
|
+
try:
|
|
21
|
+
return int(value)
|
|
22
|
+
except ValueError:
|
|
23
|
+
return default
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _read_env_float(env_name: str, default: float) -> float:
|
|
27
|
+
value = os.getenv(env_name, "").strip()
|
|
28
|
+
if not value:
|
|
29
|
+
return default
|
|
30
|
+
try:
|
|
31
|
+
return float(value)
|
|
32
|
+
except ValueError:
|
|
33
|
+
return default
|
|
34
|
+
|
|
35
|
+
|
|
16
36
|
@dataclass(frozen=True)
|
|
17
37
|
class ListenerConfig:
|
|
18
38
|
syslog_host: str = "127.0.0.1"
|
|
@@ -25,6 +45,7 @@ class ListenerConfig:
|
|
|
25
45
|
@dataclass(frozen=True)
|
|
26
46
|
class RuntimeDefaults:
|
|
27
47
|
ingest_threshold: int = 2
|
|
48
|
+
recurrence_threshold: int = 2
|
|
28
49
|
batch_threshold: int = 2
|
|
29
50
|
interesting_limit: int = 5
|
|
30
51
|
failure_limit: int = 20
|
|
@@ -41,6 +62,22 @@ class RuntimeLimits:
|
|
|
41
62
|
replay_allowed_statuses: tuple[str, ...] = ("received", "parse_failed")
|
|
42
63
|
|
|
43
64
|
|
|
65
|
+
@dataclass(frozen=True)
|
|
66
|
+
class CandidateAttentionProfile:
|
|
67
|
+
recurrence_count_normalizer: int = 10
|
|
68
|
+
recovery_signal_weight: float = 0.4
|
|
69
|
+
spread_signal_weight: float = 0.2
|
|
70
|
+
novelty_signal_weight: float = 0.3
|
|
71
|
+
impact_high_weight: float = 0.5
|
|
72
|
+
impact_default_weight: float = 0.2
|
|
73
|
+
precursor_weight: float = 0.3
|
|
74
|
+
memory_weight: float = 0.4
|
|
75
|
+
decision_band_promote: float = 0.85
|
|
76
|
+
decision_band_urgent_human_review: float = 0.65
|
|
77
|
+
decision_band_review: float = 0.45
|
|
78
|
+
decision_band_watch: float = 0.25
|
|
79
|
+
|
|
80
|
+
|
|
44
81
|
@dataclass(frozen=True)
|
|
45
82
|
class DBConfig:
|
|
46
83
|
default_path: str
|
|
@@ -51,6 +88,7 @@ class RuntimeConfig:
|
|
|
51
88
|
api_token_env_var: str = "BRAINSTEM_API_TOKEN"
|
|
52
89
|
listener: ListenerConfig = ListenerConfig()
|
|
53
90
|
defaults: RuntimeDefaults = RuntimeDefaults()
|
|
91
|
+
candidate_attention: CandidateAttentionProfile = CandidateAttentionProfile()
|
|
54
92
|
limits: RuntimeLimits = RuntimeLimits()
|
|
55
93
|
db: DBConfig = DBConfig(default_path=resolve_default_db_path())
|
|
56
94
|
|
|
@@ -59,12 +97,40 @@ class RuntimeConfig:
|
|
|
59
97
|
"api_token_env_var": self.api_token_env_var,
|
|
60
98
|
"listener": asdict(self.listener),
|
|
61
99
|
"defaults": asdict(self.defaults),
|
|
100
|
+
"candidate_attention": asdict(self.candidate_attention),
|
|
62
101
|
"limits": asdict(self.limits),
|
|
63
102
|
"db": asdict(self.db),
|
|
64
103
|
}
|
|
65
104
|
|
|
66
105
|
|
|
67
106
|
def get_runtime_config() -> RuntimeConfig:
|
|
107
|
+
defaults = RuntimeDefaults(
|
|
108
|
+
ingest_threshold=_read_env_int("BRAINSTEM_INGEST_THRESHOLD", 2),
|
|
109
|
+
recurrence_threshold=_read_env_int("BRAINSTEM_RECURRENCE_THRESHOLD", 2),
|
|
110
|
+
batch_threshold=_read_env_int("BRAINSTEM_BATCH_THRESHOLD", 2),
|
|
111
|
+
interesting_limit=_read_env_int("BRAINSTEM_INTERESTING_LIMIT", 5),
|
|
112
|
+
failure_limit=_read_env_int("BRAINSTEM_FAILURE_LIMIT", 20),
|
|
113
|
+
ingest_recent_limit=_read_env_int("BRAINSTEM_INGEST_RECENT_LIMIT", 20),
|
|
114
|
+
sources_limit=_read_env_int("BRAINSTEM_SOURCES_LIMIT", 10),
|
|
115
|
+
sources_status_limit=_read_env_int("BRAINSTEM_SOURCES_STATUS_LIMIT", 20),
|
|
116
|
+
replay_threshold=_read_env_int("BRAINSTEM_REPLAY_THRESHOLD", 2),
|
|
117
|
+
)
|
|
118
|
+
candidate_attention = CandidateAttentionProfile(
|
|
119
|
+
recurrence_count_normalizer=_read_env_int("BRAINSTEM_CANDIDATE_RECURRENCE_NORMALIZER", 10),
|
|
120
|
+
recovery_signal_weight=_read_env_float("BRAINSTEM_CANDIDATE_RECOVERY", 0.4),
|
|
121
|
+
spread_signal_weight=_read_env_float("BRAINSTEM_CANDIDATE_SPREAD", 0.2),
|
|
122
|
+
novelty_signal_weight=_read_env_float("BRAINSTEM_CANDIDATE_NOVELTY", 0.3),
|
|
123
|
+
impact_high_weight=_read_env_float("BRAINSTEM_CANDIDATE_IMPACT_HIGH", 0.5),
|
|
124
|
+
impact_default_weight=_read_env_float("BRAINSTEM_CANDIDATE_IMPACT_DEFAULT", 0.2),
|
|
125
|
+
precursor_weight=_read_env_float("BRAINSTEM_CANDIDATE_PRECURSOR", 0.3),
|
|
126
|
+
memory_weight=_read_env_float("BRAINSTEM_CANDIDATE_MEMORY_WEIGHT", 0.4),
|
|
127
|
+
decision_band_promote=_read_env_float("BRAINSTEM_DECISION_BAND_PROMOTE", 0.85),
|
|
128
|
+
decision_band_urgent_human_review=_read_env_float("BRAINSTEM_DECISION_BAND_URGENT_HUMAN_REVIEW", 0.65),
|
|
129
|
+
decision_band_review=_read_env_float("BRAINSTEM_DECISION_BAND_REVIEW", 0.45),
|
|
130
|
+
decision_band_watch=_read_env_float("BRAINSTEM_DECISION_BAND_WATCH", 0.25),
|
|
131
|
+
)
|
|
68
132
|
return RuntimeConfig(
|
|
133
|
+
defaults=defaults,
|
|
134
|
+
candidate_attention=candidate_attention,
|
|
69
135
|
db=DBConfig(default_path=resolve_default_db_path()),
|
|
70
136
|
)
|
|
@@ -1,8 +1,19 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import json
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
|
|
6
|
+
from ..adapters import RawInputAdapter, register_raw_input_adapter
|
|
7
|
+
from ..models import RawInputEnvelope
|
|
3
8
|
from .types import ConnectorEvent
|
|
4
9
|
|
|
5
10
|
|
|
11
|
+
def _coerce_mapping(payload: object) -> dict:
|
|
12
|
+
if not isinstance(payload, dict):
|
|
13
|
+
raise ValueError("logicmonitor payload must be an object")
|
|
14
|
+
return payload
|
|
15
|
+
|
|
16
|
+
|
|
6
17
|
def map_logicmonitor_event(payload: dict, *, tenant_id: str) -> ConnectorEvent:
|
|
7
18
|
metadata = payload.get("metadata") or {}
|
|
8
19
|
host = payload.get("host") or payload.get("resource_name") or ""
|
|
@@ -24,3 +35,49 @@ def map_logicmonitor_event(payload: dict, *, tenant_id: str) -> ConnectorEvent:
|
|
|
24
35
|
"cleared_at": metadata.get("cleared_at"),
|
|
25
36
|
},
|
|
26
37
|
)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def map_logicmonitor_payload_to_raw_envelope(payload: object, *, tenant_id: str, source_path: str = "") -> RawInputEnvelope:
|
|
41
|
+
event_payload = _coerce_mapping(payload)
|
|
42
|
+
event = map_logicmonitor_event(event_payload, tenant_id=tenant_id)
|
|
43
|
+
return RawInputEnvelope(
|
|
44
|
+
tenant_id=event.tenant_id,
|
|
45
|
+
source_type=event.source_type,
|
|
46
|
+
timestamp=event.timestamp,
|
|
47
|
+
message_raw=event.message_raw,
|
|
48
|
+
host=event.host,
|
|
49
|
+
service=event.service,
|
|
50
|
+
severity=event.severity,
|
|
51
|
+
source_path=source_path,
|
|
52
|
+
metadata=dict(event.metadata),
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@dataclass(frozen=True)
|
|
57
|
+
class LogicMonitorRawInputAdapter:
|
|
58
|
+
"""Adapter for LogicMonitor-shaped event payload objects."""
|
|
59
|
+
|
|
60
|
+
source_type: str = "logicmonitor"
|
|
61
|
+
|
|
62
|
+
def parse_raw_input(self, payload, *, tenant_id: str, source_path: str = "") -> RawInputEnvelope:
|
|
63
|
+
if isinstance(payload, (bytes, bytearray)):
|
|
64
|
+
payload_text = payload.decode("utf-8", errors="replace")
|
|
65
|
+
payload_obj = json.loads(payload_text)
|
|
66
|
+
return map_logicmonitor_payload_to_raw_envelope(payload_obj, tenant_id=tenant_id, source_path=source_path)
|
|
67
|
+
|
|
68
|
+
if isinstance(payload, dict):
|
|
69
|
+
return map_logicmonitor_payload_to_raw_envelope(payload, tenant_id=tenant_id, source_path=source_path)
|
|
70
|
+
|
|
71
|
+
if isinstance(payload, str):
|
|
72
|
+
payload_text = payload.strip()
|
|
73
|
+
if payload_text.startswith("{") and payload_text.endswith("}"):
|
|
74
|
+
return map_logicmonitor_payload_to_raw_envelope(
|
|
75
|
+
json.loads(payload_text),
|
|
76
|
+
tenant_id=tenant_id,
|
|
77
|
+
source_path=source_path,
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
raise ValueError("logicmonitor payload must be a mapping or JSON object string")
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
register_raw_input_adapter(LogicMonitorRawInputAdapter())
|
package/brainstem/demo.py
CHANGED
|
@@ -10,9 +10,18 @@ from .instrumentation import emit, span
|
|
|
10
10
|
from .interesting import interesting_items
|
|
11
11
|
from .recurrence import build_recurrence_candidates, digest_items
|
|
12
12
|
from .storage import init_db, store_candidates, store_events, store_signatures
|
|
13
|
+
from .config import get_runtime_config
|
|
13
14
|
|
|
14
15
|
|
|
15
|
-
def run_syslog_demo(
|
|
16
|
+
def run_syslog_demo(
|
|
17
|
+
path: str,
|
|
18
|
+
tenant_id: str,
|
|
19
|
+
threshold: int | None = None,
|
|
20
|
+
db_path: str | None = None,
|
|
21
|
+
) -> Dict[str, Any]:
|
|
22
|
+
if threshold is None:
|
|
23
|
+
threshold = get_runtime_config().defaults.recurrence_threshold
|
|
24
|
+
|
|
16
25
|
with span("syslog_demo", path=path, tenant_id=tenant_id, threshold=threshold):
|
|
17
26
|
init_db(db_path)
|
|
18
27
|
events = ingest_syslog_file(path, tenant_id=tenant_id)
|
|
@@ -52,7 +61,12 @@ def main() -> int:
|
|
|
52
61
|
parser = argparse.ArgumentParser(description="Run the brAInstem syslog weak-signal demo.")
|
|
53
62
|
parser.add_argument("path", help="Path to a syslog-like input file")
|
|
54
63
|
parser.add_argument("--tenant", default="demo-tenant", help="Tenant/environment identifier")
|
|
55
|
-
parser.add_argument(
|
|
64
|
+
parser.add_argument(
|
|
65
|
+
"--threshold",
|
|
66
|
+
type=int,
|
|
67
|
+
default=get_runtime_config().defaults.recurrence_threshold,
|
|
68
|
+
help="Minimum recurrence count for candidate emission",
|
|
69
|
+
)
|
|
56
70
|
parser.add_argument("--db-path", default=None, help="Optional SQLite path for persistent state")
|
|
57
71
|
args = parser.parse_args()
|
|
58
72
|
payload = run_syslog_demo(args.path, tenant_id=args.tenant, threshold=args.threshold, db_path=args.db_path)
|
package/brainstem/fingerprint.py
CHANGED
|
@@ -9,6 +9,54 @@ _WHITESPACE_RE = re.compile(r"\s+")
|
|
|
9
9
|
_IPV4_RE = re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}\b")
|
|
10
10
|
_NUMBER_RE = re.compile(r"\b\d+\b")
|
|
11
11
|
|
|
12
|
+
_CONNECTIVITY_SERVICE_HINTS = {
|
|
13
|
+
"charon",
|
|
14
|
+
"ipsec",
|
|
15
|
+
"wireguard",
|
|
16
|
+
"strongswan",
|
|
17
|
+
"openvpn",
|
|
18
|
+
"bgp",
|
|
19
|
+
}
|
|
20
|
+
_CONNECTIVITY_ANCHORS = {
|
|
21
|
+
"vpn",
|
|
22
|
+
"tunnel",
|
|
23
|
+
"rekey",
|
|
24
|
+
"ipsec",
|
|
25
|
+
"handshake",
|
|
26
|
+
"peer",
|
|
27
|
+
}
|
|
28
|
+
_CONNECTIVITY_STATE_HINTS = {
|
|
29
|
+
"down",
|
|
30
|
+
"up",
|
|
31
|
+
"dropped",
|
|
32
|
+
"recovered",
|
|
33
|
+
"unreachable",
|
|
34
|
+
"timeout",
|
|
35
|
+
"flapped",
|
|
36
|
+
}
|
|
37
|
+
_RESOURCE_HINTS = {
|
|
38
|
+
"disk",
|
|
39
|
+
"memory",
|
|
40
|
+
"cpu",
|
|
41
|
+
"storage",
|
|
42
|
+
"inode",
|
|
43
|
+
"pressure",
|
|
44
|
+
"filesystem",
|
|
45
|
+
"out of space",
|
|
46
|
+
"swap",
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _has_any(text: str, values: set[str]) -> bool:
|
|
51
|
+
return any(value in text for value in values)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _has_connectivity_context(message: str, service: str) -> bool:
|
|
55
|
+
service_hint = service and _has_any(service, _CONNECTIVITY_SERVICE_HINTS)
|
|
56
|
+
anchor_hint = _has_any(message, _CONNECTIVITY_ANCHORS)
|
|
57
|
+
state_hint = _has_any(message, _CONNECTIVITY_STATE_HINTS)
|
|
58
|
+
return service_hint or (anchor_hint and state_hint)
|
|
59
|
+
|
|
12
60
|
|
|
13
61
|
def normalize_message(message: str) -> str:
|
|
14
62
|
text = (message or "").strip().lower()
|
|
@@ -21,10 +69,16 @@ def normalize_message(message: str) -> str:
|
|
|
21
69
|
def event_family_for(event: Event) -> str:
|
|
22
70
|
message_normalized = getattr(event, "message_normalized", None) or normalize_message(event.message_raw)
|
|
23
71
|
base = message_normalized
|
|
72
|
+
service = (event.service or "").strip().lower()
|
|
73
|
+
|
|
24
74
|
if "fail" in base or "error" in base:
|
|
25
75
|
return "failure"
|
|
26
76
|
if "restart" in base or "stopped" in base or "started" in base:
|
|
27
77
|
return "service_lifecycle"
|
|
78
|
+
if _has_connectivity_context(base, service):
|
|
79
|
+
return "connectivity"
|
|
80
|
+
if _has_any(base, _RESOURCE_HINTS):
|
|
81
|
+
return "resource"
|
|
28
82
|
if "auth" in base or "login" in base:
|
|
29
83
|
return "auth"
|
|
30
84
|
return "generic"
|
package/brainstem/ingest.py
CHANGED
|
@@ -6,6 +6,7 @@ from datetime import datetime
|
|
|
6
6
|
from pathlib import Path
|
|
7
7
|
from typing import Callable, Iterable, List, Optional
|
|
8
8
|
|
|
9
|
+
from .config import get_runtime_config
|
|
9
10
|
from .fingerprint import fingerprint_event, normalize_message
|
|
10
11
|
from .models import Candidate, CanonicalEvent, Event, RawInputEnvelope, Signature
|
|
11
12
|
from .recurrence import build_recurrence_candidates
|
|
@@ -214,12 +215,15 @@ def replay_raw_envelopes_by_ids(
|
|
|
214
215
|
raw_envelope_ids: Iterable[int | str | object],
|
|
215
216
|
*,
|
|
216
217
|
db_path: str,
|
|
217
|
-
threshold: int =
|
|
218
|
+
threshold: int | None = None,
|
|
218
219
|
on_event: Optional[Callable[[CanonicalEvent], None]] = None,
|
|
219
220
|
on_parse_error: Optional[ErrorHandler] = None,
|
|
220
221
|
force: bool = False,
|
|
221
222
|
allowed_statuses: Iterable[str] = ("received", "parse_failed"),
|
|
222
223
|
) -> ReplayResult:
|
|
224
|
+
if threshold is None:
|
|
225
|
+
threshold = get_runtime_config().defaults.replay_threshold
|
|
226
|
+
|
|
223
227
|
requested_raw_envelope_ids = list(dict.fromkeys([_coerce_raw_envelope_id(item) for item in raw_envelope_ids]))
|
|
224
228
|
requested_raw_envelope_ids = [item for item in requested_raw_envelope_ids if item is not None]
|
|
225
229
|
|
|
@@ -294,12 +298,15 @@ def replay_raw_envelopes_by_ids(
|
|
|
294
298
|
def run_ingest_pipeline(
|
|
295
299
|
raw_envelopes: Iterable[RawInputEnvelope],
|
|
296
300
|
*,
|
|
297
|
-
threshold: int =
|
|
301
|
+
threshold: int | None = None,
|
|
298
302
|
db_path: str | None = None,
|
|
299
303
|
store_raw: bool = True,
|
|
300
304
|
on_event: Optional[Callable[[CanonicalEvent], None]] = None,
|
|
301
305
|
on_parse_error: Optional[ErrorHandler] = None,
|
|
302
306
|
) -> IngestionResult:
|
|
307
|
+
if threshold is None:
|
|
308
|
+
threshold = get_runtime_config().defaults.ingest_threshold
|
|
309
|
+
|
|
303
310
|
raw_envelopes_list = list(raw_envelopes)
|
|
304
311
|
raw_envelope_ids: List[int] = []
|
|
305
312
|
if db_path:
|
|
@@ -387,7 +394,7 @@ def run_ingest_source_payload(
|
|
|
387
394
|
*,
|
|
388
395
|
tenant_id: str,
|
|
389
396
|
source_path: str,
|
|
390
|
-
threshold: int =
|
|
397
|
+
threshold: int | None = None,
|
|
391
398
|
db_path: Optional[str] = None,
|
|
392
399
|
on_event: Optional[Callable[[CanonicalEvent], None]] = None,
|
|
393
400
|
on_parse_error: Optional[ErrorHandler] = None,
|
|
@@ -407,12 +414,34 @@ def run_ingest_source_payload(
|
|
|
407
414
|
)
|
|
408
415
|
|
|
409
416
|
|
|
417
|
+
def run_ingest_logicmonitor_events(
|
|
418
|
+
events: Iterable[object],
|
|
419
|
+
*,
|
|
420
|
+
tenant_id: str,
|
|
421
|
+
source_path: str = "/logicmonitor/ingest",
|
|
422
|
+
threshold: int | None = None,
|
|
423
|
+
db_path: Optional[str] = None,
|
|
424
|
+
on_event: Optional[Callable[[CanonicalEvent], None]] = None,
|
|
425
|
+
on_parse_error: Optional[ErrorHandler] = None,
|
|
426
|
+
) -> IngestionResult:
|
|
427
|
+
return run_ingest_source_payload(
|
|
428
|
+
"logicmonitor",
|
|
429
|
+
list(events),
|
|
430
|
+
tenant_id=tenant_id,
|
|
431
|
+
source_path=source_path,
|
|
432
|
+
threshold=threshold,
|
|
433
|
+
db_path=db_path,
|
|
434
|
+
on_event=on_event,
|
|
435
|
+
on_parse_error=on_parse_error,
|
|
436
|
+
)
|
|
437
|
+
|
|
438
|
+
|
|
410
439
|
def run_ingest_file_lines(
|
|
411
440
|
lines: Iterable[str],
|
|
412
441
|
*,
|
|
413
442
|
tenant_id: str,
|
|
414
443
|
source_path: str,
|
|
415
|
-
threshold: int =
|
|
444
|
+
threshold: int | None = None,
|
|
416
445
|
db_path: str | None = None,
|
|
417
446
|
on_event: Optional[Callable[[CanonicalEvent], None]] = None,
|
|
418
447
|
on_parse_error: Optional[ErrorHandler] = None,
|
|
@@ -451,7 +480,7 @@ def run_ingest_file(
|
|
|
451
480
|
path: str,
|
|
452
481
|
*,
|
|
453
482
|
tenant_id: str,
|
|
454
|
-
threshold: int =
|
|
483
|
+
threshold: int | None = None,
|
|
455
484
|
db_path: Optional[str] = None,
|
|
456
485
|
on_event: Optional[Callable[[CanonicalEvent], None]] = None,
|
|
457
486
|
on_parse_error: Optional[ErrorHandler] = None,
|
package/brainstem/listener.py
CHANGED
|
@@ -64,11 +64,14 @@ def run_ingest_syslog_datagram(
|
|
|
64
64
|
*,
|
|
65
65
|
tenant_id: str,
|
|
66
66
|
source_path: str = LISTENER_CONFIG.syslog_source_path,
|
|
67
|
-
threshold: int =
|
|
67
|
+
threshold: int | None = None,
|
|
68
68
|
db_path: Optional[str] = None,
|
|
69
69
|
on_event: Optional[EventHandler] = None,
|
|
70
70
|
on_parse_error: Optional[ErrorHandler] = None,
|
|
71
71
|
) -> IngestionResult:
|
|
72
|
+
if threshold is None:
|
|
73
|
+
threshold = get_runtime_config().listener.ingest_threshold
|
|
74
|
+
|
|
72
75
|
return run_ingest_source_payload(
|
|
73
76
|
"syslog",
|
|
74
77
|
payload,
|
|
@@ -90,11 +93,14 @@ def run_udp_syslog_listener(
|
|
|
90
93
|
on_event: Optional[EventHandler] = None,
|
|
91
94
|
on_parse_error: Optional[ErrorHandler] = None,
|
|
92
95
|
db_path: Optional[str] = None,
|
|
93
|
-
threshold: int =
|
|
96
|
+
threshold: int | None = None,
|
|
94
97
|
max_datagrams: Optional[int] = None,
|
|
95
98
|
socket_timeout: float = LISTENER_CONFIG.syslog_socket_timeout,
|
|
96
99
|
socket_obj: Optional[socket.socket] = None,
|
|
97
100
|
) -> int:
|
|
101
|
+
if threshold is None:
|
|
102
|
+
threshold = get_runtime_config().listener.ingest_threshold
|
|
103
|
+
|
|
98
104
|
sock = socket_obj
|
|
99
105
|
owns_socket = sock is None
|
|
100
106
|
if sock is None:
|
package/brainstem/recurrence.py
CHANGED
|
@@ -7,12 +7,15 @@ from typing import Iterable, List
|
|
|
7
7
|
from .models import Candidate, Event, Signature
|
|
8
8
|
from .interesting import _attention_explanation
|
|
9
9
|
from .scoring import score_candidate
|
|
10
|
+
from .config import get_runtime_config
|
|
10
11
|
|
|
11
12
|
|
|
12
13
|
FAMILY_TITLES = {
|
|
13
14
|
"failure": "Recurring failure pattern",
|
|
14
15
|
"auth": "Recurring authentication failure pattern",
|
|
15
16
|
"service_lifecycle": "Recurring service lifecycle instability",
|
|
17
|
+
"connectivity": "Recurring network connectivity pattern",
|
|
18
|
+
"resource": "Recurring resource pressure pattern",
|
|
16
19
|
"generic": "Recurring operational pattern",
|
|
17
20
|
}
|
|
18
21
|
|
|
@@ -75,7 +78,17 @@ def _signature_lineage_index(
|
|
|
75
78
|
return per_signature_raw_ids
|
|
76
79
|
|
|
77
80
|
|
|
78
|
-
def build_recurrence_candidates(
|
|
81
|
+
def build_recurrence_candidates(
|
|
82
|
+
events: List[Event],
|
|
83
|
+
signatures: List[Signature],
|
|
84
|
+
*,
|
|
85
|
+
threshold: int | None = None,
|
|
86
|
+
) -> List[Candidate]:
|
|
87
|
+
runtime_config = get_runtime_config()
|
|
88
|
+
profile = runtime_config.candidate_attention
|
|
89
|
+
if threshold is None:
|
|
90
|
+
threshold = runtime_config.defaults.recurrence_threshold
|
|
91
|
+
|
|
79
92
|
counts = signature_counts(signatures)
|
|
80
93
|
candidates: List[Candidate] = []
|
|
81
94
|
signature_raw_envelope_ids = _signature_lineage_index(events, signatures)
|
|
@@ -83,13 +96,17 @@ def build_recurrence_candidates(events: List[Event], signatures: List[Signature]
|
|
|
83
96
|
count = counts[signature.signature_key]
|
|
84
97
|
if count < threshold:
|
|
85
98
|
continue
|
|
86
|
-
recurrence = min(count /
|
|
87
|
-
recovery =
|
|
88
|
-
spread =
|
|
89
|
-
novelty =
|
|
90
|
-
impact =
|
|
91
|
-
|
|
92
|
-
|
|
99
|
+
recurrence = min(count / float(profile.recurrence_count_normalizer), 1.0)
|
|
100
|
+
recovery = profile.recovery_signal_weight
|
|
101
|
+
spread = profile.spread_signal_weight
|
|
102
|
+
novelty = profile.novelty_signal_weight
|
|
103
|
+
impact = (
|
|
104
|
+
profile.impact_high_weight
|
|
105
|
+
if signature.event_family in {"failure", "auth"}
|
|
106
|
+
else profile.impact_default_weight
|
|
107
|
+
)
|
|
108
|
+
precursor = profile.precursor_weight
|
|
109
|
+
memory_weight = profile.memory_weight
|
|
93
110
|
candidate = score_candidate(
|
|
94
111
|
recurrence=recurrence,
|
|
95
112
|
recovery=recovery,
|
package/brainstem/scoring.py
CHANGED
|
@@ -1,16 +1,18 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
from .models import Candidate
|
|
4
|
+
from .config import get_runtime_config
|
|
4
5
|
|
|
5
6
|
|
|
6
7
|
def decision_band(score_total: float) -> str:
|
|
7
|
-
|
|
8
|
+
profile = get_runtime_config().candidate_attention
|
|
9
|
+
if score_total >= profile.decision_band_promote:
|
|
8
10
|
return "promote_to_incident_memory"
|
|
9
|
-
if score_total >=
|
|
11
|
+
if score_total >= profile.decision_band_urgent_human_review:
|
|
10
12
|
return "urgent_human_review"
|
|
11
|
-
if score_total >=
|
|
13
|
+
if score_total >= profile.decision_band_review:
|
|
12
14
|
return "review"
|
|
13
|
-
if score_total >=
|
|
15
|
+
if score_total >= profile.decision_band_watch:
|
|
14
16
|
return "watch"
|
|
15
17
|
return "ignore"
|
|
16
18
|
|