@handsforbots/semantic-event-observability 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +9 -0
- package/README.md +141 -2
- package/adapters/genericEventBus.js +20 -0
- package/adapters/instrumentChannel.js +22 -0
- package/core/CorrelationContext.js +49 -0
- package/core/EventBuffer.js +37 -0
- package/core/MetricsRegistry.js +200 -0
- package/core/PhaseTracker.js +143 -0
- package/core/Policy.js +87 -0
- package/core/SessionTracker.js +23 -0
- package/core/TraceContextBridge.js +74 -0
- package/core/TraceMapper.js +167 -0
- package/core/TurnMetricsCollector.js +19 -0
- package/core/TurnModel.js +87 -0
- package/core/baggage.d.ts +12 -0
- package/core/baggage.js +68 -0
- package/core/bucketEventName.js +11 -0
- package/core/createObservability.js +611 -0
- package/core/definePhaseModel.js +14 -0
- package/core/eventInstrumentation.js +21 -0
- package/core/isErrorEvent.js +9 -0
- package/core/telemetryAttributes.js +44 -0
- package/core/turnSpanNaming.js +73 -0
- package/docs/architecture.md +106 -0
- package/docs/custom-exporter.md +135 -0
- package/docs/exporters.md +180 -0
- package/docs/getting-started.md +192 -0
- package/docs/grafana-validation.md +71 -0
- package/docs/handsforbots-adapter.md +64 -0
- package/docs/handsforbots-roadmap.md +193 -0
- package/docs/metrics-roadmap.md +145 -0
- package/docs/publishing.md +55 -0
- package/docs/renaming.md +65 -0
- package/docs/roadmap.md +467 -0
- package/docs/tail-sampling.md +99 -0
- package/exporters/console.js +29 -0
- package/exporters/devPanel.js +204 -0
- package/exporters/faro.js +71 -0
- package/exporters/index.js +58 -0
- package/exporters/langfuse.js +161 -0
- package/exporters/langsmith.js +168 -0
- package/exporters/llmExporterState.js +84 -0
- package/exporters/memory.js +18 -0
- package/exporters/otel.js +236 -0
- package/exporters/otelTraceBackend.js +51 -0
- package/exporters/webVitals.js +73 -0
- package/grafana/semantic-event-observability.json +250 -0
- package/grafana/semantic-event-observability.lgtm.json +399 -0
- package/index.d.ts +312 -0
- package/index.js +31 -0
- package/package.json +83 -3
- package/packageIdentity.js +41 -0
- package/utils/id.js +10 -0
- package/utils/probeOptional.js +42 -0
- package/utils/safeJson.js +74 -0
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { sevoAttributes } from './telemetryAttributes.js'
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Normalize turn root span configuration for TraceMapper.
|
|
5
|
+
*/
|
|
6
|
+
export function normalizeTurnRootSpanConfig(config) {
|
|
7
|
+
if (config === true || config === 'invoke_agent') {
|
|
8
|
+
return { mode: 'invoke_agent', kind: 'internal' }
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
if (typeof config === 'string') {
|
|
12
|
+
return { mode: config === 'invoke_agent' ? 'invoke_agent' : 'semantic' }
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
if (config && typeof config === 'object') {
|
|
16
|
+
return {
|
|
17
|
+
mode: config.mode === 'invoke_agent' ? 'invoke_agent' : 'semantic',
|
|
18
|
+
agentName: config.agentName,
|
|
19
|
+
providerName: config.providerName,
|
|
20
|
+
kind: config.kind || 'internal',
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
return { mode: 'semantic' }
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Build turn root span name and attributes (semantic or OTel invoke_agent).
|
|
29
|
+
*/
|
|
30
|
+
export function buildTurnRootSpan(event, config = {}) {
|
|
31
|
+
const normalized = normalizeTurnRootSpanConfig(config)
|
|
32
|
+
|
|
33
|
+
if (normalized.mode === 'invoke_agent') {
|
|
34
|
+
const agentName = resolveAgentName(normalized.agentName, event)
|
|
35
|
+
const name = agentName ? `invoke_agent ${agentName}` : 'invoke_agent'
|
|
36
|
+
const attributes = {
|
|
37
|
+
...sevoAttributes(event),
|
|
38
|
+
'gen_ai.operation.name': 'invoke_agent',
|
|
39
|
+
'sevo.turn_start_event': event.name,
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
if (agentName) attributes['gen_ai.agent.name'] = agentName
|
|
43
|
+
if (normalized.providerName) attributes['gen_ai.provider.name'] = normalized.providerName
|
|
44
|
+
|
|
45
|
+
return {
|
|
46
|
+
name,
|
|
47
|
+
attributes,
|
|
48
|
+
kind: normalized.kind,
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
return {
|
|
53
|
+
name: `turn:${event.name || 'conversation'}`,
|
|
54
|
+
attributes: sevoAttributes(event),
|
|
55
|
+
kind: undefined,
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function resolveAgentName(agentName, event) {
|
|
60
|
+
if (typeof agentName === 'function') {
|
|
61
|
+
try {
|
|
62
|
+
const resolved = agentName(event)
|
|
63
|
+
if (resolved) return String(resolved)
|
|
64
|
+
} catch { /* noop */ }
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
if (typeof agentName === 'string' && agentName) return agentName
|
|
68
|
+
|
|
69
|
+
const metadata = event.turnMetadata
|
|
70
|
+
if (metadata?.input_plugin) return String(metadata.input_plugin)
|
|
71
|
+
|
|
72
|
+
return undefined
|
|
73
|
+
}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
# Architecture
|
|
2
|
+
|
|
3
|
+
Host-agnostic design. For **Hands for Bots** integration, see [handsforbots-adapter.md](./handsforbots-adapter.md). For planned abstractions, see [roadmap.md](./roadmap.md).
|
|
4
|
+
|
|
5
|
+
```mermaid
|
|
6
|
+
flowchart TB
|
|
7
|
+
App["Host application"]
|
|
8
|
+
Adapter["Optional adapter preset"]
|
|
9
|
+
Bus["Event bus (on/trigger)"]
|
|
10
|
+
Core["createObservability"]
|
|
11
|
+
Policy["Policy<br/>sample · redact · rate limit"]
|
|
12
|
+
Corr["CorrelationContext / TurnModel<br/>session · turn · trace"]
|
|
13
|
+
Phase["PhaseTracker / definePhaseModel"]
|
|
14
|
+
Buffer["EventBuffer"]
|
|
15
|
+
Metrics["MetricsRegistry"]
|
|
16
|
+
Exporters["Exporters (optional)"]
|
|
17
|
+
Grafana["Grafana LGTM"]
|
|
18
|
+
LLM["Langfuse / LangSmith"]
|
|
19
|
+
|
|
20
|
+
App --> Adapter
|
|
21
|
+
Adapter --> Bus
|
|
22
|
+
Bus --> Core
|
|
23
|
+
Core --> Policy
|
|
24
|
+
Core --> Corr
|
|
25
|
+
Core --> Phase
|
|
26
|
+
Core --> Buffer
|
|
27
|
+
Core --> Metrics
|
|
28
|
+
Core --> Exporters
|
|
29
|
+
Exporters --> Grafana
|
|
30
|
+
Exporters --> LLM
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
## Scope
|
|
34
|
+
|
|
35
|
+
This library answers **how the conversational flow behaves while the app is running** — not **whether the service is reachable**.
|
|
36
|
+
|
|
37
|
+
| In scope (`sevo_*`) | Out of scope (other stack) |
|
|
38
|
+
|---------------------|----------------------------|
|
|
39
|
+
| Turn and phase latency | Uptime / synthetic monitoring |
|
|
40
|
+
| Turn completion, abandonment, errors | Backend `/health` endpoints (e.g. Rasa `/api/health`) |
|
|
41
|
+
| Queue pressure via `stateProvider` | Kubernetes liveness/readiness probes |
|
|
42
|
+
| Telemetry pipeline health (`sevo_exporter_errors_total`, policy drops) | Server-side APM for backends (`gen_ai.*`) |
|
|
43
|
+
| Optional Core Web Vitals (`sevo_web_vital`) | Binary “application is up” alerts |
|
|
44
|
+
|
|
45
|
+
**Functional degradation** can be inferred from metrics (abandoned turns, persistent queue depth, exporter failures). **Absence of metrics** does not prove an outage — export may be disabled, sampled, or the collector may be down.
|
|
46
|
+
|
|
47
|
+
Full boundaries: [roadmap.md](./roadmap.md#boundary-library-vs-host).
|
|
48
|
+
|
|
49
|
+
## Semantic event schema
|
|
50
|
+
|
|
51
|
+
Each recorded event includes:
|
|
52
|
+
|
|
53
|
+
| Field | Description |
|
|
54
|
+
|-------|-------------|
|
|
55
|
+
| `id` | Unique event id |
|
|
56
|
+
| `timestamp` | ISO-8601 |
|
|
57
|
+
| `type` | `bus.trigger`, `bus.listener`, `turn.start`, `turn.end`, `turn.abandoned`, `phase.start`, `phase.end`, `custom` |
|
|
58
|
+
| `name` | Event name (`core.input`, …) |
|
|
59
|
+
| `sessionId` | Browser session |
|
|
60
|
+
| `turnId` | Conversation turn (correlates input → output) |
|
|
61
|
+
| `traceId` | Trace id for OTel/Faro/Langfuse |
|
|
62
|
+
| `payloadSummary` | Redacted preview |
|
|
63
|
+
| `state` | Optional host state (queue depth, …) |
|
|
64
|
+
| `durationMs` | Turn duration on `turn.end` |
|
|
65
|
+
|
|
66
|
+
## Turn correlation
|
|
67
|
+
|
|
68
|
+
Configure which bus events open/close a turn via `turnStartEvents` / `turnEndEvents` (formal **TurnModel** in [roadmap.md](./roadmap.md)):
|
|
69
|
+
|
|
70
|
+
```javascript
|
|
71
|
+
turnStartEvents: ['user.message'],
|
|
72
|
+
turnEndEvents: ['bot.response'],
|
|
73
|
+
phases: definePhaseModel([
|
|
74
|
+
{ id: 'backend', startEvent: 'api.call', endEvent: 'api.done' },
|
|
75
|
+
]),
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
Hands for Bots preset: `core.input` / `core.output_ready` — see [handsforbots-adapter.md](./handsforbots-adapter.md).
|
|
79
|
+
|
|
80
|
+
Turn boundaries drive spans in OTel, runs in LangSmith, and turn panels in Grafana.
|
|
81
|
+
|
|
82
|
+
Each turn produces a **single Tempo trace** (root `turn:*`, children `phase:*` + `event:*`). Phase children are configured via **PhaseModel** (planned; today backend phase is hardcoded in the OTel exporter).
|
|
83
|
+
|
|
84
|
+
## Fail-safe design
|
|
85
|
+
|
|
86
|
+
- Exporter `init()` errors are caught per exporter.
|
|
87
|
+
- Optional modules use dynamic `import()` inside try/catch.
|
|
88
|
+
- Missing peer dependencies → exporter marked unavailable.
|
|
89
|
+
- Telemetry never throws into application event handlers.
|
|
90
|
+
|
|
91
|
+
## Extraction to standalone npm package
|
|
92
|
+
|
|
93
|
+
```
|
|
94
|
+
SemanticEventObservability/
|
|
95
|
+
├── packageIdentity.js ← rename here
|
|
96
|
+
├── package.json
|
|
97
|
+
├── index.js
|
|
98
|
+
├── core/
|
|
99
|
+
├── adapters/
|
|
100
|
+
├── exporters/
|
|
101
|
+
├── utils/
|
|
102
|
+
├── grafana/
|
|
103
|
+
└── docs/
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
The Hands for Bots repo ships a thin adapter; see [handsforbots-roadmap.md](./handsforbots-roadmap.md) for consumer-specific work.
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
# Custom exporter cookbook
|
|
2
|
+
|
|
3
|
+
Build a custom exporter when you need to send semantic events or `sevo_*` metrics to a backend that is not covered by the built-ins (Faro, OTel, Langfuse, LangSmith).
|
|
4
|
+
|
|
5
|
+
## Interface
|
|
6
|
+
|
|
7
|
+
Every exporter implements:
|
|
8
|
+
|
|
9
|
+
```javascript
|
|
10
|
+
{
|
|
11
|
+
id: 'myBackend', // unique string
|
|
12
|
+
available: false, // set true in init() when ready
|
|
13
|
+
description: '...', // optional, shown in getExporterStatus()
|
|
14
|
+
async init(context) {}, // receive identity, buffer, metricsRegistry, traceContextBridge
|
|
15
|
+
onEvent(event) {}, // semantic events (turn, phase, bus.trigger, custom)
|
|
16
|
+
onMetric(metric) {}, // sevo_* metrics from MetricsRegistry
|
|
17
|
+
onPhaseEnd(event) {}, // optional — phase.end only
|
|
18
|
+
destroy() {}, // optional cleanup
|
|
19
|
+
}
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
`init(context)` receives:
|
|
23
|
+
|
|
24
|
+
| Field | Use |
|
|
25
|
+
|-------|-----|
|
|
26
|
+
| `context.identity` | Package slug, OTel service name |
|
|
27
|
+
| `context.metricsRegistry` | Subscribe or inspect canonical metrics |
|
|
28
|
+
| `context.traceContextBridge` | Active turn trace propagation |
|
|
29
|
+
| `context.getTimeline()` | Recent events (memory buffer) |
|
|
30
|
+
| `context.getMetrics()` | Recent metric records |
|
|
31
|
+
| `context.isError(event)` | Default error heuristic |
|
|
32
|
+
|
|
33
|
+
## Minimal example
|
|
34
|
+
|
|
35
|
+
```javascript
|
|
36
|
+
import { createObservability } from '@handsforbots/semantic-event-observability'
|
|
37
|
+
|
|
38
|
+
const myExporter = {
|
|
39
|
+
id: 'webhook',
|
|
40
|
+
available: false,
|
|
41
|
+
description: 'POST semantic events to internal API',
|
|
42
|
+
|
|
43
|
+
async init(context) {
|
|
44
|
+
this.endpoint = context.identity.otelServiceName // or from config
|
|
45
|
+
this.available = Boolean(this.endpoint)
|
|
46
|
+
},
|
|
47
|
+
|
|
48
|
+
onEvent(event) {
|
|
49
|
+
if (event.type === 'turn.end') {
|
|
50
|
+
fetch('/telemetry/turns', {
|
|
51
|
+
method: 'POST',
|
|
52
|
+
headers: { 'content-type': 'application/json' },
|
|
53
|
+
body: JSON.stringify({
|
|
54
|
+
turnId: event.turnId,
|
|
55
|
+
durationMs: event.durationMs,
|
|
56
|
+
name: event.name,
|
|
57
|
+
}),
|
|
58
|
+
}).catch(() => {})
|
|
59
|
+
}
|
|
60
|
+
},
|
|
61
|
+
|
|
62
|
+
onMetric(metric) {
|
|
63
|
+
// metric.name is sevo_* ; metric.type is histogram | counter | gauge
|
|
64
|
+
},
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const observability = createObservability({
|
|
68
|
+
turnStartEvents: ['user.message'],
|
|
69
|
+
turnEndEvents: ['bot.response'],
|
|
70
|
+
exporters: ['memory'],
|
|
71
|
+
})
|
|
72
|
+
|
|
73
|
+
await observability.init()
|
|
74
|
+
await observability.registerExporter(myExporter)
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
## Register after init
|
|
78
|
+
|
|
79
|
+
Use `registerExporter()` to attach custom exporters at runtime. The lib calls `init()` with the same context as built-in exporters.
|
|
80
|
+
|
|
81
|
+
```javascript
|
|
82
|
+
const status = await observability.registerExporter(myExporter)
|
|
83
|
+
console.log(status) // { id: 'webhook', available: true }
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
You can also push directly before `init()` if you manage lifecycle yourself:
|
|
87
|
+
|
|
88
|
+
```javascript
|
|
89
|
+
observability.exporters.push(myExporter)
|
|
90
|
+
await observability.init()
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
## Phase hooks
|
|
94
|
+
|
|
95
|
+
`onPhaseEnd` fires for every `phase.end` semantic event — bus-driven phases, manual `startPhase`/`endPhase`, and computed render phase.
|
|
96
|
+
|
|
97
|
+
```javascript
|
|
98
|
+
onPhaseEnd(event) {
|
|
99
|
+
console.log(event.phase, event.durationMs, event.source) // bus | manual | computed
|
|
100
|
+
}
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
## Metrics-only exporter
|
|
104
|
+
|
|
105
|
+
If you only care about Prometheus-style metrics, implement `onMetric` and skip `onEvent`:
|
|
106
|
+
|
|
107
|
+
```javascript
|
|
108
|
+
const metricsSink = {
|
|
109
|
+
id: 'metrics-log',
|
|
110
|
+
available: true,
|
|
111
|
+
onMetric(metric) {
|
|
112
|
+
console.log(metric.name, metric.value, metric.labels)
|
|
113
|
+
},
|
|
114
|
+
}
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
Canonical names are in `SEVO_METRICS` (`sevo_turn_duration_ms`, `sevo_phase_duration_ms`, etc.).
|
|
118
|
+
|
|
119
|
+
## Error handling
|
|
120
|
+
|
|
121
|
+
Exporter handlers run inside try/catch. Uncaught errors increment `sevo_exporter_errors_total{exporter="your-id"}`. Keep handlers small and swallow expected network failures.
|
|
122
|
+
|
|
123
|
+
## Production checklist
|
|
124
|
+
|
|
125
|
+
- [ ] Set `available: false` until `init()` confirms dependencies
|
|
126
|
+
- [ ] Respect host sampling — do not bypass `Policy`
|
|
127
|
+
- [ ] Avoid PII in `onEvent`; use `payloadSummary` fields when present
|
|
128
|
+
- [ ] Use host-prefixed metric names for product analytics (`hfb_*`), not `sevo_*`
|
|
129
|
+
- [ ] Call `destroy()` to flush timers or close connections
|
|
130
|
+
|
|
131
|
+
## See also
|
|
132
|
+
|
|
133
|
+
- [exporters.md](./exporters.md) — built-in exporters
|
|
134
|
+
- [metrics-roadmap.md](./metrics-roadmap.md) — `sevo_*` catalog
|
|
135
|
+
- [test/testHarness.js](../test/testHarness.js) — integration test helper
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
# Exporters
|
|
2
|
+
|
|
3
|
+
All exporters implement the same interface:
|
|
4
|
+
|
|
5
|
+
```javascript
|
|
6
|
+
{
|
|
7
|
+
id: string,
|
|
8
|
+
available: boolean,
|
|
9
|
+
description: string,
|
|
10
|
+
init(context): Promise<void>,
|
|
11
|
+
onEvent(event): void,
|
|
12
|
+
onMetric(metric): void,
|
|
13
|
+
onPhaseEnd?(phaseEvent): void,
|
|
14
|
+
destroy(): void,
|
|
15
|
+
}
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## Built-in (zero dependencies)
|
|
19
|
+
|
|
20
|
+
### `memory`
|
|
21
|
+
|
|
22
|
+
Ring buffer for timeline and debug JSON export. Always safe to enable.
|
|
23
|
+
|
|
24
|
+
### `console`
|
|
25
|
+
|
|
26
|
+
Structured console output. Useful in development.
|
|
27
|
+
|
|
28
|
+
### `devPanel`
|
|
29
|
+
|
|
30
|
+
Floating UI panel. Enable with:
|
|
31
|
+
|
|
32
|
+
```javascript
|
|
33
|
+
localStorage.setItem('semantic-event-observability:debug', 'true')
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
Or `exporterConfig.devPanel.enabled = true`.
|
|
37
|
+
|
|
38
|
+
The header has **export** (download recent events and the browser), **−** (collapse to the title bar) and **×** (hide the panel and set the debug key to `false`). The collapsed choice is stored in `localStorage` under `<slug>:panel` (`minimized`) and restored on the next load. Events keep arriving while it is collapsed.
|
|
39
|
+
|
|
40
|
+
## Grafana stack
|
|
41
|
+
|
|
42
|
+
### `faro`
|
|
43
|
+
|
|
44
|
+
Requires `@grafana/faro-web-sdk` **or** `window.faro`.
|
|
45
|
+
|
|
46
|
+
Sends Faro custom events consumed by Loki. Pair with Faro Collector → Grafana.
|
|
47
|
+
|
|
48
|
+
```javascript
|
|
49
|
+
exporterConfig: {
|
|
50
|
+
faro: { client: faroInstance },
|
|
51
|
+
}
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
### `otel`
|
|
55
|
+
|
|
56
|
+
Requires `@opentelemetry/api` **or** injected tracer, meter, and trace API.
|
|
57
|
+
|
|
58
|
+
Creates **one trace tree per conversation turn**:
|
|
59
|
+
|
|
60
|
+
- `turn.start` → root span `turn:<startEvent>`
|
|
61
|
+
- Configured `phases` → `phase:<id>` spans (Hands for Bots: `phase:backend`)
|
|
62
|
+
- Each `bus.trigger` → child span `event:<name>` under the turn root
|
|
63
|
+
- Root attributes: `turn.duration_ms`, `phase.<id>_ms`, `phase.render_ms`
|
|
64
|
+
- **Metrics:** `sevo_*` histograms/counters via OTel Metrics API when `getMeter` is provided
|
|
65
|
+
- **Turn root spans:** default `turn:<startEvent>`; opt in to OTel GenAI `invoke_agent` via `turnRootSpan: 'invoke_agent'` or `{ mode: 'invoke_agent', providerName, agentName }`
|
|
66
|
+
|
|
67
|
+
```javascript
|
|
68
|
+
exporterConfig: {
|
|
69
|
+
otel: {
|
|
70
|
+
getTracer: stack.getTracer,
|
|
71
|
+
getMeter: stack.getMeter,
|
|
72
|
+
traceApi: { context, trace, SpanStatusCode },
|
|
73
|
+
},
|
|
74
|
+
}
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
`phases` are set on `createObservability({ phases })`. The Hands for Bots adapter passes `HFB_PHASE_MODEL`.
|
|
78
|
+
|
|
79
|
+
## LLM observability
|
|
80
|
+
|
|
81
|
+
### `langfuse`
|
|
82
|
+
|
|
83
|
+
Requires `@langfuse/tracing` **or** injected `startObservation`.
|
|
84
|
+
|
|
85
|
+
Maps:
|
|
86
|
+
|
|
87
|
+
- `turn.start` → Langfuse span (turn root)
|
|
88
|
+
- `phase.start` / `phase.end` → nested phase spans under turn root
|
|
89
|
+
- `turn.end` → span end + duration
|
|
90
|
+
- `bus.trigger` → Langfuse event (child of turn when nested API available)
|
|
91
|
+
|
|
92
|
+
```javascript
|
|
93
|
+
exporterConfig: {
|
|
94
|
+
langfuse: {
|
|
95
|
+
project: 'my-app',
|
|
96
|
+
startObservation, // optional injection
|
|
97
|
+
},
|
|
98
|
+
}
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
Ensure Langfuse OTel processor or cloud ingestion is configured in the host app.
|
|
102
|
+
|
|
103
|
+
### `langsmith`
|
|
104
|
+
|
|
105
|
+
Requires `langsmith` **or** injected `RunTree`.
|
|
106
|
+
|
|
107
|
+
Maps:
|
|
108
|
+
|
|
109
|
+
- `turn.start` → chain run (turn root)
|
|
110
|
+
- `phase.start` / `phase.end` → nested chain runs under turn root
|
|
111
|
+
- `turn.end` → run end
|
|
112
|
+
- `bus.trigger` → tool run (child of turn)
|
|
113
|
+
|
|
114
|
+
```javascript
|
|
115
|
+
exporterConfig: {
|
|
116
|
+
langsmith: {
|
|
117
|
+
projectName: 'my-app',
|
|
118
|
+
RunTree, // optional injection
|
|
119
|
+
tags: ['handsforbots'],
|
|
120
|
+
},
|
|
121
|
+
}
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
Set `LANGSMITH_API_KEY` in the host environment when using the LangSmith SDK directly.
|
|
125
|
+
|
|
126
|
+
## Web Vitals
|
|
127
|
+
|
|
128
|
+
### `webVitals`
|
|
129
|
+
|
|
130
|
+
Requires `web-vitals` **or** injected `onReport` callback.
|
|
131
|
+
|
|
132
|
+
Records Core Web Vitals as `sevo_web_vital` histograms with `vital` label (`LCP`, `INP`, `CLS`, `FCP`, `TTFB`).
|
|
133
|
+
|
|
134
|
+
```javascript
|
|
135
|
+
exporters: ['memory', 'otel', 'webVitals'],
|
|
136
|
+
exporterConfig: {
|
|
137
|
+
webVitals: {
|
|
138
|
+
// optional: vitals module injection
|
|
139
|
+
// vitals: webVitals,
|
|
140
|
+
metrics: ['LCP', 'INP', 'CLS'],
|
|
141
|
+
labels: { surface: 'chat' },
|
|
142
|
+
},
|
|
143
|
+
},
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
Install optional peer in the host app:
|
|
147
|
+
|
|
148
|
+
```bash
|
|
149
|
+
npm i web-vitals
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
## Sampling
|
|
153
|
+
|
|
154
|
+
See [tail-sampling.md](./tail-sampling.md) for head vs tail sampling with OTel Collector policies.
|
|
155
|
+
|
|
156
|
+
## Custom exporter
|
|
157
|
+
|
|
158
|
+
See [custom-exporter.md](./custom-exporter.md) for a full cookbook with `registerExporter()`, phase hooks, and production checklist.
|
|
159
|
+
|
|
160
|
+
Quick sketch:
|
|
161
|
+
|
|
162
|
+
```javascript
|
|
163
|
+
await observability.registerExporter({
|
|
164
|
+
id: 'myBackend',
|
|
165
|
+
available: true,
|
|
166
|
+
onEvent(event) { /* send to API */ },
|
|
167
|
+
onMetric(metric) { /* sevo_* metrics */ },
|
|
168
|
+
})
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
## Recommended production set
|
|
172
|
+
|
|
173
|
+
| Environment | Exporters |
|
|
174
|
+
|-------------|-----------|
|
|
175
|
+
| Local dev | `memory`, `console`, `devPanel` |
|
|
176
|
+
| Staging | `memory`, `faro`, `otel` |
|
|
177
|
+
| Production | `faro`, `otel` (+ `langfuse`/`langsmith` if LLM-heavy) |
|
|
178
|
+
| LLM debugging | add `langfuse`, `langsmith` |
|
|
179
|
+
|
|
180
|
+
Always keep `sampleRate` and `maxEventsPerMinute` conservative in production.
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
# Getting started
|
|
2
|
+
|
|
3
|
+
## Library vs Hands for Bots
|
|
4
|
+
|
|
5
|
+
Semantic Event Observability is **host-agnostic**. Hands for Bots is one integration path.
|
|
6
|
+
|
|
7
|
+
| Path | Doc |
|
|
8
|
+
|------|-----|
|
|
9
|
+
| Any `on` / `trigger` event bus | § Generic bus below |
|
|
10
|
+
| Hands for Bots plugin | § 1. Enable in Hands for Bots |
|
|
11
|
+
| Custom backend exporter | [custom-exporter.md](./custom-exporter.md) |
|
|
12
|
+
| Roadmap & abstractions | [roadmap.md](./roadmap.md) |
|
|
13
|
+
|
|
14
|
+
## Generic bus (framework-agnostic)
|
|
15
|
+
|
|
16
|
+
Use this path for React, Vue, Node event emitters, or any object with `on()` + `trigger()`.
|
|
17
|
+
|
|
18
|
+
### 1. Install
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
npm i @handsforbots/semantic-event-observability
|
|
22
|
+
# optional peers for Grafana stack:
|
|
23
|
+
npm i @opentelemetry/api @grafana/faro-web-sdk
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
### 2. Configure turn boundaries and phases
|
|
27
|
+
|
|
28
|
+
```javascript
|
|
29
|
+
import {
|
|
30
|
+
createObservability,
|
|
31
|
+
defineTurnModel,
|
|
32
|
+
definePhaseModel,
|
|
33
|
+
instrumentEventBus,
|
|
34
|
+
SEVO_METRICS,
|
|
35
|
+
} from '@handsforbots/semantic-event-observability'
|
|
36
|
+
|
|
37
|
+
const observability = createObservability({
|
|
38
|
+
environment: 'production',
|
|
39
|
+
turn: defineTurnModel({
|
|
40
|
+
start: ['user.message'],
|
|
41
|
+
end: ['bot.response'],
|
|
42
|
+
}),
|
|
43
|
+
phases: definePhaseModel([
|
|
44
|
+
{ id: 'backend', startEvent: 'api.call', endEvent: 'api.done' },
|
|
45
|
+
]),
|
|
46
|
+
exporters: ['memory', 'console'],
|
|
47
|
+
// optional: reduce noise on high-volume buses
|
|
48
|
+
eventAllowlist: ['user.message', 'api.call', 'api.done', 'bot.response'],
|
|
49
|
+
})
|
|
50
|
+
|
|
51
|
+
instrumentEventBus(observability, myBus, {
|
|
52
|
+
stateProvider: () => ({ queueDepth: myQueue.length }),
|
|
53
|
+
})
|
|
54
|
+
|
|
55
|
+
await observability.init()
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
Turn detection **always runs** even when `eventAllowlist` filters recording. Metrics use the `sevo_*` prefix (`SEVO_METRICS.TURN_DURATION`, etc.).
|
|
59
|
+
|
|
60
|
+
### 3. Manual phases and trace propagation
|
|
61
|
+
|
|
62
|
+
```javascript
|
|
63
|
+
myBus.trigger('user.message', [{ text: 'hello' }])
|
|
64
|
+
|
|
65
|
+
observability.startPhase('fetch')
|
|
66
|
+
await observability.withFetch('/api/chat', { method: 'POST', body: JSON.stringify({ text: 'hello' }) })
|
|
67
|
+
observability.endPhase('fetch')
|
|
68
|
+
|
|
69
|
+
myBus.trigger('bot.response', [{ text: 'hi there' }])
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
`withFetch` / `getTraceHeaders()` inject active turn trace context for backend correlation.
|
|
73
|
+
|
|
74
|
+
### 4. Inspect locally
|
|
75
|
+
|
|
76
|
+
```javascript
|
|
77
|
+
observability.getTimeline() // recent semantic events
|
|
78
|
+
observability.getMetrics() // sevo_* metric records
|
|
79
|
+
observability.getExporterStatus()
|
|
80
|
+
observability.getPolicyStats()
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
### 5. Add Grafana (optional)
|
|
84
|
+
|
|
85
|
+
Wire OTel + Faro in your app shell, then:
|
|
86
|
+
|
|
87
|
+
```javascript
|
|
88
|
+
createObservability({
|
|
89
|
+
exporters: ['memory', 'faro', 'otel'],
|
|
90
|
+
exporterConfig: {
|
|
91
|
+
otel: {
|
|
92
|
+
getTracer: stack.getTracer,
|
|
93
|
+
getMeter: stack.getMeter,
|
|
94
|
+
traceApi: stack.traceApi,
|
|
95
|
+
},
|
|
96
|
+
faro: { client: faro },
|
|
97
|
+
},
|
|
98
|
+
})
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
See [exporters.md](./exporters.md) and the Vite example in `examples/OBSERVABILITY.md`.
|
|
102
|
+
|
|
103
|
+
### 6. Custom exporter
|
|
104
|
+
|
|
105
|
+
```javascript
|
|
106
|
+
await observability.registerExporter({
|
|
107
|
+
id: 'webhook',
|
|
108
|
+
available: true,
|
|
109
|
+
onEvent(event) { /* send to your API */ },
|
|
110
|
+
})
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
Full walkthrough: [custom-exporter.md](./custom-exporter.md).
|
|
114
|
+
|
|
115
|
+
---
|
|
116
|
+
|
|
117
|
+
## 1. Enable in Hands for Bots
|
|
118
|
+
|
|
119
|
+
Register the Observability output plugin:
|
|
120
|
+
|
|
121
|
+
```javascript
|
|
122
|
+
bot_settings.plugins.push({
|
|
123
|
+
type: 'output',
|
|
124
|
+
plugin: 'Observability',
|
|
125
|
+
})
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
Omit the plugin (or set `enabled: false`) to disable observability entirely.
|
|
129
|
+
|
|
130
|
+
## 2. Dev panel (local triage)
|
|
131
|
+
|
|
132
|
+
```javascript
|
|
133
|
+
localStorage.setItem('semantic-event-observability:debug', 'true')
|
|
134
|
+
location.reload()
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
The panel shows recent semantic events and exports a sanitized JSON bundle. Full analysis belongs in Grafana.
|
|
138
|
+
|
|
139
|
+
## 3. Production stack (Grafana LGTM)
|
|
140
|
+
|
|
141
|
+
1. Install and init **Grafana Faro** in your app shell (optional but recommended).
|
|
142
|
+
2. Install **OpenTelemetry browser SDK** if you want Tempo traces (optional).
|
|
143
|
+
3. Configure the plugin:
|
|
144
|
+
|
|
145
|
+
```javascript
|
|
146
|
+
{
|
|
147
|
+
type: 'output',
|
|
148
|
+
plugin: 'Observability',
|
|
149
|
+
environment: 'production',
|
|
150
|
+
sampleRate: 0.1,
|
|
151
|
+
maxEventsPerMinute: 60,
|
|
152
|
+
exporters: ['memory', 'faro', 'otel'],
|
|
153
|
+
}
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
4. Run a collector with auth + rate limiting in front of Loki/Tempo.
|
|
157
|
+
5. Import the dashboard from `grafana/semantic-event-observability.json`.
|
|
158
|
+
|
|
159
|
+
## 4. Langfuse / LangSmith (optional)
|
|
160
|
+
|
|
161
|
+
```javascript
|
|
162
|
+
{
|
|
163
|
+
type: 'output',
|
|
164
|
+
plugin: 'Observability',
|
|
165
|
+
exporters: ['memory', 'langfuse', 'langsmith'],
|
|
166
|
+
exporterConfig: {
|
|
167
|
+
langfuse: { project: 'handsforbots' },
|
|
168
|
+
langsmith: { projectName: 'handsforbots' },
|
|
169
|
+
},
|
|
170
|
+
}
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
Install optional packages in the host app:
|
|
174
|
+
|
|
175
|
+
```bash
|
|
176
|
+
npm i @langfuse/tracing langsmith
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
If packages are absent, those exporters stay inactive — no runtime error.
|
|
180
|
+
|
|
181
|
+
## 5. Debug API on `window`
|
|
182
|
+
|
|
183
|
+
After the plugin loads:
|
|
184
|
+
|
|
185
|
+
```javascript
|
|
186
|
+
window.__SEMANTIC_EVENT_OBSERVABILITY__
|
|
187
|
+
// .getTimeline()
|
|
188
|
+
// .getExporterStatus()
|
|
189
|
+
// .getPolicyStats()
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
See [renaming.md](./renaming.md) for how the global key changes with the package slug.
|