@handsforbots/semantic-event-observability 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +9 -0
- package/README.md +141 -2
- package/adapters/genericEventBus.js +20 -0
- package/adapters/instrumentChannel.js +22 -0
- package/core/CorrelationContext.js +49 -0
- package/core/EventBuffer.js +37 -0
- package/core/MetricsRegistry.js +200 -0
- package/core/PhaseTracker.js +143 -0
- package/core/Policy.js +87 -0
- package/core/SessionTracker.js +23 -0
- package/core/TraceContextBridge.js +74 -0
- package/core/TraceMapper.js +167 -0
- package/core/TurnMetricsCollector.js +19 -0
- package/core/TurnModel.js +87 -0
- package/core/baggage.d.ts +12 -0
- package/core/baggage.js +68 -0
- package/core/bucketEventName.js +11 -0
- package/core/createObservability.js +611 -0
- package/core/definePhaseModel.js +14 -0
- package/core/eventInstrumentation.js +21 -0
- package/core/isErrorEvent.js +9 -0
- package/core/telemetryAttributes.js +44 -0
- package/core/turnSpanNaming.js +73 -0
- package/docs/architecture.md +106 -0
- package/docs/custom-exporter.md +135 -0
- package/docs/exporters.md +180 -0
- package/docs/getting-started.md +192 -0
- package/docs/grafana-validation.md +71 -0
- package/docs/handsforbots-adapter.md +64 -0
- package/docs/handsforbots-roadmap.md +193 -0
- package/docs/metrics-roadmap.md +145 -0
- package/docs/publishing.md +55 -0
- package/docs/renaming.md +65 -0
- package/docs/roadmap.md +467 -0
- package/docs/tail-sampling.md +99 -0
- package/exporters/console.js +29 -0
- package/exporters/devPanel.js +204 -0
- package/exporters/faro.js +71 -0
- package/exporters/index.js +58 -0
- package/exporters/langfuse.js +161 -0
- package/exporters/langsmith.js +168 -0
- package/exporters/llmExporterState.js +84 -0
- package/exporters/memory.js +18 -0
- package/exporters/otel.js +236 -0
- package/exporters/otelTraceBackend.js +51 -0
- package/exporters/webVitals.js +73 -0
- package/grafana/semantic-event-observability.json +250 -0
- package/grafana/semantic-event-observability.lgtm.json +399 -0
- package/index.d.ts +312 -0
- package/index.js +31 -0
- package/package.json +83 -3
- package/packageIdentity.js +41 -0
- package/utils/id.js +10 -0
- package/utils/probeOptional.js +42 -0
- package/utils/safeJson.js +74 -0
package/docs/roadmap.md
ADDED
|
@@ -0,0 +1,467 @@
|
|
|
1
|
+
# Library roadmap
|
|
2
|
+
|
|
3
|
+
Development plan for **Semantic Event Observability** as a **standalone, host-agnostic** front-end library.
|
|
4
|
+
|
|
5
|
+
> Hands for Bots is one consumer. Its metrics and adapter-specific work live in [handsforbots-roadmap.md](./handsforbots-roadmap.md).
|
|
6
|
+
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
## Vision
|
|
10
|
+
|
|
11
|
+
Offer industry-grade **semantic observability for browser event-driven applications** — chatbots, agent UIs, wizards, games, multi-step flows — without coupling to a specific framework.
|
|
12
|
+
|
|
13
|
+
**Value proposition for the industry:**
|
|
14
|
+
|
|
15
|
+
| Problem | What we offer |
|
|
16
|
+
|---------|----------------|
|
|
17
|
+
| Event buses are invisible in APM | Automatic instrumentation of `on` / `trigger` with correlation |
|
|
18
|
+
| User “turns” span many async steps | Configurable turn boundaries + trace trees |
|
|
19
|
+
| Front-end latency is a black box | Phase timing (request → response → render) on the client |
|
|
20
|
+
| Telemetry must not break the app | Fail-safe exporters, optional peers, policy guardrails |
|
|
21
|
+
| Vendor lock-in | OTel, Faro, Langfuse, LangSmith — pluggable exporters |
|
|
22
|
+
| LLM observability ignores the UI layer | Client-side turn root span; backend joins via trace context |
|
|
23
|
+
|
|
24
|
+
**Non-goals (stay out of the lib):**
|
|
25
|
+
|
|
26
|
+
- Backend LLM SDK instrumentation (`gen_ai.*` — host responsibility)
|
|
27
|
+
- Product analytics schemas (`hfb_*` — host adapter responsibility)
|
|
28
|
+
- Storing or evaluating prompts at scale (delegate to Langfuse / eval platforms)
|
|
29
|
+
- **Service uptime / availability** — synthetic HTTP probes, `/health` polling, “site down” alerting (infra / deploy responsibility)
|
|
30
|
+
- **Platform health checks** — Kubernetes liveness/readiness, Docker healthchecks for app containers (not the telemetry collector)
|
|
31
|
+
|
|
32
|
+
---
|
|
33
|
+
|
|
34
|
+
## Boundary: library vs host
|
|
35
|
+
|
|
36
|
+
```mermaid
|
|
37
|
+
flowchart TB
|
|
38
|
+
subgraph lib ["Semantic Event Observability"]
|
|
39
|
+
Turn["TurnModel"]
|
|
40
|
+
Phase["PhaseModel"]
|
|
41
|
+
Bus["BusInstrumenter"]
|
|
42
|
+
Metrics["MetricsRegistry"]
|
|
43
|
+
Trace["TraceMapper"]
|
|
44
|
+
Export["Exporter pipeline"]
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
subgraph host ["Host application e.g. Hands for Bots"]
|
|
48
|
+
Events["Domain events"]
|
|
49
|
+
State["StateProvider"]
|
|
50
|
+
Adapter["Thin adapter preset"]
|
|
51
|
+
Product["Product metrics hfb_*"]
|
|
52
|
+
Backend["Backend OTel / gen_ai.*"]
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
Events --> Adapter --> Bus
|
|
56
|
+
State --> Bus
|
|
57
|
+
Bus --> Turn & Phase
|
|
58
|
+
Turn & Phase --> Metrics & Trace
|
|
59
|
+
Metrics & Trace --> Export
|
|
60
|
+
Product -.->|recordMetric| Metrics
|
|
61
|
+
Backend -.->|traceparent| Trace
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
| Concern | Library (`sevo_*`) | Host (`hfb_*`, backend) |
|
|
65
|
+
|---------|-------------------|-------------------------|
|
|
66
|
+
| Turn start / end detection | Configurable `TurnModel` | Preset event names |
|
|
67
|
+
| Phase timing (generic) | `PhaseModel` e.g. `fetch`, `render` | Map events → phase config |
|
|
68
|
+
| Bus instrumentation | `instrument()` | Choose which bus |
|
|
69
|
+
| Runtime state snapshot | `stateProvider` callback | Return queue depth, etc. |
|
|
70
|
+
| Metric names & labels | `sevo_*` only | Host prefix + labels |
|
|
71
|
+
| Voice / plugin / modality | — | Host cores & plugins |
|
|
72
|
+
| Token usage, LLM latency | Trace correlation hook | Backend instrumentation |
|
|
73
|
+
| Grafana dashboard template | Generic turn/phase panels | Host-specific panels optional |
|
|
74
|
+
| Uptime / “app is up” | — | Infra, synthetics, backend `/health` endpoints |
|
|
75
|
+
| Telemetry pipeline health | `sevo_exporter_errors_total`, policy drops | Collector uptime (LGTM / Alloy deploy) |
|
|
76
|
+
|
|
77
|
+
> **Scope detail:** [architecture.md](./architecture.md#scope) · Hands for Bots plugin: [handsforbots-adapter.md](./handsforbots-adapter.md#scope)
|
|
78
|
+
|
|
79
|
+
---
|
|
80
|
+
|
|
81
|
+
## Industry alignment (front-end focus)
|
|
82
|
+
|
|
83
|
+
Production front-end and agent-UI observability expects:
|
|
84
|
+
|
|
85
|
+
### Three signals
|
|
86
|
+
|
|
87
|
+
| Signal | Library role |
|
|
88
|
+
|--------|----------------|
|
|
89
|
+
| **Traces** | One trace per turn; phase + event child spans |
|
|
90
|
+
| **Metrics** | Histograms/counters for turn duration, phases, drops, throughput |
|
|
91
|
+
| **Logs** | Semantic events → Faro/Loki (sampled bus traffic) |
|
|
92
|
+
|
|
93
|
+
### Four measurement layers (client-owned portions)
|
|
94
|
+
|
|
95
|
+
| Layer | Industry examples | Library coverage |
|
|
96
|
+
|-------|-------------------|------------------|
|
|
97
|
+
| **Infrastructure** | Load time, JS errors, service uptime | Partial: telemetry health (`sevo_exporter_errors_total`), optional Web Vitals; **not** endpoint uptime or K8s probes |
|
|
98
|
+
| **Execution** | Stage latency breakdown | Turn + **PhaseModel** durations |
|
|
99
|
+
| **Reliability** | Completion rate, errors, queue pressure | Turn completion/abandonment; state gauge from provider |
|
|
100
|
+
| **Outcome** | Task success, user feedback | `record()` / exporter only — evals stay external |
|
|
101
|
+
|
|
102
|
+
### Standards we align with
|
|
103
|
+
|
|
104
|
+
- **OpenTelemetry** — spans, metrics API, W3C trace context propagation
|
|
105
|
+
- **OTel GenAI conventions** — compatible trace IDs; backend children use `gen_ai.*` (not emitted by this lib)
|
|
106
|
+
- **Grafana Faro** — browser RUM + custom events
|
|
107
|
+
- **Langfuse / LangSmith** — turn-as-run mapping for LLM workflows
|
|
108
|
+
|
|
109
|
+
References: [OTel AI agent observability](https://opentelemetry.io/blog/2025/ai-agent-observability/), [GenAI metrics](https://hexdocs.pm/opentelemetry_semantic_conventions/gen-ai-metrics.html).
|
|
110
|
+
|
|
111
|
+
---
|
|
112
|
+
|
|
113
|
+
## Planned abstractions
|
|
114
|
+
|
|
115
|
+
Abstractions are ordered by dependency. **Bold** = new or needs extraction from current code.
|
|
116
|
+
|
|
117
|
+
### 1. Core envelope (exists — stabilize API)
|
|
118
|
+
|
|
119
|
+
| Abstraction | Module | Responsibility |
|
|
120
|
+
|-------------|--------|----------------|
|
|
121
|
+
| `SemanticEvent` | `core/createObservability.js` | Canonical fields: `id`, `type`, `name`, `timestamp`, correlation ids, `payloadSummary`, `state`, `durationMs` |
|
|
122
|
+
| `CorrelationContext` | `core/CorrelationContext.js` | Session, turn, trace ids; observes bus events |
|
|
123
|
+
| `Policy` | `core/Policy.js` | Sample, rate-limit, redact; never sample `turn.*` / `metric.*` |
|
|
124
|
+
| `EventBuffer` | `core/EventBuffer.js` | Ring buffer for timeline + debug export |
|
|
125
|
+
|
|
126
|
+
**Roadmap:** Document schema in [architecture.md](./architecture.md); add TypeScript/JSDoc typedef export.
|
|
127
|
+
|
|
128
|
+
---
|
|
129
|
+
|
|
130
|
+
### 2. TurnModel (exists as options — formalize)
|
|
131
|
+
|
|
132
|
+
Configurable boundaries for a correlated user interaction (conversation turn, wizard step, game round).
|
|
133
|
+
|
|
134
|
+
```javascript
|
|
135
|
+
const turnModel = {
|
|
136
|
+
startEvents: ['user.message'], // opens turn + new traceId
|
|
137
|
+
endEvents: ['ui.ready'], // closes turn + durationMs
|
|
138
|
+
onAbandoned: 'count', // 'count' | 'event' | 'ignore'
|
|
139
|
+
}
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
| Method | Purpose |
|
|
143
|
+
|--------|---------|
|
|
144
|
+
| `observeBusEvent(name)` | Returns `turn.start` / `turn.end` / null |
|
|
145
|
+
| `getContext()` | Current `sessionId`, `turnId`, `traceId` |
|
|
146
|
+
| `getActiveTurnAge()` | For stuck-turn detection |
|
|
147
|
+
|
|
148
|
+
**Roadmap:**
|
|
149
|
+
|
|
150
|
+
- [ ] Extract `TurnModel` from inline `CorrelationContext` options
|
|
151
|
+
- [~] Abandoned turn detection (new turn before `endEvents`) — in `CorrelationContext`
|
|
152
|
+
- [ ] Optional manual `startTurn()` / `endTurn()` for non-bus flows
|
|
153
|
+
- [ ] Export preset factories: `createConversationTurnModel({ start, end })`
|
|
154
|
+
|
|
155
|
+
---
|
|
156
|
+
|
|
157
|
+
### 3. PhaseModel (new — replaces hardcoded backend phase)
|
|
158
|
+
|
|
159
|
+
Declarative phases within a turn. Today `exporters/otel.js` hardcodes `core.calling_backend` → `core.backend_responded`. This moves to configuration.
|
|
160
|
+
|
|
161
|
+
```javascript
|
|
162
|
+
const phaseModel = [
|
|
163
|
+
{
|
|
164
|
+
id: 'remote',
|
|
165
|
+
startEvent: 'api.request',
|
|
166
|
+
endEvent: 'api.response',
|
|
167
|
+
metric: 'sevo_phase_duration_ms', // label phase=remote
|
|
168
|
+
spanName: 'phase:remote',
|
|
169
|
+
},
|
|
170
|
+
{
|
|
171
|
+
id: 'render',
|
|
172
|
+
startEvent: 'api.response', // implicit: wall-clock after previous phase
|
|
173
|
+
endEvent: 'ui.painted',
|
|
174
|
+
spanName: 'phase:render',
|
|
175
|
+
},
|
|
176
|
+
]
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
| Component | Responsibility |
|
|
180
|
+
|-----------|----------------|
|
|
181
|
+
| **`PhaseTracker`** | Subscribes to semantic events; starts/ends phases; computes durations |
|
|
182
|
+
| **`PhaseState`** | Per-turn open phases, timestamps |
|
|
183
|
+
| **Span hints** | Pass `phaseSpan` metadata to TraceMapper |
|
|
184
|
+
|
|
185
|
+
**Roadmap:**
|
|
186
|
+
|
|
187
|
+
- [x] `core/PhaseTracker.js` — event-driven phase timing + manual API
|
|
188
|
+
- [x] `core/PhaseModel.js` — validate + normalize phase config (`definePhaseModel`)
|
|
189
|
+
- [x] `exporters/otel.js` consumes `phase.start` / `phase.end` semantic events via `TraceMapper`
|
|
190
|
+
- [x] Manual API: `observability.startPhase(id)` / `endPhase(id)` for non-bus work
|
|
191
|
+
- [x] Default: zero phases (turn + events only) for minimal adopters
|
|
192
|
+
|
|
193
|
+
---
|
|
194
|
+
|
|
195
|
+
### 4. MetricsRegistry (new — single source of truth)
|
|
196
|
+
|
|
197
|
+
Central recording for `sevo_*` instruments. Exporters subscribe; no duplicate logic in OTel exporter.
|
|
198
|
+
|
|
199
|
+
```javascript
|
|
200
|
+
// core/MetricsRegistry.js
|
|
201
|
+
const instruments = {
|
|
202
|
+
turnDuration: histogram('sevo_turn_duration_ms', { unit: 'ms' }),
|
|
203
|
+
phaseDuration: histogram('sevo_phase_duration_ms', { labels: ['phase'] }),
|
|
204
|
+
turnsTotal: counter('sevo_turns_total', { labels: ['status'] }),
|
|
205
|
+
eventsDropped: counter('sevo_events_dropped_total', { labels: ['reason'] }),
|
|
206
|
+
stateGauge: observableGauge('sevo_state_gauge', { labels: ['key'] }),
|
|
207
|
+
}
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
| Rule | Detail |
|
|
211
|
+
|------|--------|
|
|
212
|
+
| Prefix | Always `sevo_` |
|
|
213
|
+
| Recording triggers | `turn.end`, phase end, `Policy.recordDrop()`, periodic state scrape |
|
|
214
|
+
| OTel export | Metrics API histograms/counters — **not** one span per metric |
|
|
215
|
+
| Host metrics | `recordMetric()` bypasses registry naming or uses `host.` prefix policy |
|
|
216
|
+
|
|
217
|
+
**Roadmap:** See [metrics-roadmap.md](./metrics-roadmap.md) checklist.
|
|
218
|
+
|
|
219
|
+
---
|
|
220
|
+
|
|
221
|
+
### 5. TraceMapper (extract from otel exporter)
|
|
222
|
+
|
|
223
|
+
Maps semantic events + phase state → OpenTelemetry span tree.
|
|
224
|
+
|
|
225
|
+
```
|
|
226
|
+
turn:<startEvent> [root]
|
|
227
|
+
├── phase:<id> [PhaseModel]
|
|
228
|
+
├── event:<busName> [each bus.trigger]
|
|
229
|
+
└── attributes: turn.duration_ms, phase.*_ms
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
| Concern | Approach |
|
|
233
|
+
|---------|----------|
|
|
234
|
+
| Error marking | Pluggable `isError(event)` — default heuristic, host override |
|
|
235
|
+
| Attributes | `sevo.*` namespace on spans |
|
|
236
|
+
| Active context | Set span context on turn root for child manual spans |
|
|
237
|
+
|
|
238
|
+
**Roadmap:**
|
|
239
|
+
|
|
240
|
+
- [x] `core/TraceMapper.js` — span lifecycle independent of OTel import
|
|
241
|
+
- [x] `exporters/otel.js` — thin adapter over TraceMapper + `otelTraceBackend.js`
|
|
242
|
+
- [x] Align span names with OTel incubating `invoke_agent` guidance where applicable
|
|
243
|
+
|
|
244
|
+
---
|
|
245
|
+
|
|
246
|
+
### 6. TraceContextBridge (new — backend correlation)
|
|
247
|
+
|
|
248
|
+
Enables distributed traces: browser turn root + server LLM/tool children.
|
|
249
|
+
|
|
250
|
+
```javascript
|
|
251
|
+
import { getTraceHeaders, withTraceContext } from './core/TraceContextBridge.js'
|
|
252
|
+
|
|
253
|
+
fetch(url, withTraceContext(observability, { method: 'POST', body }))
|
|
254
|
+
// injects traceparent from active turn trace
|
|
255
|
+
```
|
|
256
|
+
|
|
257
|
+
**Roadmap:**
|
|
258
|
+
|
|
259
|
+
- [x] Read active `traceId` / span context from turn root
|
|
260
|
+
- [x] `withTraceContext` / `withFetch` for outbound calls
|
|
261
|
+
- [x] Document host integration (no fetch patching inside lib by default)
|
|
262
|
+
|
|
263
|
+
---
|
|
264
|
+
|
|
265
|
+
### 7. BusInstrumenter (exists — extend)
|
|
266
|
+
|
|
267
|
+
| API | Status |
|
|
268
|
+
|-----|--------|
|
|
269
|
+
| `instrument(bus, { stateProvider, wrapListeners })` | Exists |
|
|
270
|
+
| `instrumentEventBus()` | Generic adapter |
|
|
271
|
+
| **`instrumentChannel(channel, options)`** | Generic BroadcastChannel / MessagePort helper |
|
|
272
|
+
| **`createInstrumentedBus()`** | Bootstrap helper |
|
|
273
|
+
|
|
274
|
+
**Roadmap:**
|
|
275
|
+
|
|
276
|
+
- [x] Extract channel instrumentation from HfB adapter into generic `instrumentChannel`
|
|
277
|
+
- [x] Optional `eventFilter(name) => boolean` to reduce noise
|
|
278
|
+
- [x] `eventAllowlist` mode for high-volume buses
|
|
279
|
+
|
|
280
|
+
---
|
|
281
|
+
|
|
282
|
+
### 8. Exporter pipeline (exists — extend)
|
|
283
|
+
|
|
284
|
+
```javascript
|
|
285
|
+
interface Exporter {
|
|
286
|
+
id: string
|
|
287
|
+
available: boolean
|
|
288
|
+
init(context): Promise<void>
|
|
289
|
+
onEvent(event): void
|
|
290
|
+
onMetric(metric): void // registry reading
|
|
291
|
+
onPhaseEnd?(phase): void // optional
|
|
292
|
+
destroy(): void
|
|
293
|
+
}
|
|
294
|
+
```
|
|
295
|
+
|
|
296
|
+
**Roadmap:**
|
|
297
|
+
|
|
298
|
+
- [x] `onPhaseEnd` for exporters that need phase granularity
|
|
299
|
+
- [ ] Shared `ExporterContext` with MetricsRegistry + TurnModel stats
|
|
300
|
+
- [ ] Custom exporter template in docs
|
|
301
|
+
|
|
302
|
+
---
|
|
303
|
+
|
|
304
|
+
### 9. Host adapter pattern (thin presets only)
|
|
305
|
+
|
|
306
|
+
Adapters **configure** the lib; they do not add core logic.
|
|
307
|
+
|
|
308
|
+
```javascript
|
|
309
|
+
// adapters/my-app.js
|
|
310
|
+
export function attachMyAppObservability(app, options) {
|
|
311
|
+
return createObservability({
|
|
312
|
+
turnModel: { startEvents: ['app.submit'], endEvents: ['app.rendered'] },
|
|
313
|
+
phases: MY_PHASES,
|
|
314
|
+
stateProvider: () => ({ queue: app.queue.length }),
|
|
315
|
+
exporters: options.exporters,
|
|
316
|
+
}).instrument(app.events, options)
|
|
317
|
+
}
|
|
318
|
+
```
|
|
319
|
+
|
|
320
|
+
**Hands for Bots** adapter: [handsforbots-roadmap.md](./handsforbots-roadmap.md).
|
|
321
|
+
|
|
322
|
+
---
|
|
323
|
+
|
|
324
|
+
## Public API target (facilitate adoption)
|
|
325
|
+
|
|
326
|
+
### Minimal (today)
|
|
327
|
+
|
|
328
|
+
```javascript
|
|
329
|
+
import { createObservability, instrumentEventBus, definePhaseModel } from '@handsforbots/semantic-event-observability'
|
|
330
|
+
|
|
331
|
+
const obs = createObservability({
|
|
332
|
+
turnStartEvents: ['user.message'],
|
|
333
|
+
turnEndEvents: ['bot.response'],
|
|
334
|
+
phases: definePhaseModel([
|
|
335
|
+
{ id: 'backend', startEvent: 'api.call', endEvent: 'api.done' },
|
|
336
|
+
]),
|
|
337
|
+
exporters: ['memory', 'console'],
|
|
338
|
+
})
|
|
339
|
+
instrumentEventBus(obs, myBus, { stateProvider: () => ({}) })
|
|
340
|
+
|
|
341
|
+
obs.startPhase('custom-work') // manual phase within active turn
|
|
342
|
+
obs.endPhase('custom-work')
|
|
343
|
+
```
|
|
344
|
+
|
|
345
|
+
### Target (after abstraction work)
|
|
346
|
+
|
|
347
|
+
```javascript
|
|
348
|
+
import {
|
|
349
|
+
createObservability,
|
|
350
|
+
defineTurnModel,
|
|
351
|
+
definePhaseModel,
|
|
352
|
+
instrumentEventBus,
|
|
353
|
+
withTraceContext,
|
|
354
|
+
} from '@handsforbots/semantic-event-observability'
|
|
355
|
+
|
|
356
|
+
const obs = createObservability({
|
|
357
|
+
turn: defineTurnModel({ start: ['user.message'], end: ['bot.response'] }),
|
|
358
|
+
phases: definePhaseModel([
|
|
359
|
+
{ id: 'backend', start: 'api.call', end: 'api.done' },
|
|
360
|
+
{ id: 'render', end: 'ui.ready' }, // render_ms after last phase end
|
|
361
|
+
]),
|
|
362
|
+
exporters: ['otel', 'faro'],
|
|
363
|
+
exporterConfig: { otel: { /* injected */ } },
|
|
364
|
+
})
|
|
365
|
+
|
|
366
|
+
instrumentEventBus(obs, bus, { stateProvider })
|
|
367
|
+
|
|
368
|
+
obs.record('custom.action', { key: 'value' })
|
|
369
|
+
obs.recordMetric('cart.add', 1, { sku: 'x' }) // host-named; document convention
|
|
370
|
+
|
|
371
|
+
await fetch('/api', withTraceContext(obs, { method: 'POST' }))
|
|
372
|
+
```
|
|
373
|
+
|
|
374
|
+
---
|
|
375
|
+
|
|
376
|
+
## Development phases
|
|
377
|
+
|
|
378
|
+
Status: `[ ]` not started · `[~]` partial · `[x]` done
|
|
379
|
+
|
|
380
|
+
### Phase A — Metrics foundation (P0)
|
|
381
|
+
|
|
382
|
+
| # | Item | Status |
|
|
383
|
+
|---|------|--------|
|
|
384
|
+
| A.1 | `MetricsRegistry` module | [x] |
|
|
385
|
+
| A.2 | Emit `sevo_turn_duration_ms`, `sevo_phase_duration_ms`, `sevo_events_dropped_total` | [x] |
|
|
386
|
+
| A.3 | Turn completion / abandonment counters | [x] |
|
|
387
|
+
| A.4 | OTel Metrics API export (replace span-per-metric) | [x] |
|
|
388
|
+
| A.5 | State gauge from `stateProvider` keys | [x] |
|
|
389
|
+
| A.6 | Grafana dashboard validation | [x] |
|
|
390
|
+
|
|
391
|
+
Details: [metrics-roadmap.md](./metrics-roadmap.md).
|
|
392
|
+
|
|
393
|
+
### Phase B — PhaseModel & TraceMapper (P1)
|
|
394
|
+
|
|
395
|
+
| # | Item | Status |
|
|
396
|
+
|---|------|--------|
|
|
397
|
+
| B.1 | `PhaseTracker` + `PhaseModel` config | [x] |
|
|
398
|
+
| B.2 | Extract `TraceMapper` from otel exporter | [x] |
|
|
399
|
+
| B.3 | Remove hardcoded `core.calling_backend` from lib core | [x] |
|
|
400
|
+
| B.4 | Manual `startPhase` / `endPhase` API | [x] |
|
|
401
|
+
| B.5 | `onPhaseEnd` exporter hook | [x] |
|
|
402
|
+
| B.6 | Pluggable error detection | [x] |
|
|
403
|
+
|
|
404
|
+
### Phase C — Correlation & instrumentation (P1)
|
|
405
|
+
|
|
406
|
+
| # | Item | Status |
|
|
407
|
+
|---|------|--------|
|
|
408
|
+
| C.1 | Formal `TurnModel` with abandonment | [x] |
|
|
409
|
+
| C.2 | `TraceContextBridge` + `withTraceContext` | [x] |
|
|
410
|
+
| C.3 | Generic `instrumentChannel` | [x] |
|
|
411
|
+
| C.4 | `eventFilter` / allowlist for bus | [x] |
|
|
412
|
+
| C.5 | Exporter error metrics | [x] |
|
|
413
|
+
|
|
414
|
+
### Phase D — DX & packaging (P2)
|
|
415
|
+
|
|
416
|
+
| # | Item | Status |
|
|
417
|
+
|---|------|--------|
|
|
418
|
+
| D.1 | TypeScript types or `.d.ts` for public API | [x] |
|
|
419
|
+
| D.2 | `defineTurnModel` / `definePhaseModel` helpers | [x] |
|
|
420
|
+
| D.3 | Custom exporter cookbook | [x] |
|
|
421
|
+
| D.4 | Framework-agnostic getting started (non-HfB) | [x] |
|
|
422
|
+
| D.5 | npm publish from `packageIdentity.js` | [~] |
|
|
423
|
+
| D.6 | Integration test harness (memory exporter) | [x] |
|
|
424
|
+
|
|
425
|
+
### Phase E — Industry depth (P2–P3)
|
|
426
|
+
|
|
427
|
+
| # | Item | Status |
|
|
428
|
+
|---|------|--------|
|
|
429
|
+
| E.1 | OTel `invoke_agent` span naming option on turn root | [x] |
|
|
430
|
+
| E.2 | Tail sampling guidance doc | [x] |
|
|
431
|
+
| E.3 | Langfuse/LangSmith phase-aware mapping | [x] |
|
|
432
|
+
| E.4 | Web Vitals bridge (optional exporter) | [x] |
|
|
433
|
+
| E.5 | Session-level rollup metrics (`sevo_session_turns_total`) | [x] |
|
|
434
|
+
|
|
435
|
+
---
|
|
436
|
+
|
|
437
|
+
## What hosts must still provide
|
|
438
|
+
|
|
439
|
+
| Need | Owner |
|
|
440
|
+
|------|-------|
|
|
441
|
+
| Domain event names | Host app |
|
|
442
|
+
| Phase presets for domain | Host adapter |
|
|
443
|
+
| `gen_ai.*` metrics | Backend / BFF |
|
|
444
|
+
| Product analytics (`hfb_*`) | Host plugins |
|
|
445
|
+
| Eval scores | Langfuse / custom |
|
|
446
|
+
| Collector auth & sampling | Ops / platform |
|
|
447
|
+
|
|
448
|
+
---
|
|
449
|
+
|
|
450
|
+
## Related documents
|
|
451
|
+
|
|
452
|
+
| Document | Scope |
|
|
453
|
+
|----------|-------|
|
|
454
|
+
| [metrics-roadmap.md](./metrics-roadmap.md) | `sevo_*` metrics checklist (lib only) |
|
|
455
|
+
| [handsforbots-roadmap.md](./handsforbots-roadmap.md) | Hands for Bots adapter & `hfb_*` metrics |
|
|
456
|
+
| [architecture.md](./architecture.md) | Current runtime design |
|
|
457
|
+
| [exporters.md](./exporters.md) | Exporter interface |
|
|
458
|
+
| [handsforbots-adapter.md](./handsforbots-adapter.md) | HfB integration guide |
|
|
459
|
+
|
|
460
|
+
---
|
|
461
|
+
|
|
462
|
+
## Changelog
|
|
463
|
+
|
|
464
|
+
| Date | Change |
|
|
465
|
+
|------|--------|
|
|
466
|
+
| 2026-06-15 | Initial library roadmap; split from combined metrics doc |
|
|
467
|
+
| 2026-06-15 | Phase B: PhaseTracker, TraceMapper, phase semantic events, manual API, onPhaseEnd hook |
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
# Tail sampling guidance
|
|
2
|
+
|
|
3
|
+
Semantic Event Observability applies **head sampling** in the browser via `Policy` (`sampleRate`, `maxEventsPerMinute`). For production conversational apps you typically combine that with **tail sampling** in the collector so error and slow turns are kept while routine traffic is dropped.
|
|
4
|
+
|
|
5
|
+
## Two layers
|
|
6
|
+
|
|
7
|
+
| Layer | Where | What it controls |
|
|
8
|
+
|-------|--------|------------------|
|
|
9
|
+
| **Head** | Lib `Policy` | Whether a semantic event is recorded at all |
|
|
10
|
+
| **Tail** | OTel Collector / Tempo | Which completed traces are exported to long-term storage |
|
|
11
|
+
|
|
12
|
+
Head sampling reduces browser and ingress cost. Tail sampling keeps debugging signal for turns that matter after you know latency or outcome.
|
|
13
|
+
|
|
14
|
+
## Lib defaults (always keep)
|
|
15
|
+
|
|
16
|
+
These event types bypass `sampleRate` random drop (still subject to `maxEventsPerMinute` unless disabled):
|
|
17
|
+
|
|
18
|
+
- `turn.*` — turn start, end, abandoned
|
|
19
|
+
- `phase.*` — phase start/end (backend, render, manual)
|
|
20
|
+
- `session.*` — session rollups
|
|
21
|
+
- `metric.*` — host and lib metrics
|
|
22
|
+
|
|
23
|
+
Bus events (`bus.trigger`) are sampled. That is intentional: high-volume UI noise should not fill Tempo.
|
|
24
|
+
|
|
25
|
+
## Recommended Policy settings
|
|
26
|
+
|
|
27
|
+
| Environment | `sampleRate` | `maxEventsPerMinute` | Notes |
|
|
28
|
+
|-------------|--------------|----------------------|-------|
|
|
29
|
+
| Local dev | `1` | `120+` | Full visibility |
|
|
30
|
+
| Staging | `0.25`–`1` | `60` | Validate tail rules |
|
|
31
|
+
| Production | `0.05`–`0.2` | `30`–`60` | Pair with collector tail sampling |
|
|
32
|
+
|
|
33
|
+
Turn and phase spans still reach exporters when turns occur; only non-critical bus events are throttled.
|
|
34
|
+
|
|
35
|
+
## OpenTelemetry Collector tail sampling
|
|
36
|
+
|
|
37
|
+
Route browser traces through an OTel Collector with a `tail_sampling` processor. Example policy sketch:
|
|
38
|
+
|
|
39
|
+
```yaml
|
|
40
|
+
processors:
|
|
41
|
+
tail_sampling:
|
|
42
|
+
decision_wait: 10s
|
|
43
|
+
num_traces: 100000
|
|
44
|
+
expected_new_traces_per_sec: 100
|
|
45
|
+
policies:
|
|
46
|
+
- name: errors
|
|
47
|
+
type: status_code
|
|
48
|
+
status_code:
|
|
49
|
+
status_codes: [ERROR]
|
|
50
|
+
- name: slow-turns
|
|
51
|
+
type: latency
|
|
52
|
+
latency:
|
|
53
|
+
threshold_ms: 5000
|
|
54
|
+
- name: abandoned-turns
|
|
55
|
+
type: string_attribute
|
|
56
|
+
string_attribute:
|
|
57
|
+
key: sevo.event_name
|
|
58
|
+
values: [turn.abandoned]
|
|
59
|
+
- name: baseline-sample
|
|
60
|
+
type: probabilistic
|
|
61
|
+
probabilistic:
|
|
62
|
+
sampling_percentage: 10
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
Adjust keys to match your exporter attributes (`sevo.turn_id`, `gen_ai.operation.name`, etc.).
|
|
66
|
+
|
|
67
|
+
### Hands for Bots–specific hints
|
|
68
|
+
|
|
69
|
+
| Signal | Attribute / span | Tail keep rule |
|
|
70
|
+
|--------|------------------|----------------|
|
|
71
|
+
| Slow user wait | `turn.duration_ms` on turn root | latency > SLO (e.g. 3s) |
|
|
72
|
+
| Backend stall | `phase:backend` duration | latency > backend SLO |
|
|
73
|
+
| Abandoned turn | `turn.abandoned` event / status | always |
|
|
74
|
+
| Error heuristic | `turn.error_event`, `error.type` | status ERROR |
|
|
75
|
+
|
|
76
|
+
## Grafana Tempo / Mimir
|
|
77
|
+
|
|
78
|
+
- **Tempo:** tail sampling happens at ingest (collector) or via Grafana Cloud sampling policies.
|
|
79
|
+
- **Metrics (`sevo_*`):** not trace-sampled — use Prometheus/Mimir retention and aggregation. Histograms remain statistically useful at low head sample rates for bus events; turn metrics are emitted per turn regardless of bus sample rate.
|
|
80
|
+
|
|
81
|
+
## Error-biased head sampling (optional host pattern)
|
|
82
|
+
|
|
83
|
+
The lib does not implement error-biased head sampling internally. Hosts can raise fidelity on errors by calling `observability.record()` for error paths or setting `sampleRate: 1` in staging.
|
|
84
|
+
|
|
85
|
+
For production, prefer **tail** keep rules for errors so happy-path traffic stays cheap.
|
|
86
|
+
|
|
87
|
+
## Checklist before go-live
|
|
88
|
+
|
|
89
|
+
- [ ] `sampleRate` < 1 in production plugin config
|
|
90
|
+
- [ ] Collector tail policy keeps ERROR and above-SLO turns
|
|
91
|
+
- [ ] Abandoned-turn rule validated in Tempo
|
|
92
|
+
- [ ] `sevo_events_dropped_total` dashboard panel for policy transparency
|
|
93
|
+
- [ ] Auth + rate limit on OTLP/Faro ingress (see [getting-started.md](./getting-started.md))
|
|
94
|
+
|
|
95
|
+
## Related
|
|
96
|
+
|
|
97
|
+
- [architecture.md](./architecture.md) — Policy and event flow
|
|
98
|
+
- [metrics-roadmap.md](./metrics-roadmap.md) — `sevo_events_dropped_total`
|
|
99
|
+
- [exporters.md](./exporters.md) — OTel and Faro export paths
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
export function createConsoleExporter(config = {}) {
|
|
2
|
+
const id = 'console'
|
|
3
|
+
const level = config.level || 'debug'
|
|
4
|
+
const prefix = config.prefix || '[sevo]'
|
|
5
|
+
|
|
6
|
+
return {
|
|
7
|
+
id,
|
|
8
|
+
available: true,
|
|
9
|
+
description: 'Structured console output for local debugging',
|
|
10
|
+
|
|
11
|
+
async init() {},
|
|
12
|
+
|
|
13
|
+
onEvent(event) {
|
|
14
|
+
const logger = console[level] || console.log
|
|
15
|
+
logger(`${prefix} ${event.type}`, {
|
|
16
|
+
name: event.name,
|
|
17
|
+
turnId: event.turnId,
|
|
18
|
+
traceId: event.traceId,
|
|
19
|
+
payload: event.payloadSummary,
|
|
20
|
+
state: event.state,
|
|
21
|
+
})
|
|
22
|
+
},
|
|
23
|
+
|
|
24
|
+
onMetric(metric) {
|
|
25
|
+
const logger = console[level] || console.log
|
|
26
|
+
logger(`${prefix} metric`, metric)
|
|
27
|
+
},
|
|
28
|
+
}
|
|
29
|
+
}
|