@handsforbots/semantic-event-observability 0.0.0-stage → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/LICENSE.md +9 -0
  2. package/README.md +141 -2
  3. package/adapters/genericEventBus.js +20 -0
  4. package/adapters/instrumentChannel.js +22 -0
  5. package/core/CorrelationContext.js +49 -0
  6. package/core/EventBuffer.js +37 -0
  7. package/core/MetricsRegistry.js +200 -0
  8. package/core/PhaseTracker.js +143 -0
  9. package/core/Policy.js +87 -0
  10. package/core/SessionTracker.js +23 -0
  11. package/core/TraceContextBridge.js +74 -0
  12. package/core/TraceMapper.js +167 -0
  13. package/core/TurnMetricsCollector.js +19 -0
  14. package/core/TurnModel.js +87 -0
  15. package/core/baggage.d.ts +12 -0
  16. package/core/baggage.js +68 -0
  17. package/core/bucketEventName.js +11 -0
  18. package/core/createObservability.js +611 -0
  19. package/core/definePhaseModel.js +14 -0
  20. package/core/eventInstrumentation.js +21 -0
  21. package/core/isErrorEvent.js +9 -0
  22. package/core/telemetryAttributes.js +44 -0
  23. package/core/turnSpanNaming.js +73 -0
  24. package/docs/architecture.md +106 -0
  25. package/docs/custom-exporter.md +135 -0
  26. package/docs/exporters.md +180 -0
  27. package/docs/getting-started.md +192 -0
  28. package/docs/grafana-validation.md +71 -0
  29. package/docs/handsforbots-adapter.md +64 -0
  30. package/docs/handsforbots-roadmap.md +193 -0
  31. package/docs/metrics-roadmap.md +145 -0
  32. package/docs/publishing.md +55 -0
  33. package/docs/renaming.md +65 -0
  34. package/docs/roadmap.md +467 -0
  35. package/docs/tail-sampling.md +99 -0
  36. package/exporters/console.js +29 -0
  37. package/exporters/devPanel.js +204 -0
  38. package/exporters/faro.js +71 -0
  39. package/exporters/index.js +58 -0
  40. package/exporters/langfuse.js +161 -0
  41. package/exporters/langsmith.js +168 -0
  42. package/exporters/llmExporterState.js +84 -0
  43. package/exporters/memory.js +18 -0
  44. package/exporters/otel.js +236 -0
  45. package/exporters/otelTraceBackend.js +51 -0
  46. package/exporters/webVitals.js +73 -0
  47. package/grafana/semantic-event-observability.json +250 -0
  48. package/grafana/semantic-event-observability.lgtm.json +399 -0
  49. package/index.d.ts +312 -0
  50. package/index.js +31 -0
  51. package/package.json +83 -3
  52. package/packageIdentity.js +41 -0
  53. package/utils/id.js +10 -0
  54. package/utils/probeOptional.js +42 -0
  55. package/utils/safeJson.js +74 -0
@@ -0,0 +1,467 @@
1
+ # Library roadmap
2
+
3
+ Development plan for **Semantic Event Observability** as a **standalone, host-agnostic** front-end library.
4
+
5
+ > Hands for Bots is one consumer. Its metrics and adapter-specific work live in [handsforbots-roadmap.md](./handsforbots-roadmap.md).
6
+
7
+ ---
8
+
9
+ ## Vision
10
+
11
+ Offer industry-grade **semantic observability for browser event-driven applications** — chatbots, agent UIs, wizards, games, multi-step flows — without coupling to a specific framework.
12
+
13
+ **Value proposition for the industry:**
14
+
15
+ | Problem | What we offer |
16
+ |---------|----------------|
17
+ | Event buses are invisible in APM | Automatic instrumentation of `on` / `trigger` with correlation |
18
+ | User “turns” span many async steps | Configurable turn boundaries + trace trees |
19
+ | Front-end latency is a black box | Phase timing (request → response → render) on the client |
20
+ | Telemetry must not break the app | Fail-safe exporters, optional peers, policy guardrails |
21
+ | Vendor lock-in | OTel, Faro, Langfuse, LangSmith — pluggable exporters |
22
+ | LLM observability ignores the UI layer | Client-side turn root span; backend joins via trace context |
23
+
24
+ **Non-goals (stay out of the lib):**
25
+
26
+ - Backend LLM SDK instrumentation (`gen_ai.*` — host responsibility)
27
+ - Product analytics schemas (`hfb_*` — host adapter responsibility)
28
+ - Storing or evaluating prompts at scale (delegate to Langfuse / eval platforms)
29
+ - **Service uptime / availability** — synthetic HTTP probes, `/health` polling, “site down” alerting (infra / deploy responsibility)
30
+ - **Platform health checks** — Kubernetes liveness/readiness, Docker healthchecks for app containers (not the telemetry collector)
31
+
32
+ ---
33
+
34
+ ## Boundary: library vs host
35
+
36
+ ```mermaid
37
+ flowchart TB
38
+ subgraph lib ["Semantic Event Observability"]
39
+ Turn["TurnModel"]
40
+ Phase["PhaseModel"]
41
+ Bus["BusInstrumenter"]
42
+ Metrics["MetricsRegistry"]
43
+ Trace["TraceMapper"]
44
+ Export["Exporter pipeline"]
45
+ end
46
+
47
+ subgraph host ["Host application e.g. Hands for Bots"]
48
+ Events["Domain events"]
49
+ State["StateProvider"]
50
+ Adapter["Thin adapter preset"]
51
+ Product["Product metrics hfb_*"]
52
+ Backend["Backend OTel / gen_ai.*"]
53
+ end
54
+
55
+ Events --> Adapter --> Bus
56
+ State --> Bus
57
+ Bus --> Turn & Phase
58
+ Turn & Phase --> Metrics & Trace
59
+ Metrics & Trace --> Export
60
+ Product -.->|recordMetric| Metrics
61
+ Backend -.->|traceparent| Trace
62
+ ```
63
+
64
+ | Concern | Library (`sevo_*`) | Host (`hfb_*`, backend) |
65
+ |---------|-------------------|-------------------------|
66
+ | Turn start / end detection | Configurable `TurnModel` | Preset event names |
67
+ | Phase timing (generic) | `PhaseModel` e.g. `fetch`, `render` | Map events → phase config |
68
+ | Bus instrumentation | `instrument()` | Choose which bus |
69
+ | Runtime state snapshot | `stateProvider` callback | Return queue depth, etc. |
70
+ | Metric names & labels | `sevo_*` only | Host prefix + labels |
71
+ | Voice / plugin / modality | — | Host cores & plugins |
72
+ | Token usage, LLM latency | Trace correlation hook | Backend instrumentation |
73
+ | Grafana dashboard template | Generic turn/phase panels | Host-specific panels optional |
74
+ | Uptime / “app is up” | — | Infra, synthetics, backend `/health` endpoints |
75
+ | Telemetry pipeline health | `sevo_exporter_errors_total`, policy drops | Collector uptime (LGTM / Alloy deploy) |
76
+
77
+ > **Scope detail:** [architecture.md](./architecture.md#scope) · Hands for Bots plugin: [handsforbots-adapter.md](./handsforbots-adapter.md#scope)
78
+
79
+ ---
80
+
81
+ ## Industry alignment (front-end focus)
82
+
83
+ Production front-end and agent-UI observability expects:
84
+
85
+ ### Three signals
86
+
87
+ | Signal | Library role |
88
+ |--------|----------------|
89
+ | **Traces** | One trace per turn; phase + event child spans |
90
+ | **Metrics** | Histograms/counters for turn duration, phases, drops, throughput |
91
+ | **Logs** | Semantic events → Faro/Loki (sampled bus traffic) |
92
+
93
+ ### Four measurement layers (client-owned portions)
94
+
95
+ | Layer | Industry examples | Library coverage |
96
+ |-------|-------------------|------------------|
97
+ | **Infrastructure** | Load time, JS errors, service uptime | Partial: telemetry health (`sevo_exporter_errors_total`), optional Web Vitals; **not** endpoint uptime or K8s probes |
98
+ | **Execution** | Stage latency breakdown | Turn + **PhaseModel** durations |
99
+ | **Reliability** | Completion rate, errors, queue pressure | Turn completion/abandonment; state gauge from provider |
100
+ | **Outcome** | Task success, user feedback | `record()` / exporter only — evals stay external |
101
+
102
+ ### Standards we align with
103
+
104
+ - **OpenTelemetry** — spans, metrics API, W3C trace context propagation
105
+ - **OTel GenAI conventions** — compatible trace IDs; backend children use `gen_ai.*` (not emitted by this lib)
106
+ - **Grafana Faro** — browser RUM + custom events
107
+ - **Langfuse / LangSmith** — turn-as-run mapping for LLM workflows
108
+
109
+ References: [OTel AI agent observability](https://opentelemetry.io/blog/2025/ai-agent-observability/), [GenAI metrics](https://hexdocs.pm/opentelemetry_semantic_conventions/gen-ai-metrics.html).
110
+
111
+ ---
112
+
113
+ ## Planned abstractions
114
+
115
+ Abstractions are ordered by dependency. **Bold** = new or needs extraction from current code.
116
+
117
+ ### 1. Core envelope (exists — stabilize API)
118
+
119
+ | Abstraction | Module | Responsibility |
120
+ |-------------|--------|----------------|
121
+ | `SemanticEvent` | `core/createObservability.js` | Canonical fields: `id`, `type`, `name`, `timestamp`, correlation ids, `payloadSummary`, `state`, `durationMs` |
122
+ | `CorrelationContext` | `core/CorrelationContext.js` | Session, turn, trace ids; observes bus events |
123
+ | `Policy` | `core/Policy.js` | Sample, rate-limit, redact; never sample `turn.*` / `metric.*` |
124
+ | `EventBuffer` | `core/EventBuffer.js` | Ring buffer for timeline + debug export |
125
+
126
+ **Roadmap:** Document schema in [architecture.md](./architecture.md); add TypeScript/JSDoc typedef export.
127
+
128
+ ---
129
+
130
+ ### 2. TurnModel (exists as options — formalize)
131
+
132
+ Configurable boundaries for a correlated user interaction (conversation turn, wizard step, game round).
133
+
134
+ ```javascript
135
+ const turnModel = {
136
+ startEvents: ['user.message'], // opens turn + new traceId
137
+ endEvents: ['ui.ready'], // closes turn + durationMs
138
+ onAbandoned: 'count', // 'count' | 'event' | 'ignore'
139
+ }
140
+ ```
141
+
142
+ | Method | Purpose |
143
+ |--------|---------|
144
+ | `observeBusEvent(name)` | Returns `turn.start` / `turn.end` / null |
145
+ | `getContext()` | Current `sessionId`, `turnId`, `traceId` |
146
+ | `getActiveTurnAge()` | For stuck-turn detection |
147
+
148
+ **Roadmap:**
149
+
150
+ - [ ] Extract `TurnModel` from inline `CorrelationContext` options
151
+ - [~] Abandoned turn detection (new turn before `endEvents`) — in `CorrelationContext`
152
+ - [ ] Optional manual `startTurn()` / `endTurn()` for non-bus flows
153
+ - [ ] Export preset factories: `createConversationTurnModel({ start, end })`
154
+
155
+ ---
156
+
157
+ ### 3. PhaseModel (new — replaces hardcoded backend phase)
158
+
159
+ Declarative phases within a turn. Today `exporters/otel.js` hardcodes `core.calling_backend` → `core.backend_responded`. This moves to configuration.
160
+
161
+ ```javascript
162
+ const phaseModel = [
163
+ {
164
+ id: 'remote',
165
+ startEvent: 'api.request',
166
+ endEvent: 'api.response',
167
+ metric: 'sevo_phase_duration_ms', // label phase=remote
168
+ spanName: 'phase:remote',
169
+ },
170
+ {
171
+ id: 'render',
172
+ startEvent: 'api.response', // implicit: wall-clock after previous phase
173
+ endEvent: 'ui.painted',
174
+ spanName: 'phase:render',
175
+ },
176
+ ]
177
+ ```
178
+
179
+ | Component | Responsibility |
180
+ |-----------|----------------|
181
+ | **`PhaseTracker`** | Subscribes to semantic events; starts/ends phases; computes durations |
182
+ | **`PhaseState`** | Per-turn open phases, timestamps |
183
+ | **Span hints** | Pass `phaseSpan` metadata to TraceMapper |
184
+
185
+ **Roadmap:**
186
+
187
+ - [x] `core/PhaseTracker.js` — event-driven phase timing + manual API
188
+ - [x] `core/PhaseModel.js` — validate + normalize phase config (`definePhaseModel`)
189
+ - [x] `exporters/otel.js` consumes `phase.start` / `phase.end` semantic events via `TraceMapper`
190
+ - [x] Manual API: `observability.startPhase(id)` / `endPhase(id)` for non-bus work
191
+ - [x] Default: zero phases (turn + events only) for minimal adopters
192
+
193
+ ---
194
+
195
+ ### 4. MetricsRegistry (new — single source of truth)
196
+
197
+ Central recording for `sevo_*` instruments. Exporters subscribe; no duplicate logic in OTel exporter.
198
+
199
+ ```javascript
200
+ // core/MetricsRegistry.js
201
+ const instruments = {
202
+ turnDuration: histogram('sevo_turn_duration_ms', { unit: 'ms' }),
203
+ phaseDuration: histogram('sevo_phase_duration_ms', { labels: ['phase'] }),
204
+ turnsTotal: counter('sevo_turns_total', { labels: ['status'] }),
205
+ eventsDropped: counter('sevo_events_dropped_total', { labels: ['reason'] }),
206
+ stateGauge: observableGauge('sevo_state_gauge', { labels: ['key'] }),
207
+ }
208
+ ```
209
+
210
+ | Rule | Detail |
211
+ |------|--------|
212
+ | Prefix | Always `sevo_` |
213
+ | Recording triggers | `turn.end`, phase end, `Policy.recordDrop()`, periodic state scrape |
214
+ | OTel export | Metrics API histograms/counters — **not** one span per metric |
215
+ | Host metrics | `recordMetric()` bypasses registry naming or uses `host.` prefix policy |
216
+
217
+ **Roadmap:** See [metrics-roadmap.md](./metrics-roadmap.md) checklist.
218
+
219
+ ---
220
+
221
+ ### 5. TraceMapper (extract from otel exporter)
222
+
223
+ Maps semantic events + phase state → OpenTelemetry span tree.
224
+
225
+ ```
226
+ turn:<startEvent> [root]
227
+ ├── phase:<id> [PhaseModel]
228
+ ├── event:<busName> [each bus.trigger]
229
+ └── attributes: turn.duration_ms, phase.*_ms
230
+ ```
231
+
232
+ | Concern | Approach |
233
+ |---------|----------|
234
+ | Error marking | Pluggable `isError(event)` — default heuristic, host override |
235
+ | Attributes | `sevo.*` namespace on spans |
236
+ | Active context | Set span context on turn root for child manual spans |
237
+
238
+ **Roadmap:**
239
+
240
+ - [x] `core/TraceMapper.js` — span lifecycle independent of OTel import
241
+ - [x] `exporters/otel.js` — thin adapter over TraceMapper + `otelTraceBackend.js`
242
+ - [x] Align span names with OTel incubating `invoke_agent` guidance where applicable
243
+
244
+ ---
245
+
246
+ ### 6. TraceContextBridge (new — backend correlation)
247
+
248
+ Enables distributed traces: browser turn root + server LLM/tool children.
249
+
250
+ ```javascript
251
+ import { getTraceHeaders, withTraceContext } from './core/TraceContextBridge.js'
252
+
253
+ fetch(url, withTraceContext(observability, { method: 'POST', body }))
254
+ // injects traceparent from active turn trace
255
+ ```
256
+
257
+ **Roadmap:**
258
+
259
+ - [x] Read active `traceId` / span context from turn root
260
+ - [x] `withTraceContext` / `withFetch` for outbound calls
261
+ - [x] Document host integration (no fetch patching inside lib by default)
262
+
263
+ ---
264
+
265
+ ### 7. BusInstrumenter (exists — extend)
266
+
267
+ | API | Status |
268
+ |-----|--------|
269
+ | `instrument(bus, { stateProvider, wrapListeners })` | Exists |
270
+ | `instrumentEventBus()` | Generic adapter |
271
+ | **`instrumentChannel(channel, options)`** | Generic BroadcastChannel / MessagePort helper |
272
+ | **`createInstrumentedBus()`** | Bootstrap helper |
273
+
274
+ **Roadmap:**
275
+
276
+ - [x] Extract channel instrumentation from HfB adapter into generic `instrumentChannel`
277
+ - [x] Optional `eventFilter(name) => boolean` to reduce noise
278
+ - [x] `eventAllowlist` mode for high-volume buses
279
+
280
+ ---
281
+
282
+ ### 8. Exporter pipeline (exists — extend)
283
+
284
+ ```javascript
285
+ interface Exporter {
286
+ id: string
287
+ available: boolean
288
+ init(context): Promise<void>
289
+ onEvent(event): void
290
+ onMetric(metric): void // registry reading
291
+ onPhaseEnd?(phase): void // optional
292
+ destroy(): void
293
+ }
294
+ ```
295
+
296
+ **Roadmap:**
297
+
298
+ - [x] `onPhaseEnd` for exporters that need phase granularity
299
+ - [ ] Shared `ExporterContext` with MetricsRegistry + TurnModel stats
300
+ - [ ] Custom exporter template in docs
301
+
302
+ ---
303
+
304
+ ### 9. Host adapter pattern (thin presets only)
305
+
306
+ Adapters **configure** the lib; they do not add core logic.
307
+
308
+ ```javascript
309
+ // adapters/my-app.js
310
+ export function attachMyAppObservability(app, options) {
311
+ return createObservability({
312
+ turnModel: { startEvents: ['app.submit'], endEvents: ['app.rendered'] },
313
+ phases: MY_PHASES,
314
+ stateProvider: () => ({ queue: app.queue.length }),
315
+ exporters: options.exporters,
316
+ }).instrument(app.events, options)
317
+ }
318
+ ```
319
+
320
+ **Hands for Bots** adapter: [handsforbots-roadmap.md](./handsforbots-roadmap.md).
321
+
322
+ ---
323
+
324
+ ## Public API target (facilitate adoption)
325
+
326
+ ### Minimal (today)
327
+
328
+ ```javascript
329
+ import { createObservability, instrumentEventBus, definePhaseModel } from '@handsforbots/semantic-event-observability'
330
+
331
+ const obs = createObservability({
332
+ turnStartEvents: ['user.message'],
333
+ turnEndEvents: ['bot.response'],
334
+ phases: definePhaseModel([
335
+ { id: 'backend', startEvent: 'api.call', endEvent: 'api.done' },
336
+ ]),
337
+ exporters: ['memory', 'console'],
338
+ })
339
+ instrumentEventBus(obs, myBus, { stateProvider: () => ({}) })
340
+
341
+ obs.startPhase('custom-work') // manual phase within active turn
342
+ obs.endPhase('custom-work')
343
+ ```
344
+
345
+ ### Target (after abstraction work)
346
+
347
+ ```javascript
348
+ import {
349
+ createObservability,
350
+ defineTurnModel,
351
+ definePhaseModel,
352
+ instrumentEventBus,
353
+ withTraceContext,
354
+ } from '@handsforbots/semantic-event-observability'
355
+
356
+ const obs = createObservability({
357
+ turn: defineTurnModel({ start: ['user.message'], end: ['bot.response'] }),
358
+ phases: definePhaseModel([
359
+ { id: 'backend', start: 'api.call', end: 'api.done' },
360
+ { id: 'render', end: 'ui.ready' }, // render_ms after last phase end
361
+ ]),
362
+ exporters: ['otel', 'faro'],
363
+ exporterConfig: { otel: { /* injected */ } },
364
+ })
365
+
366
+ instrumentEventBus(obs, bus, { stateProvider })
367
+
368
+ obs.record('custom.action', { key: 'value' })
369
+ obs.recordMetric('cart.add', 1, { sku: 'x' }) // host-named; document convention
370
+
371
+ await fetch('/api', withTraceContext(obs, { method: 'POST' }))
372
+ ```
373
+
374
+ ---
375
+
376
+ ## Development phases
377
+
378
+ Status: `[ ]` not started · `[~]` partial · `[x]` done
379
+
380
+ ### Phase A — Metrics foundation (P0)
381
+
382
+ | # | Item | Status |
383
+ |---|------|--------|
384
+ | A.1 | `MetricsRegistry` module | [x] |
385
+ | A.2 | Emit `sevo_turn_duration_ms`, `sevo_phase_duration_ms`, `sevo_events_dropped_total` | [x] |
386
+ | A.3 | Turn completion / abandonment counters | [x] |
387
+ | A.4 | OTel Metrics API export (replace span-per-metric) | [x] |
388
+ | A.5 | State gauge from `stateProvider` keys | [x] |
389
+ | A.6 | Grafana dashboard validation | [x] |
390
+
391
+ Details: [metrics-roadmap.md](./metrics-roadmap.md).
392
+
393
+ ### Phase B — PhaseModel & TraceMapper (P1)
394
+
395
+ | # | Item | Status |
396
+ |---|------|--------|
397
+ | B.1 | `PhaseTracker` + `PhaseModel` config | [x] |
398
+ | B.2 | Extract `TraceMapper` from otel exporter | [x] |
399
+ | B.3 | Remove hardcoded `core.calling_backend` from lib core | [x] |
400
+ | B.4 | Manual `startPhase` / `endPhase` API | [x] |
401
+ | B.5 | `onPhaseEnd` exporter hook | [x] |
402
+ | B.6 | Pluggable error detection | [x] |
403
+
404
+ ### Phase C — Correlation & instrumentation (P1)
405
+
406
+ | # | Item | Status |
407
+ |---|------|--------|
408
+ | C.1 | Formal `TurnModel` with abandonment | [x] |
409
+ | C.2 | `TraceContextBridge` + `withTraceContext` | [x] |
410
+ | C.3 | Generic `instrumentChannel` | [x] |
411
+ | C.4 | `eventFilter` / allowlist for bus | [x] |
412
+ | C.5 | Exporter error metrics | [x] |
413
+
414
+ ### Phase D — DX & packaging (P2)
415
+
416
+ | # | Item | Status |
417
+ |---|------|--------|
418
+ | D.1 | TypeScript types or `.d.ts` for public API | [x] |
419
+ | D.2 | `defineTurnModel` / `definePhaseModel` helpers | [x] |
420
+ | D.3 | Custom exporter cookbook | [x] |
421
+ | D.4 | Framework-agnostic getting started (non-HfB) | [x] |
422
+ | D.5 | npm publish from `packageIdentity.js` | [~] |
423
+ | D.6 | Integration test harness (memory exporter) | [x] |
424
+
425
+ ### Phase E — Industry depth (P2–P3)
426
+
427
+ | # | Item | Status |
428
+ |---|------|--------|
429
+ | E.1 | OTel `invoke_agent` span naming option on turn root | [x] |
430
+ | E.2 | Tail sampling guidance doc | [x] |
431
+ | E.3 | Langfuse/LangSmith phase-aware mapping | [x] |
432
+ | E.4 | Web Vitals bridge (optional exporter) | [x] |
433
+ | E.5 | Session-level rollup metrics (`sevo_session_turns_total`) | [x] |
434
+
435
+ ---
436
+
437
+ ## What hosts must still provide
438
+
439
+ | Need | Owner |
440
+ |------|-------|
441
+ | Domain event names | Host app |
442
+ | Phase presets for domain | Host adapter |
443
+ | `gen_ai.*` metrics | Backend / BFF |
444
+ | Product analytics (`hfb_*`) | Host plugins |
445
+ | Eval scores | Langfuse / custom |
446
+ | Collector auth & sampling | Ops / platform |
447
+
448
+ ---
449
+
450
+ ## Related documents
451
+
452
+ | Document | Scope |
453
+ |----------|-------|
454
+ | [metrics-roadmap.md](./metrics-roadmap.md) | `sevo_*` metrics checklist (lib only) |
455
+ | [handsforbots-roadmap.md](./handsforbots-roadmap.md) | Hands for Bots adapter & `hfb_*` metrics |
456
+ | [architecture.md](./architecture.md) | Current runtime design |
457
+ | [exporters.md](./exporters.md) | Exporter interface |
458
+ | [handsforbots-adapter.md](./handsforbots-adapter.md) | HfB integration guide |
459
+
460
+ ---
461
+
462
+ ## Changelog
463
+
464
+ | Date | Change |
465
+ |------|--------|
466
+ | 2026-06-15 | Initial library roadmap; split from combined metrics doc |
467
+ | 2026-06-15 | Phase B: PhaseTracker, TraceMapper, phase semantic events, manual API, onPhaseEnd hook |
@@ -0,0 +1,99 @@
1
+ # Tail sampling guidance
2
+
3
+ Semantic Event Observability applies **head sampling** in the browser via `Policy` (`sampleRate`, `maxEventsPerMinute`). For production conversational apps you typically combine that with **tail sampling** in the collector so error and slow turns are kept while routine traffic is dropped.
4
+
5
+ ## Two layers
6
+
7
+ | Layer | Where | What it controls |
8
+ |-------|--------|------------------|
9
+ | **Head** | Lib `Policy` | Whether a semantic event is recorded at all |
10
+ | **Tail** | OTel Collector / Tempo | Which completed traces are exported to long-term storage |
11
+
12
+ Head sampling reduces browser and ingress cost. Tail sampling keeps debugging signal for turns that matter after you know latency or outcome.
13
+
14
+ ## Lib defaults (always keep)
15
+
16
+ These event types bypass `sampleRate` random drop (still subject to `maxEventsPerMinute` unless disabled):
17
+
18
+ - `turn.*` — turn start, end, abandoned
19
+ - `phase.*` — phase start/end (backend, render, manual)
20
+ - `session.*` — session rollups
21
+ - `metric.*` — host and lib metrics
22
+
23
+ Bus events (`bus.trigger`) are sampled. That is intentional: high-volume UI noise should not fill Tempo.
24
+
25
+ ## Recommended Policy settings
26
+
27
+ | Environment | `sampleRate` | `maxEventsPerMinute` | Notes |
28
+ |-------------|--------------|----------------------|-------|
29
+ | Local dev | `1` | `120+` | Full visibility |
30
+ | Staging | `0.25`–`1` | `60` | Validate tail rules |
31
+ | Production | `0.05`–`0.2` | `30`–`60` | Pair with collector tail sampling |
32
+
33
+ Turn and phase spans still reach exporters when turns occur; only non-critical bus events are throttled.
34
+
35
+ ## OpenTelemetry Collector tail sampling
36
+
37
+ Route browser traces through an OTel Collector with a `tail_sampling` processor. Example policy sketch:
38
+
39
+ ```yaml
40
+ processors:
41
+ tail_sampling:
42
+ decision_wait: 10s
43
+ num_traces: 100000
44
+ expected_new_traces_per_sec: 100
45
+ policies:
46
+ - name: errors
47
+ type: status_code
48
+ status_code:
49
+ status_codes: [ERROR]
50
+ - name: slow-turns
51
+ type: latency
52
+ latency:
53
+ threshold_ms: 5000
54
+ - name: abandoned-turns
55
+ type: string_attribute
56
+ string_attribute:
57
+ key: sevo.event_name
58
+ values: [turn.abandoned]
59
+ - name: baseline-sample
60
+ type: probabilistic
61
+ probabilistic:
62
+ sampling_percentage: 10
63
+ ```
64
+
65
+ Adjust keys to match your exporter attributes (`sevo.turn_id`, `gen_ai.operation.name`, etc.).
66
+
67
+ ### Hands for Bots–specific hints
68
+
69
+ | Signal | Attribute / span | Tail keep rule |
70
+ |--------|------------------|----------------|
71
+ | Slow user wait | `turn.duration_ms` on turn root | latency > SLO (e.g. 3s) |
72
+ | Backend stall | `phase:backend` duration | latency > backend SLO |
73
+ | Abandoned turn | `turn.abandoned` event / status | always |
74
+ | Error heuristic | `turn.error_event`, `error.type` | status ERROR |
75
+
76
+ ## Grafana Tempo / Mimir
77
+
78
+ - **Tempo:** tail sampling happens at ingest (collector) or via Grafana Cloud sampling policies.
79
+ - **Metrics (`sevo_*`):** not trace-sampled — use Prometheus/Mimir retention and aggregation. Histograms remain statistically useful at low head sample rates for bus events; turn metrics are emitted per turn regardless of bus sample rate.
80
+
81
+ ## Error-biased head sampling (optional host pattern)
82
+
83
+ The lib does not implement error-biased head sampling internally. Hosts can raise fidelity on errors by calling `observability.record()` for error paths or setting `sampleRate: 1` in staging.
84
+
85
+ For production, prefer **tail** keep rules for errors so happy-path traffic stays cheap.
86
+
87
+ ## Checklist before go-live
88
+
89
+ - [ ] `sampleRate` < 1 in production plugin config
90
+ - [ ] Collector tail policy keeps ERROR and above-SLO turns
91
+ - [ ] Abandoned-turn rule validated in Tempo
92
+ - [ ] `sevo_events_dropped_total` dashboard panel for policy transparency
93
+ - [ ] Auth + rate limit on OTLP/Faro ingress (see [getting-started.md](./getting-started.md))
94
+
95
+ ## Related
96
+
97
+ - [architecture.md](./architecture.md) — Policy and event flow
98
+ - [metrics-roadmap.md](./metrics-roadmap.md) — `sevo_events_dropped_total`
99
+ - [exporters.md](./exporters.md) — OTel and Faro export paths
@@ -0,0 +1,29 @@
1
+ export function createConsoleExporter(config = {}) {
2
+ const id = 'console'
3
+ const level = config.level || 'debug'
4
+ const prefix = config.prefix || '[sevo]'
5
+
6
+ return {
7
+ id,
8
+ available: true,
9
+ description: 'Structured console output for local debugging',
10
+
11
+ async init() {},
12
+
13
+ onEvent(event) {
14
+ const logger = console[level] || console.log
15
+ logger(`${prefix} ${event.type}`, {
16
+ name: event.name,
17
+ turnId: event.turnId,
18
+ traceId: event.traceId,
19
+ payload: event.payloadSummary,
20
+ state: event.state,
21
+ })
22
+ },
23
+
24
+ onMetric(metric) {
25
+ const logger = console[level] || console.log
26
+ logger(`${prefix} metric`, metric)
27
+ },
28
+ }
29
+ }