@handsforbots/semantic-event-observability 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/LICENSE.md +9 -0
  2. package/README.md +141 -2
  3. package/adapters/genericEventBus.js +20 -0
  4. package/adapters/instrumentChannel.js +22 -0
  5. package/core/CorrelationContext.js +49 -0
  6. package/core/EventBuffer.js +37 -0
  7. package/core/MetricsRegistry.js +200 -0
  8. package/core/PhaseTracker.js +143 -0
  9. package/core/Policy.js +87 -0
  10. package/core/SessionTracker.js +23 -0
  11. package/core/TraceContextBridge.js +74 -0
  12. package/core/TraceMapper.js +167 -0
  13. package/core/TurnMetricsCollector.js +19 -0
  14. package/core/TurnModel.js +87 -0
  15. package/core/baggage.d.ts +12 -0
  16. package/core/baggage.js +68 -0
  17. package/core/bucketEventName.js +11 -0
  18. package/core/createObservability.js +611 -0
  19. package/core/definePhaseModel.js +14 -0
  20. package/core/eventInstrumentation.js +21 -0
  21. package/core/isErrorEvent.js +9 -0
  22. package/core/telemetryAttributes.js +44 -0
  23. package/core/turnSpanNaming.js +73 -0
  24. package/docs/architecture.md +106 -0
  25. package/docs/custom-exporter.md +135 -0
  26. package/docs/exporters.md +180 -0
  27. package/docs/getting-started.md +192 -0
  28. package/docs/grafana-validation.md +71 -0
  29. package/docs/handsforbots-adapter.md +64 -0
  30. package/docs/handsforbots-roadmap.md +193 -0
  31. package/docs/metrics-roadmap.md +145 -0
  32. package/docs/publishing.md +55 -0
  33. package/docs/renaming.md +65 -0
  34. package/docs/roadmap.md +467 -0
  35. package/docs/tail-sampling.md +99 -0
  36. package/exporters/console.js +29 -0
  37. package/exporters/devPanel.js +204 -0
  38. package/exporters/faro.js +71 -0
  39. package/exporters/index.js +58 -0
  40. package/exporters/langfuse.js +161 -0
  41. package/exporters/langsmith.js +168 -0
  42. package/exporters/llmExporterState.js +84 -0
  43. package/exporters/memory.js +18 -0
  44. package/exporters/otel.js +236 -0
  45. package/exporters/otelTraceBackend.js +51 -0
  46. package/exporters/webVitals.js +73 -0
  47. package/grafana/semantic-event-observability.json +250 -0
  48. package/grafana/semantic-event-observability.lgtm.json +399 -0
  49. package/index.d.ts +312 -0
  50. package/index.js +31 -0
  51. package/package.json +80 -3
  52. package/packageIdentity.js +41 -0
  53. package/utils/id.js +10 -0
  54. package/utils/probeOptional.js +42 -0
  55. package/utils/safeJson.js +74 -0
@@ -0,0 +1,84 @@
1
+ /**
2
+ * Shared turn/phase tracking for Langfuse and LangSmith exporters.
3
+ */
4
+ export function createLlmExporterState() {
5
+ /** @type {Map<string, unknown>} */
6
+ const turns = new Map()
7
+ /** @type {Map<string, unknown>} */
8
+ const phases = new Map()
9
+
10
+ return {
11
+ setTurn(turnId, handle) {
12
+ if (turnId) turns.set(turnId, handle)
13
+ },
14
+
15
+ getTurn(turnId) {
16
+ return turnId ? turns.get(turnId) : undefined
17
+ },
18
+
19
+ deleteTurn(turnId) {
20
+ if (!turnId) return
21
+ turns.delete(turnId)
22
+ for (const key of [...phases.keys()]) {
23
+ if (key.startsWith(`${turnId}:`)) phases.delete(key)
24
+ }
25
+ },
26
+
27
+ setPhase(turnId, phaseId, handle) {
28
+ if (turnId && phaseId) phases.set(phaseKey(turnId, phaseId), handle)
29
+ },
30
+
31
+ getPhase(turnId, phaseId) {
32
+ if (!turnId || !phaseId) return undefined
33
+ return phases.get(phaseKey(turnId, phaseId))
34
+ },
35
+
36
+ deletePhase(turnId, phaseId) {
37
+ if (turnId && phaseId) phases.delete(phaseKey(turnId, phaseId))
38
+ },
39
+
40
+ endAllPhases(turnId, endFn) {
41
+ if (!turnId) return
42
+ for (const [key, handle] of phases.entries()) {
43
+ if (!key.startsWith(`${turnId}:`)) continue
44
+ try { endFn?.(handle) } catch { /* noop */ }
45
+ phases.delete(key)
46
+ }
47
+ },
48
+
49
+ clear() {
50
+ turns.clear()
51
+ phases.clear()
52
+ },
53
+ }
54
+ }
55
+
56
+ export function baseEventMetadata(event, config = {}) {
57
+ return {
58
+ sessionId: event.sessionId,
59
+ turnId: event.turnId,
60
+ traceId: event.traceId,
61
+ eventName: event.name,
62
+ environment: event.environment,
63
+ project: config.project,
64
+ phase: event.phase,
65
+ phaseSource: event.source,
66
+ }
67
+ }
68
+
69
+ function phaseKey(turnId, phaseId) {
70
+ return `${turnId}:${phaseId}`
71
+ }
72
+
73
+ /**
74
+ * End an observation/run handle if supported.
75
+ */
76
+ export function endLlmHandle(handle, payload = {}) {
77
+ if (!handle) return
78
+ if (payload.output) handle.update?.({ output: payload.output })
79
+ if (payload.outputs) {
80
+ handle.end?.(payload)
81
+ } else {
82
+ handle.end?.()
83
+ }
84
+ }
@@ -0,0 +1,18 @@
1
+ export function createMemoryExporter(config = {}) {
2
+ const id = 'memory'
3
+
4
+ return {
5
+ id,
6
+ available: true,
7
+ description: 'Ring buffer used by dev panel and debug export (buffer owned by core)',
8
+
9
+ async init(context) {
10
+ // Buffer is populated by createObservability; memory exporter is a marker for dev tooling.
11
+ void context.buffer
12
+ },
13
+
14
+ onEvent() {},
15
+
16
+ onMetric() {},
17
+ }
18
+ }
@@ -0,0 +1,236 @@
1
+ import { resolveOptionalDependency } from '../utils/probeOptional.js'
2
+ import { createTraceMapper } from '../core/TraceMapper.js'
3
+ import { SEVO_METRICS } from '../core/MetricsRegistry.js'
4
+ import { sevoMetricLabelAttributes } from '../core/telemetryAttributes.js'
5
+ import { createOtelSpanBackend } from './otelTraceBackend.js'
6
+
7
+ /**
8
+ * OpenTelemetry exporter (optional dependency).
9
+ * Trace spans are driven by semantic events via TraceMapper.
10
+ */
11
+ export function createOtelExporter(config = {}) {
12
+ const id = 'otel'
13
+ let tracer = null
14
+ let meter = null
15
+ let otelContext = null
16
+ let otelTrace = null
17
+ let otelPropagation = null
18
+ let SpanStatusCode = null
19
+ let SpanKind = null
20
+ let traceMapper = null
21
+ let isError = config.isError
22
+ /** @type {Map<string, import('@opentelemetry/api').Counter>} */
23
+ const counters = new Map()
24
+ /** @type {Map<string, import('@opentelemetry/api').Histogram>} */
25
+ const histograms = new Map()
26
+ /** @type {Map<string, import('@opentelemetry/api').Gauge>} */
27
+ const gauges = new Map()
28
+
29
+ if (typeof config.getTracer === 'function') {
30
+ tracer = config.getTracer('semantic-event-observability', '0.0.0')
31
+ }
32
+ if (typeof config.getMeter === 'function') {
33
+ meter = config.getMeter('semantic-event-observability', '0.0.0')
34
+ }
35
+ if (config.traceApi) {
36
+ otelContext = config.traceApi.context
37
+ otelTrace = config.traceApi.trace
38
+ otelPropagation = config.traceApi.propagation
39
+ SpanStatusCode = config.traceApi.SpanStatusCode
40
+ SpanKind = config.traceApi.SpanKind
41
+ }
42
+
43
+ return {
44
+ id,
45
+ available: false,
46
+ description: 'OpenTelemetry spans and metrics for Grafana Tempo/Mimir',
47
+
48
+ async init(context) {
49
+ if (typeof config.isError !== 'function' && typeof context.isError === 'function') {
50
+ isError = context.isError
51
+ }
52
+
53
+ const resolved = await resolveOptionalDependency(config.api, {
54
+ globalName: 'otel',
55
+ moduleSpecifier: '@opentelemetry/api',
56
+ })
57
+
58
+ if (typeof config.getTracer === 'function') {
59
+ tracer = config.getTracer(context.identity.otelServiceName, context.identity.version)
60
+ } else if (resolved) {
61
+ const traceApi = resolved.module.trace || resolved.module
62
+ if (typeof traceApi?.getTracer === 'function') {
63
+ tracer = traceApi.getTracer(context.identity.otelServiceName, context.identity.version)
64
+ }
65
+ }
66
+
67
+ if (typeof config.getMeter === 'function') {
68
+ meter = config.getMeter(context.identity.otelServiceName, context.identity.version)
69
+ } else if (resolved?.module?.metrics) {
70
+ const metricsApi = resolved.module.metrics
71
+ if (typeof metricsApi?.getMeter === 'function') {
72
+ meter = metricsApi.getMeter(context.identity.otelServiceName, context.identity.version)
73
+ }
74
+ }
75
+
76
+ if (config.traceApi) {
77
+ otelContext = config.traceApi.context
78
+ otelTrace = config.traceApi.trace
79
+ otelPropagation = config.traceApi.propagation
80
+ SpanStatusCode = config.traceApi.SpanStatusCode
81
+ SpanKind = config.traceApi.SpanKind
82
+ } else if (resolved?.module) {
83
+ otelContext = resolved.module.context
84
+ otelTrace = resolved.module.trace
85
+ otelPropagation = resolved.module.propagation
86
+ SpanStatusCode = resolved.module.SpanStatusCode
87
+ SpanKind = resolved.module.SpanKind
88
+ }
89
+
90
+ const turnRootSpan = config.turnRootSpan ?? context.turnRootSpan
91
+
92
+ const backend = createOtelSpanBackend({
93
+ tracer,
94
+ context: otelContext,
95
+ trace: otelTrace,
96
+ SpanStatusCode,
97
+ SpanKind,
98
+ propagation: otelPropagation,
99
+ })
100
+
101
+ if (backend) {
102
+ traceMapper = createTraceMapper({
103
+ backend,
104
+ isError,
105
+ turnRootSpan,
106
+ onTurnContext(_turnId, carrier) {
107
+ const bridge = context.traceContextBridge
108
+ if (!bridge) return
109
+ if (carrier) {
110
+ bridge.setHeaderInjector(() => carrier)
111
+ } else {
112
+ bridge.setHeaderInjector(null)
113
+ }
114
+ },
115
+ })
116
+ }
117
+
118
+ if (meter) {
119
+ initMetricInstruments(meter)
120
+ }
121
+
122
+ this.available = Boolean(traceMapper)
123
+ },
124
+
125
+ onEvent(event) {
126
+ traceMapper?.handleEvent(event)
127
+ },
128
+
129
+ onPhaseEnd(phaseEvent) {
130
+ void phaseEvent
131
+ },
132
+
133
+ onMetric(metric) {
134
+ recordOtelMetric(metric)
135
+ },
136
+
137
+ destroy() {
138
+ traceMapper?.destroy()
139
+ traceMapper = null
140
+ counters.clear()
141
+ histograms.clear()
142
+ gauges.clear()
143
+ },
144
+ }
145
+
146
+ function initMetricInstruments(activeMeter) {
147
+ // Names already end with _ms — omit unit so Prometheus gets *_bucket, not *_milliseconds_bucket.
148
+ histograms.set(SEVO_METRICS.TURN_DURATION, activeMeter.createHistogram(SEVO_METRICS.TURN_DURATION, {
149
+ description: 'Conversation turn duration (milliseconds)',
150
+ }))
151
+ histograms.set(SEVO_METRICS.PHASE_DURATION, activeMeter.createHistogram(SEVO_METRICS.PHASE_DURATION, {
152
+ description: 'Phase duration within a turn (milliseconds)',
153
+ }))
154
+ histograms.set(SEVO_METRICS.PHASE_WAIT, activeMeter.createHistogram(SEVO_METRICS.PHASE_WAIT, {
155
+ description: 'Wait from turn start to phase start (milliseconds)',
156
+ }))
157
+ counters.set(SEVO_METRICS.TURNS_TOTAL, activeMeter.createCounter(SEVO_METRICS.TURNS_TOTAL, {
158
+ description: 'Turn outcomes',
159
+ }))
160
+ counters.set(SEVO_METRICS.EVENTS_DROPPED, activeMeter.createCounter(SEVO_METRICS.EVENTS_DROPPED, {
161
+ description: 'Telemetry events dropped by policy',
162
+ }))
163
+ counters.set(SEVO_METRICS.EVENTS_EMITTED, activeMeter.createCounter(SEVO_METRICS.EVENTS_EMITTED, {
164
+ description: 'Semantic events emitted',
165
+ }))
166
+ counters.set(SEVO_METRICS.EXPORTER_ERRORS, activeMeter.createCounter(SEVO_METRICS.EXPORTER_ERRORS, {
167
+ description: 'Exporter handler errors',
168
+ }))
169
+ counters.set(SEVO_METRICS.SESSION_TURNS_TOTAL, activeMeter.createCounter(SEVO_METRICS.SESSION_TURNS_TOTAL, {
170
+ description: 'Turn outcomes rolled up at session boundary',
171
+ }))
172
+ counters.set(SEVO_METRICS.BUS_EVENTS_TOTAL, activeMeter.createCounter(SEVO_METRICS.BUS_EVENTS_TOTAL, {
173
+ description: 'Instrumented bus.trigger events',
174
+ }))
175
+ counters.set(SEVO_METRICS.CUSTOM_METRICS_TOTAL, activeMeter.createCounter(SEVO_METRICS.CUSTOM_METRICS_TOTAL, {
176
+ description: 'Allowlisted custom metrics via recordMetric',
177
+ }))
178
+ histograms.set(SEVO_METRICS.WEB_VITAL, activeMeter.createHistogram(SEVO_METRICS.WEB_VITAL, {
179
+ description: 'Core Web Vitals measurements (milliseconds for LCP/FCP/TTFB/INP; unitless for CLS)',
180
+ }))
181
+ histograms.set(SEVO_METRICS.LISTENER_DURATION, activeMeter.createHistogram(SEVO_METRICS.LISTENER_DURATION, {
182
+ description: 'Instrumented bus listener duration (milliseconds)',
183
+ }))
184
+
185
+ if (typeof activeMeter.createGauge === 'function') {
186
+ gauges.set(SEVO_METRICS.STATE_GAUGE, activeMeter.createGauge(SEVO_METRICS.STATE_GAUGE, {
187
+ description: 'Host state gauge from stateProvider',
188
+ }))
189
+ gauges.set(SEVO_METRICS.ACTIVE_TURNS, activeMeter.createGauge(SEVO_METRICS.ACTIVE_TURNS, {
190
+ description: 'Open conversation turns',
191
+ }))
192
+ }
193
+ }
194
+
195
+ function recordOtelMetric(metric) {
196
+ if (!metric) return
197
+
198
+ // recordCounter(): a counter of the app's own, created on first use, labels as given.
199
+ if (metric.custom && metric.type === 'counter') {
200
+ if (!meter) return
201
+ let counter = counters.get(metric.name)
202
+ if (!counter) {
203
+ counter = meter.createCounter(metric.name)
204
+ counters.set(metric.name, counter)
205
+ }
206
+ counter.add(metric.value ?? 1, plainAttributes(metric.labels))
207
+ return
208
+ }
209
+
210
+ const attributes = sevoMetricLabelAttributes(metric.labels)
211
+
212
+ if (metric.type === 'histogram') {
213
+ histograms.get(metric.name)?.record(metric.value, attributes)
214
+ return
215
+ }
216
+
217
+ if (metric.type === 'gauge') {
218
+ const gauge = gauges.get(metric.name)
219
+ gauge?.record(metric.value, attributes)
220
+ return
221
+ }
222
+
223
+ if (metric.type === 'counter') {
224
+ counters.get(metric.name)?.add(metric.value ?? 1, attributes)
225
+ }
226
+ }
227
+ }
228
+
229
+ function plainAttributes(labels = {}) {
230
+ const output = {}
231
+ for (const [key, value] of Object.entries(labels || {})) {
232
+ if (value == null) continue
233
+ output[key] = typeof value === 'number' || typeof value === 'boolean' ? value : String(value)
234
+ }
235
+ return output
236
+ }
@@ -0,0 +1,51 @@
1
+ /**
2
+ * OpenTelemetry span backend for TraceMapper.
3
+ */
4
+ export function createOtelSpanBackend({ tracer, context, trace, SpanStatusCode, SpanKind, propagation }) {
5
+ if (!tracer || !context || !trace) {
6
+ return null
7
+ }
8
+
9
+ return {
10
+ startSpan(name, attributes, parentSpan, options = {}) {
11
+ const spanOptions = { attributes }
12
+ const kind = resolveSpanKind(options.kind, SpanKind)
13
+ if (kind != null) spanOptions.kind = kind
14
+
15
+ const parentCtx = parentSpan
16
+ ? trace.setSpan(context.active(), parentSpan)
17
+ : undefined
18
+ return tracer.startSpan(name, spanOptions, parentCtx)
19
+ },
20
+
21
+ endSpan(spanRef) {
22
+ spanRef?.end()
23
+ },
24
+
25
+ setAttribute(spanRef, key, value) {
26
+ spanRef?.setAttribute(key, value)
27
+ },
28
+
29
+ setError(spanRef, message) {
30
+ if (!SpanStatusCode || !spanRef) return
31
+ spanRef.setStatus({ code: SpanStatusCode.ERROR, message })
32
+ },
33
+
34
+ getPropagationContext(spanRef) {
35
+ if (!spanRef || !propagation?.inject) return null
36
+ const activeContext = trace.setSpan(context.active(), spanRef)
37
+ const carrier = {}
38
+ propagation.inject(activeContext, carrier)
39
+ return carrier
40
+ },
41
+ }
42
+ }
43
+
44
+ function resolveSpanKind(kind, SpanKind) {
45
+ if (kind == null || !SpanKind) return undefined
46
+ if (typeof kind === 'number') return kind
47
+
48
+ const normalized = String(kind).toUpperCase()
49
+ if (SpanKind[normalized] != null) return SpanKind[normalized]
50
+ return undefined
51
+ }
@@ -0,0 +1,73 @@
1
+ import { resolveOptionalDependency } from '../utils/probeOptional.js'
2
+
3
+ const DEFAULT_VITALS = ['LCP', 'INP', 'CLS', 'FCP', 'TTFB']
4
+
5
+ /**
6
+ * Web Vitals exporter (optional dependency).
7
+ * Subscribes to web-vitals callbacks and records sevo_web_vital histograms.
8
+ */
9
+ export function createWebVitalsExporter(config = {}) {
10
+ const id = 'webVitals'
11
+ let metricsRegistry = null
12
+ let reportHandler = null
13
+ const subscribed = []
14
+
15
+ return {
16
+ id,
17
+ available: false,
18
+ description: 'Bridge Core Web Vitals into sevo_web_vital metrics',
19
+
20
+ async init(context) {
21
+ metricsRegistry = context.metricsRegistry
22
+ reportHandler = createReportHandler(config, metricsRegistry)
23
+
24
+ if (typeof config.onReport === 'function') {
25
+ config.onReport(reportHandler)
26
+ this.available = true
27
+ return
28
+ }
29
+
30
+ const resolved = await resolveOptionalDependency(config.vitals, {
31
+ moduleSpecifier: 'web-vitals',
32
+ })
33
+
34
+ if (!resolved?.module) return
35
+
36
+ const vitals = resolved.module
37
+ const names = config.metrics || DEFAULT_VITALS
38
+
39
+ for (const name of names) {
40
+ const register = vitals[`on${name}`]
41
+ if (typeof register !== 'function') continue
42
+ register(reportHandler)
43
+ subscribed.push(name)
44
+ }
45
+
46
+ this.available = subscribed.length > 0
47
+ },
48
+
49
+ onEvent() {},
50
+
51
+ onMetric() {},
52
+
53
+ destroy() {
54
+ subscribed.length = 0
55
+ },
56
+ }
57
+ }
58
+
59
+ function createReportHandler(config, metricsRegistry) {
60
+ return function reportWebVital(metric) {
61
+ if (!metric || metricsRegistry?.recordWebVital == null) return
62
+
63
+ const labels = {
64
+ rating: metric.rating,
65
+ navigationType: metric.navigationType,
66
+ ...(config.labels || {}),
67
+ }
68
+
69
+ metricsRegistry.recordWebVital(metric.name, metric.value, labels)
70
+
71
+ config.onVital?.(metric)
72
+ }
73
+ }