@handsforbots/semantic-event-observability 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +9 -0
- package/README.md +141 -2
- package/adapters/genericEventBus.js +20 -0
- package/adapters/instrumentChannel.js +22 -0
- package/core/CorrelationContext.js +49 -0
- package/core/EventBuffer.js +37 -0
- package/core/MetricsRegistry.js +200 -0
- package/core/PhaseTracker.js +143 -0
- package/core/Policy.js +87 -0
- package/core/SessionTracker.js +23 -0
- package/core/TraceContextBridge.js +74 -0
- package/core/TraceMapper.js +167 -0
- package/core/TurnMetricsCollector.js +19 -0
- package/core/TurnModel.js +87 -0
- package/core/baggage.d.ts +12 -0
- package/core/baggage.js +68 -0
- package/core/bucketEventName.js +11 -0
- package/core/createObservability.js +611 -0
- package/core/definePhaseModel.js +14 -0
- package/core/eventInstrumentation.js +21 -0
- package/core/isErrorEvent.js +9 -0
- package/core/telemetryAttributes.js +44 -0
- package/core/turnSpanNaming.js +73 -0
- package/docs/architecture.md +106 -0
- package/docs/custom-exporter.md +135 -0
- package/docs/exporters.md +180 -0
- package/docs/getting-started.md +192 -0
- package/docs/grafana-validation.md +71 -0
- package/docs/handsforbots-adapter.md +64 -0
- package/docs/handsforbots-roadmap.md +193 -0
- package/docs/metrics-roadmap.md +145 -0
- package/docs/publishing.md +55 -0
- package/docs/renaming.md +65 -0
- package/docs/roadmap.md +467 -0
- package/docs/tail-sampling.md +99 -0
- package/exporters/console.js +29 -0
- package/exporters/devPanel.js +204 -0
- package/exporters/faro.js +71 -0
- package/exporters/index.js +58 -0
- package/exporters/langfuse.js +161 -0
- package/exporters/langsmith.js +168 -0
- package/exporters/llmExporterState.js +84 -0
- package/exporters/memory.js +18 -0
- package/exporters/otel.js +236 -0
- package/exporters/otelTraceBackend.js +51 -0
- package/exporters/webVitals.js +73 -0
- package/grafana/semantic-event-observability.json +250 -0
- package/grafana/semantic-event-observability.lgtm.json +399 -0
- package/index.d.ts +312 -0
- package/index.js +31 -0
- package/package.json +83 -3
- package/packageIdentity.js +41 -0
- package/utils/id.js +10 -0
- package/utils/probeOptional.js +42 -0
- package/utils/safeJson.js +74 -0
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
import { getPackageIdentity } from '../packageIdentity.js'
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Lightweight in-browser panel for quick triage.
|
|
5
|
+
* Full analysis remains in Grafana.
|
|
6
|
+
*/
|
|
7
|
+
export function createDevPanelExporter(config = {}) {
|
|
8
|
+
const id = 'devPanel'
|
|
9
|
+
const identity = getPackageIdentity(config.identity)
|
|
10
|
+
let panel = null
|
|
11
|
+
let host = null
|
|
12
|
+
let contextRef = null
|
|
13
|
+
|
|
14
|
+
return {
|
|
15
|
+
id,
|
|
16
|
+
available: typeof document !== 'undefined',
|
|
17
|
+
description: 'Optional floating panel with recent semantic events',
|
|
18
|
+
|
|
19
|
+
async init(context) {
|
|
20
|
+
contextRef = context
|
|
21
|
+
if (!this.available) return
|
|
22
|
+
if (config.enabled === false) return
|
|
23
|
+
if (!shouldEnablePanel(identity, config)) return
|
|
24
|
+
mountPanel(identity)
|
|
25
|
+
},
|
|
26
|
+
|
|
27
|
+
onEvent(event) {
|
|
28
|
+
if (!panel) return
|
|
29
|
+
appendEventRow(panel.list, event)
|
|
30
|
+
},
|
|
31
|
+
|
|
32
|
+
onMetric(metric) {
|
|
33
|
+
if (!panel) return
|
|
34
|
+
appendMetricRow(panel.metrics, metric)
|
|
35
|
+
},
|
|
36
|
+
|
|
37
|
+
destroy() {
|
|
38
|
+
host?.remove()
|
|
39
|
+
panel = null
|
|
40
|
+
host = null
|
|
41
|
+
},
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function mountPanel(identity) {
|
|
45
|
+
host = document.createElement('div')
|
|
46
|
+
host.setAttribute('data-seo-devpanel', identity.slug)
|
|
47
|
+
host.setAttribute('data-seo-minimized', 'false')
|
|
48
|
+
host.style.cssText = [
|
|
49
|
+
'position:fixed',
|
|
50
|
+
'bottom:12px',
|
|
51
|
+
'right:12px',
|
|
52
|
+
'width:360px',
|
|
53
|
+
'max-width:min(360px, calc(100vw - 24px))',
|
|
54
|
+
'max-height:45vh',
|
|
55
|
+
'background:#111',
|
|
56
|
+
'color:#eee',
|
|
57
|
+
'font:12px/1.4 monospace',
|
|
58
|
+
'border:1px solid #444',
|
|
59
|
+
'border-radius:8px',
|
|
60
|
+
'z-index:99999',
|
|
61
|
+
'overflow:hidden',
|
|
62
|
+
'box-shadow:0 8px 24px rgba(0,0,0,.35)',
|
|
63
|
+
].join(';')
|
|
64
|
+
|
|
65
|
+
const header = document.createElement('div')
|
|
66
|
+
header.style.cssText = 'display:flex;align-items:center;justify-content:space-between;gap:8px;padding:8px 10px;background:#1b1b1b;border-bottom:1px solid #333'
|
|
67
|
+
|
|
68
|
+
const title = document.createElement('strong')
|
|
69
|
+
title.textContent = identity.displayName
|
|
70
|
+
title.style.cssText = 'min-width:0'
|
|
71
|
+
|
|
72
|
+
const actions = document.createElement('div')
|
|
73
|
+
actions.style.cssText = 'display:flex;flex:none;gap:6px'
|
|
74
|
+
const exportButton = actionButton('export', 'export', 'Export')
|
|
75
|
+
const minimizeButton = actionButton('minimize', '–', 'Minimize')
|
|
76
|
+
minimizeButton.setAttribute('aria-expanded', 'true')
|
|
77
|
+
const closeButton = actionButton('close', '×', 'Close')
|
|
78
|
+
actions.append(exportButton, minimizeButton, closeButton)
|
|
79
|
+
header.append(title, actions)
|
|
80
|
+
|
|
81
|
+
const body = document.createElement('div')
|
|
82
|
+
body.setAttribute('data-seo-body', '')
|
|
83
|
+
|
|
84
|
+
const metrics = document.createElement('div')
|
|
85
|
+
metrics.setAttribute('data-seo-metrics', '')
|
|
86
|
+
metrics.style.cssText = 'padding:6px 10px;border-bottom:1px solid #222;color:#9fd'
|
|
87
|
+
|
|
88
|
+
const list = document.createElement('div')
|
|
89
|
+
list.setAttribute('data-seo-list', '')
|
|
90
|
+
list.style.cssText = 'overflow:auto;max-height:calc(45vh - 88px);padding:6px 10px'
|
|
91
|
+
|
|
92
|
+
body.append(metrics, list)
|
|
93
|
+
host.append(header, body)
|
|
94
|
+
|
|
95
|
+
panel = { metrics, list }
|
|
96
|
+
|
|
97
|
+
minimizeButton.addEventListener('click', () => {
|
|
98
|
+
const minimized = host.getAttribute('data-seo-minimized') !== 'true'
|
|
99
|
+
applyMinimized(host, minimized)
|
|
100
|
+
rememberMinimized(identity, minimized)
|
|
101
|
+
})
|
|
102
|
+
|
|
103
|
+
closeButton.addEventListener('click', () => {
|
|
104
|
+
localStorage.setItem(identity.debugStorageKey, 'false')
|
|
105
|
+
host.remove()
|
|
106
|
+
panel = null
|
|
107
|
+
})
|
|
108
|
+
|
|
109
|
+
exportButton.addEventListener('click', () => {
|
|
110
|
+
const bundle = {
|
|
111
|
+
identity,
|
|
112
|
+
exportedAt: new Date().toISOString(),
|
|
113
|
+
environment: clientEnvironment(),
|
|
114
|
+
timeline: contextRef?.getTimeline?.() || [],
|
|
115
|
+
metrics: contextRef?.getMetrics?.() || [],
|
|
116
|
+
policy: contextRef?.getPolicyStats?.() || {},
|
|
117
|
+
}
|
|
118
|
+
downloadJson(`${identity.slug}-debug.json`, bundle)
|
|
119
|
+
})
|
|
120
|
+
|
|
121
|
+
if (panelStartsMinimized(identity)) applyMinimized(host, true)
|
|
122
|
+
|
|
123
|
+
document.body.appendChild(host)
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
function actionButton(action, label, name) {
|
|
128
|
+
const button = document.createElement('button')
|
|
129
|
+
button.setAttribute('type', 'button')
|
|
130
|
+
button.setAttribute('data-seo-action', action)
|
|
131
|
+
button.setAttribute('aria-label', name)
|
|
132
|
+
button.title = name
|
|
133
|
+
button.textContent = label
|
|
134
|
+
return button
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/** Collapse to the title bar, or open the event list again. */
|
|
138
|
+
function applyMinimized(host, minimized) {
|
|
139
|
+
host.setAttribute('data-seo-minimized', minimized ? 'true' : 'false')
|
|
140
|
+
const body = host.querySelector('[data-seo-body]')
|
|
141
|
+
if (body) body.hidden = minimized
|
|
142
|
+
host.style.width = minimized ? 'max-content' : '360px'
|
|
143
|
+
host.style.maxHeight = minimized ? 'none' : '45vh'
|
|
144
|
+
const button = host.querySelector('[data-seo-action="minimize"]')
|
|
145
|
+
if (!button) return
|
|
146
|
+
button.textContent = minimized ? '+' : '–'
|
|
147
|
+
button.setAttribute('aria-expanded', minimized ? 'false' : 'true')
|
|
148
|
+
button.setAttribute('aria-label', minimized ? 'Restore' : 'Minimize')
|
|
149
|
+
button.title = minimized ? 'Restore' : 'Minimize'
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function panelStartsMinimized(identity) {
|
|
153
|
+
if (typeof localStorage === 'undefined') return false
|
|
154
|
+
return localStorage.getItem(identity.panelStorageKey) === 'minimized'
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function rememberMinimized(identity, minimized) {
|
|
158
|
+
if (typeof localStorage === 'undefined') return
|
|
159
|
+
if (minimized) localStorage.setItem(identity.panelStorageKey, 'minimized')
|
|
160
|
+
else localStorage.removeItem(identity.panelStorageKey)
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/** The browser the bundle came from: many bugs live in one engine only. */
|
|
164
|
+
export function clientEnvironment() {
|
|
165
|
+
const nav = typeof navigator !== 'undefined' ? navigator : undefined
|
|
166
|
+
return {
|
|
167
|
+
userAgent: nav?.userAgent,
|
|
168
|
+
language: nav?.language,
|
|
169
|
+
platform: nav?.userAgentData?.platform ?? nav?.platform,
|
|
170
|
+
viewport: typeof window !== 'undefined' ? `${window.innerWidth}x${window.innerHeight}` : undefined,
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
function shouldEnablePanel(identity, config) {
|
|
175
|
+
if (config.enabled === true) return true
|
|
176
|
+
if (typeof localStorage !== 'undefined') {
|
|
177
|
+
return localStorage.getItem(identity.debugStorageKey) === 'true'
|
|
178
|
+
}
|
|
179
|
+
return false
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
function appendEventRow(container, event) {
|
|
183
|
+
const row = document.createElement('div')
|
|
184
|
+
row.style.marginBottom = '6px'
|
|
185
|
+
row.textContent = `${event.timestamp} ${event.type} ${event.name || ''} turn=${event.turnId || '-'}`
|
|
186
|
+
container.prepend(row)
|
|
187
|
+
while (container.childElementCount > 80) {
|
|
188
|
+
container.lastChild.remove()
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
function appendMetricRow(container, metric) {
|
|
193
|
+
container.textContent = `${metric.name}=${metric.value} ${JSON.stringify(metric.labels || {})}`
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
function downloadJson(filename, data) {
|
|
197
|
+
const blob = new Blob([JSON.stringify(data, null, 2)], { type: 'application/json' })
|
|
198
|
+
const url = URL.createObjectURL(blob)
|
|
199
|
+
const anchor = document.createElement('a')
|
|
200
|
+
anchor.href = url
|
|
201
|
+
anchor.download = filename
|
|
202
|
+
anchor.click()
|
|
203
|
+
URL.revokeObjectURL(url)
|
|
204
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { resolveOptionalDependency } from '../utils/probeOptional.js'
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Grafana Faro exporter (optional dependency).
|
|
5
|
+
* Sends semantic events as Faro custom events/logs.
|
|
6
|
+
*/
|
|
7
|
+
export function createFaroExporter(config = {}) {
|
|
8
|
+
const id = 'faro'
|
|
9
|
+
let faro = null
|
|
10
|
+
|
|
11
|
+
return {
|
|
12
|
+
id,
|
|
13
|
+
available: false,
|
|
14
|
+
description: 'Grafana Faro custom events for Loki/Grafana dashboards',
|
|
15
|
+
|
|
16
|
+
async init() {
|
|
17
|
+
if (config.client?.api) {
|
|
18
|
+
faro = config.client
|
|
19
|
+
this.available = true
|
|
20
|
+
return
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
const resolved = await resolveOptionalDependency(config.client, {
|
|
24
|
+
globalName: 'faro',
|
|
25
|
+
moduleSpecifier: '@grafana/faro-web-sdk',
|
|
26
|
+
})
|
|
27
|
+
|
|
28
|
+
faro = resolved?.module?.faro || resolved?.module?.default || resolved?.module
|
|
29
|
+
if (!faro?.api?.pushEvent && !faro?.api?.pushLog) return
|
|
30
|
+
this.available = true
|
|
31
|
+
},
|
|
32
|
+
|
|
33
|
+
onEvent(event) {
|
|
34
|
+
if (!faro?.api) return
|
|
35
|
+
|
|
36
|
+
const payload = compactEvent(event)
|
|
37
|
+
|
|
38
|
+
if (typeof faro.api.pushEvent === 'function') {
|
|
39
|
+
faro.api.pushEvent('semantic_event', payload)
|
|
40
|
+
return
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
if (typeof faro.api.pushLog === 'function') {
|
|
44
|
+
faro.api.pushLog([JSON.stringify(payload)])
|
|
45
|
+
}
|
|
46
|
+
},
|
|
47
|
+
|
|
48
|
+
onMetric(metric) {
|
|
49
|
+
if (!faro?.api?.pushMeasurement) return
|
|
50
|
+
faro.api.pushMeasurement({
|
|
51
|
+
type: 'sevo_metric',
|
|
52
|
+
values: { [metric.name]: metric.value },
|
|
53
|
+
labels: metric.labels || {},
|
|
54
|
+
})
|
|
55
|
+
},
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function compactEvent(event) {
|
|
60
|
+
return {
|
|
61
|
+
type: event.type,
|
|
62
|
+
name: event.name,
|
|
63
|
+
sessionId: event.sessionId,
|
|
64
|
+
turnId: event.turnId,
|
|
65
|
+
traceId: event.traceId,
|
|
66
|
+
durationMs: event.durationMs,
|
|
67
|
+
payloadPreview: event.payloadSummary?.preview,
|
|
68
|
+
queueDepth: event.state?.queueDepth,
|
|
69
|
+
callingBackend: event.state?.callingBackend,
|
|
70
|
+
}
|
|
71
|
+
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { createMemoryExporter } from './memory.js'
|
|
2
|
+
import { createConsoleExporter } from './console.js'
|
|
3
|
+
import { createDevPanelExporter } from './devPanel.js'
|
|
4
|
+
import { createOtelExporter } from './otel.js'
|
|
5
|
+
import { createFaroExporter } from './faro.js'
|
|
6
|
+
import { createLangfuseExporter } from './langfuse.js'
|
|
7
|
+
import { createLangsmithExporter } from './langsmith.js'
|
|
8
|
+
import { createWebVitalsExporter } from './webVitals.js'
|
|
9
|
+
|
|
10
|
+
export const BUILTIN_EXPORTERS = {
|
|
11
|
+
memory: createMemoryExporter,
|
|
12
|
+
console: createConsoleExporter,
|
|
13
|
+
devPanel: createDevPanelExporter,
|
|
14
|
+
otel: createOtelExporter,
|
|
15
|
+
faro: createFaroExporter,
|
|
16
|
+
langfuse: createLangfuseExporter,
|
|
17
|
+
langsmith: createLangsmithExporter,
|
|
18
|
+
webVitals: createWebVitalsExporter,
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export async function createExporters(requested = [], config = {}) {
|
|
22
|
+
const exporters = []
|
|
23
|
+
const unique = [...new Set(requested)]
|
|
24
|
+
|
|
25
|
+
for (const name of unique) {
|
|
26
|
+
const factory = BUILTIN_EXPORTERS[name]
|
|
27
|
+
if (!factory) continue
|
|
28
|
+
|
|
29
|
+
const exporterConfig = config[name] || {}
|
|
30
|
+
const exporter = factory(exporterConfig)
|
|
31
|
+
exporters.push(exporter)
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
return exporters
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export async function initExporters(exporters, context) {
|
|
38
|
+
const status = []
|
|
39
|
+
|
|
40
|
+
for (const exporter of exporters) {
|
|
41
|
+
try {
|
|
42
|
+
await exporter.init?.(context)
|
|
43
|
+
status.push({
|
|
44
|
+
id: exporter.id,
|
|
45
|
+
available: exporter.available !== false,
|
|
46
|
+
description: exporter.description,
|
|
47
|
+
})
|
|
48
|
+
} catch (error) {
|
|
49
|
+
status.push({
|
|
50
|
+
id: exporter.id,
|
|
51
|
+
available: false,
|
|
52
|
+
error: error?.message || String(error),
|
|
53
|
+
})
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
return status
|
|
58
|
+
}
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
import { resolveOptionalDependency } from '../utils/probeOptional.js'
|
|
2
|
+
import {
|
|
3
|
+
baseEventMetadata,
|
|
4
|
+
createLlmExporterState,
|
|
5
|
+
endLlmHandle,
|
|
6
|
+
} from './llmExporterState.js'
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Langfuse exporter (optional dependency).
|
|
10
|
+
* Uses @langfuse/tracing startObservation when available,
|
|
11
|
+
* or injected client from config.
|
|
12
|
+
*/
|
|
13
|
+
export function createLangfuseExporter(config = {}) {
|
|
14
|
+
const id = 'langfuse'
|
|
15
|
+
let startObservation = null
|
|
16
|
+
const state = createLlmExporterState()
|
|
17
|
+
let exporterConfig = {}
|
|
18
|
+
|
|
19
|
+
return {
|
|
20
|
+
id,
|
|
21
|
+
available: false,
|
|
22
|
+
description: 'Langfuse traces/events for LLM-centric observability',
|
|
23
|
+
|
|
24
|
+
async init() {
|
|
25
|
+
exporterConfig = config
|
|
26
|
+
const resolved = await resolveOptionalDependency(config.tracing, {
|
|
27
|
+
moduleSpecifier: '@langfuse/tracing',
|
|
28
|
+
})
|
|
29
|
+
|
|
30
|
+
startObservation = config.startObservation || resolved?.module?.startObservation
|
|
31
|
+
if (typeof startObservation !== 'function') return
|
|
32
|
+
|
|
33
|
+
this.available = true
|
|
34
|
+
},
|
|
35
|
+
|
|
36
|
+
onEvent(event) {
|
|
37
|
+
if (!startObservation) return
|
|
38
|
+
|
|
39
|
+
switch (event.type) {
|
|
40
|
+
case 'turn.start':
|
|
41
|
+
handleTurnStart(event)
|
|
42
|
+
break
|
|
43
|
+
case 'turn.end':
|
|
44
|
+
handleTurnEnd(event)
|
|
45
|
+
break
|
|
46
|
+
case 'turn.abandoned':
|
|
47
|
+
handleTurnAbandoned(event)
|
|
48
|
+
break
|
|
49
|
+
case 'phase.start':
|
|
50
|
+
handlePhaseStart(event)
|
|
51
|
+
break
|
|
52
|
+
case 'phase.end':
|
|
53
|
+
handlePhaseEnd(event)
|
|
54
|
+
break
|
|
55
|
+
default:
|
|
56
|
+
handleBusEvent(event)
|
|
57
|
+
}
|
|
58
|
+
},
|
|
59
|
+
|
|
60
|
+
onPhaseEnd(event) {
|
|
61
|
+
handlePhaseEnd(event)
|
|
62
|
+
},
|
|
63
|
+
|
|
64
|
+
onMetric(metric) {
|
|
65
|
+
if (!startObservation) return
|
|
66
|
+
startObservation(`metric:${metric.name}`, {
|
|
67
|
+
input: { value: metric.value, labels: metric.labels || {} },
|
|
68
|
+
metadata: { source: 'semantic-event-observability' },
|
|
69
|
+
}, { asType: 'event' })
|
|
70
|
+
},
|
|
71
|
+
|
|
72
|
+
destroy() {
|
|
73
|
+
state.clear()
|
|
74
|
+
},
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function handleTurnStart(event) {
|
|
78
|
+
const observation = startObservation(`turn:${event.name || 'conversation'}`, {
|
|
79
|
+
input: { event: event.name },
|
|
80
|
+
metadata: baseEventMetadata(event, exporterConfig),
|
|
81
|
+
}, { asType: 'span' })
|
|
82
|
+
state.setTurn(event.turnId, observation)
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function handleTurnEnd(event) {
|
|
86
|
+
state.endAllPhases(event.turnId, (phase) => {
|
|
87
|
+
endLlmHandle(phase, { output: { status: 'open_on_turn_end' } })
|
|
88
|
+
})
|
|
89
|
+
|
|
90
|
+
const observation = state.getTurn(event.turnId)
|
|
91
|
+
endLlmHandle(observation, {
|
|
92
|
+
output: { durationMs: event.durationMs },
|
|
93
|
+
})
|
|
94
|
+
state.deleteTurn(event.turnId)
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function handleTurnAbandoned(event) {
|
|
98
|
+
state.endAllPhases(event.turnId, (phase) => {
|
|
99
|
+
endLlmHandle(phase, { output: { status: 'abandoned' } })
|
|
100
|
+
})
|
|
101
|
+
|
|
102
|
+
const observation = state.getTurn(event.turnId)
|
|
103
|
+
endLlmHandle(observation, { output: { status: 'abandoned' } })
|
|
104
|
+
state.deleteTurn(event.turnId)
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function handlePhaseStart(event) {
|
|
108
|
+
const turn = state.getTurn(event.turnId)
|
|
109
|
+
if (!turn || !event.phase) return
|
|
110
|
+
|
|
111
|
+
const metadata = baseEventMetadata(event, exporterConfig)
|
|
112
|
+
const body = {
|
|
113
|
+
input: { phase: event.phase, source: event.source },
|
|
114
|
+
metadata,
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
let phaseObservation
|
|
118
|
+
if (typeof turn.startObservation === 'function') {
|
|
119
|
+
phaseObservation = turn.startObservation(`phase:${event.phase}`, body, { asType: 'span' })
|
|
120
|
+
} else {
|
|
121
|
+
phaseObservation = startObservation(`phase:${event.phase}`, {
|
|
122
|
+
...body,
|
|
123
|
+
metadata: { ...metadata, parentTurnId: event.turnId },
|
|
124
|
+
}, { asType: 'span' })
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
state.setPhase(event.turnId, event.phase, phaseObservation)
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
function handlePhaseEnd(event) {
|
|
131
|
+
const phaseObservation = state.getPhase(event.turnId, event.phase)
|
|
132
|
+
if (!phaseObservation) return
|
|
133
|
+
|
|
134
|
+
endLlmHandle(phaseObservation, {
|
|
135
|
+
output: {
|
|
136
|
+
durationMs: event.durationMs,
|
|
137
|
+
source: event.source,
|
|
138
|
+
},
|
|
139
|
+
})
|
|
140
|
+
state.deletePhase(event.turnId, event.phase)
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
function handleBusEvent(event) {
|
|
144
|
+
if (event.type !== 'bus.trigger') return
|
|
145
|
+
|
|
146
|
+
const turn = event.turnId ? state.getTurn(event.turnId) : null
|
|
147
|
+
const metadata = baseEventMetadata(event, exporterConfig)
|
|
148
|
+
const body = {
|
|
149
|
+
input: { preview: event.payloadSummary?.preview },
|
|
150
|
+
metadata,
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
if (turn && typeof turn.startObservation === 'function') {
|
|
154
|
+
const child = turn.startObservation(`event:${event.name}`, body, { asType: 'event' })
|
|
155
|
+
endLlmHandle(child)
|
|
156
|
+
return
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
startObservation(`event:${event.name}`, body, { asType: 'event' })
|
|
160
|
+
}
|
|
161
|
+
}
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
import { resolveOptionalDependency } from '../utils/probeOptional.js'
|
|
2
|
+
import {
|
|
3
|
+
baseEventMetadata,
|
|
4
|
+
createLlmExporterState,
|
|
5
|
+
endLlmHandle,
|
|
6
|
+
} from './llmExporterState.js'
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* LangSmith exporter (optional dependency).
|
|
10
|
+
* Uses RunTree API when langsmith package is present or injected.
|
|
11
|
+
*/
|
|
12
|
+
export function createLangsmithExporter(config = {}) {
|
|
13
|
+
const id = 'langsmith'
|
|
14
|
+
let RunTree = null
|
|
15
|
+
const state = createLlmExporterState()
|
|
16
|
+
let exporterConfig = {}
|
|
17
|
+
let projectName = 'semantic-event-observability'
|
|
18
|
+
|
|
19
|
+
return {
|
|
20
|
+
id,
|
|
21
|
+
available: false,
|
|
22
|
+
description: 'LangSmith runs for LLM workflow debugging',
|
|
23
|
+
|
|
24
|
+
async init(context) {
|
|
25
|
+
exporterConfig = config
|
|
26
|
+
const resolved = await resolveOptionalDependency(config.client, {
|
|
27
|
+
moduleSpecifier: 'langsmith',
|
|
28
|
+
})
|
|
29
|
+
|
|
30
|
+
RunTree = config.RunTree || resolved?.module?.RunTree
|
|
31
|
+
if (typeof RunTree !== 'function') return
|
|
32
|
+
|
|
33
|
+
projectName = config.projectName || context.identity.slug
|
|
34
|
+
this.available = true
|
|
35
|
+
this.projectName = projectName
|
|
36
|
+
},
|
|
37
|
+
|
|
38
|
+
onEvent(event) {
|
|
39
|
+
if (!RunTree) return
|
|
40
|
+
|
|
41
|
+
switch (event.type) {
|
|
42
|
+
case 'turn.start':
|
|
43
|
+
handleTurnStart(event)
|
|
44
|
+
break
|
|
45
|
+
case 'turn.end':
|
|
46
|
+
handleTurnEnd(event)
|
|
47
|
+
break
|
|
48
|
+
case 'turn.abandoned':
|
|
49
|
+
handleTurnAbandoned(event)
|
|
50
|
+
break
|
|
51
|
+
case 'phase.start':
|
|
52
|
+
handlePhaseStart(event)
|
|
53
|
+
break
|
|
54
|
+
case 'phase.end':
|
|
55
|
+
handlePhaseEnd(event)
|
|
56
|
+
break
|
|
57
|
+
default:
|
|
58
|
+
handleBusEvent(event)
|
|
59
|
+
}
|
|
60
|
+
},
|
|
61
|
+
|
|
62
|
+
onPhaseEnd(event) {
|
|
63
|
+
handlePhaseEnd(event)
|
|
64
|
+
},
|
|
65
|
+
|
|
66
|
+
onMetric(metric) {
|
|
67
|
+
if (!RunTree) return
|
|
68
|
+
const run = new RunTree({
|
|
69
|
+
name: `metric:${metric.name}`,
|
|
70
|
+
run_type: 'tool',
|
|
71
|
+
project_name: projectName,
|
|
72
|
+
inputs: { value: metric.value, labels: metric.labels || {} },
|
|
73
|
+
})
|
|
74
|
+
run.postRun()
|
|
75
|
+
run.end()
|
|
76
|
+
},
|
|
77
|
+
|
|
78
|
+
destroy() {
|
|
79
|
+
state.clear()
|
|
80
|
+
},
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function handleTurnStart(event) {
|
|
84
|
+
const run = new RunTree({
|
|
85
|
+
name: `turn:${event.name || 'conversation'}`,
|
|
86
|
+
run_type: 'chain',
|
|
87
|
+
project_name: projectName,
|
|
88
|
+
inputs: { event: event.name },
|
|
89
|
+
extra: langsmithExtra(event, exporterConfig),
|
|
90
|
+
})
|
|
91
|
+
run.postRun()
|
|
92
|
+
state.setTurn(event.turnId, run)
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function handleTurnEnd(event) {
|
|
96
|
+
state.endAllPhases(event.turnId, (phaseRun) => {
|
|
97
|
+
endLlmHandle(phaseRun, { outputs: { status: 'open_on_turn_end' } })
|
|
98
|
+
})
|
|
99
|
+
|
|
100
|
+
const run = state.getTurn(event.turnId)
|
|
101
|
+
endLlmHandle(run, { outputs: { durationMs: event.durationMs } })
|
|
102
|
+
state.deleteTurn(event.turnId)
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function handleTurnAbandoned(event) {
|
|
106
|
+
state.endAllPhases(event.turnId, (phaseRun) => {
|
|
107
|
+
endLlmHandle(phaseRun, { outputs: { status: 'abandoned' } })
|
|
108
|
+
})
|
|
109
|
+
|
|
110
|
+
const run = state.getTurn(event.turnId)
|
|
111
|
+
endLlmHandle(run, { outputs: { status: 'abandoned' } })
|
|
112
|
+
state.deleteTurn(event.turnId)
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function handlePhaseStart(event) {
|
|
116
|
+
const parent = state.getTurn(event.turnId)
|
|
117
|
+
if (!parent || !event.phase) return
|
|
118
|
+
|
|
119
|
+
const phaseRun = new RunTree({
|
|
120
|
+
name: `phase:${event.phase}`,
|
|
121
|
+
run_type: 'chain',
|
|
122
|
+
project_name: projectName,
|
|
123
|
+
inputs: { phase: event.phase, source: event.source },
|
|
124
|
+
parent_run: parent,
|
|
125
|
+
extra: langsmithExtra(event, exporterConfig),
|
|
126
|
+
})
|
|
127
|
+
phaseRun.postRun()
|
|
128
|
+
state.setPhase(event.turnId, event.phase, phaseRun)
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function handlePhaseEnd(event) {
|
|
132
|
+
const phaseRun = state.getPhase(event.turnId, event.phase)
|
|
133
|
+
if (!phaseRun) return
|
|
134
|
+
|
|
135
|
+
endLlmHandle(phaseRun, {
|
|
136
|
+
outputs: {
|
|
137
|
+
durationMs: event.durationMs,
|
|
138
|
+
source: event.source,
|
|
139
|
+
},
|
|
140
|
+
})
|
|
141
|
+
state.deletePhase(event.turnId, event.phase)
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function handleBusEvent(event) {
|
|
145
|
+
if (event.type !== 'bus.trigger') return
|
|
146
|
+
|
|
147
|
+
const parent = event.turnId ? state.getTurn(event.turnId) : null
|
|
148
|
+
const child = new RunTree({
|
|
149
|
+
name: `event:${event.name}`,
|
|
150
|
+
run_type: 'tool',
|
|
151
|
+
project_name: projectName,
|
|
152
|
+
inputs: { preview: event.payloadSummary?.preview },
|
|
153
|
+
parent_run: parent || undefined,
|
|
154
|
+
extra: langsmithExtra(event, exporterConfig),
|
|
155
|
+
})
|
|
156
|
+
child.postRun()
|
|
157
|
+
child.end()
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function langsmithExtra(event, config = {}) {
|
|
162
|
+
return {
|
|
163
|
+
metadata: {
|
|
164
|
+
...baseEventMetadata(event, config),
|
|
165
|
+
tags: config.tags || [],
|
|
166
|
+
},
|
|
167
|
+
}
|
|
168
|
+
}
|