@lmnr-ai/lmnr 0.8.49 → 0.8.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.cjs +19 -14
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.d.cts +2 -2
- package/dist/cli.d.mts +2 -2
- package/dist/cli.mjs +16 -11
- package/dist/cli.mjs.map +1 -1
- package/dist/{evaluations-4bQ4UuTj.d.mts → evaluations-BVaPPf0j.d.mts} +1679 -1305
- package/dist/{evaluations-CbHnDCAY.d.cts → evaluations-C8cI_XIE.d.cts} +1682 -1307
- package/dist/{file-utils-Ct8PiWkV.mjs → file-utils-CVTUiqFU.mjs} +47 -18
- package/dist/file-utils-CVTUiqFU.mjs.map +1 -0
- package/dist/{file-utils-N65ihgSU.cjs → file-utils-Cyx-2kqE.cjs} +47 -18
- package/dist/file-utils-Cyx-2kqE.cjs.map +1 -0
- package/dist/index.cjs +280 -431
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +6938 -4632
- package/dist/index.d.mts +6938 -4632
- package/dist/index.mjs +280 -431
- package/dist/index.mjs.map +1 -1
- package/dist/opentelemetry-lib/instrumentation/temporal/workflow-interceptors.cjs +114 -102
- package/dist/opentelemetry-lib/instrumentation/temporal/workflow-interceptors.cjs.map +1 -1
- package/dist/opentelemetry-lib/instrumentation/temporal/workflow-interceptors.d.cts +776 -372
- package/dist/opentelemetry-lib/instrumentation/temporal/workflow-interceptors.d.mts +776 -372
- package/dist/opentelemetry-lib/instrumentation/temporal/workflow-interceptors.mjs +114 -102
- package/dist/opentelemetry-lib/instrumentation/temporal/workflow-interceptors.mjs.map +1 -1
- package/dist/{rolldown-runtime-2rV9d50f.mjs → rolldown-runtime-B4iAMlE-.mjs} +2 -2
- package/dist/{rolldown-runtime-CVvi-lCc.cjs → rolldown-runtime-Ctb5zv51.cjs} +1 -1
- package/dist/{utils-CbvnF9B9.mjs → utils-Ch0R_9XE.mjs} +9 -8
- package/dist/{utils-CbvnF9B9.mjs.map → utils-Ch0R_9XE.mjs.map} +1 -1
- package/dist/{utils-D8kgcFxb.cjs → utils-Cq24YsD_.cjs} +10 -9
- package/dist/utils-Cq24YsD_.cjs.map +1 -0
- package/package.json +15 -17
- package/dist/file-utils-Ct8PiWkV.mjs.map +0 -1
- package/dist/file-utils-N65ihgSU.cjs.map +0 -1
- package/dist/utils-D8kgcFxb.cjs.map +0 -1
package/dist/index.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { r as __require } from "./rolldown-runtime-
|
|
2
|
-
import { A as SPAN_INSTRUMENTATION_SOURCE, B as LaminarClient, C as PARENT_SPAN_IDS_PATH, D as SPAN_INPUT, E as SPAN_IDS_PATH, F as SPAN_TYPE, H as DEBUG_SESSION_FILE, I as TRACE_HAS_BROWSER_SESSION, L as TRACE_TYPE, M as SPAN_OUTPUT, N as SPAN_PATH, O as SPAN_INSTRUMENTATION_SCOPE_NAME, P as SPAN_SDK_VERSION, R as USER_ID, S as LaminarAttributes, T as SESSION_ID, U as TracingLevel, V as DEBUG_SESSION_DIR, W as errorMessage, _ as tryToOtelSpanContext, a as getFrontendUrl, b as ASSOCIATION_PROPERTIES_OVERRIDES, c as loadEnv, d as normalizeOtelSpanId, f as normalizeOtelTraceId, g as truncateSpanPayload, h as parseOtelHeaders, j as SPAN_LANGUAGE_VERSION, k as SPAN_INSTRUMENTATION_SCOPE_VERSION, l as metadataToAttributes, m as otelTraceIdToUUID, n as Semaphore, o as getOtelEnvVar, p as otelSpanIdToUUID, r as deserializeLaminarSpanContext, s as initializeLogger, t as NIL_UUID, u as newUUID, v as validateTracingConfig, w as PARENT_SPAN_PATH, x as HUMAN_EVALUATOR_OPTIONS, y as ASSOCIATION_PROPERTIES, z as version } from "./utils-
|
|
1
|
+
import { r as __require } from "./rolldown-runtime-B4iAMlE-.mjs";
|
|
2
|
+
import { A as SPAN_INSTRUMENTATION_SOURCE, B as LaminarClient, C as PARENT_SPAN_IDS_PATH, D as SPAN_INPUT, E as SPAN_IDS_PATH, F as SPAN_TYPE, H as DEBUG_SESSION_FILE, I as TRACE_HAS_BROWSER_SESSION, L as TRACE_TYPE, M as SPAN_OUTPUT, N as SPAN_PATH, O as SPAN_INSTRUMENTATION_SCOPE_NAME, P as SPAN_SDK_VERSION, R as USER_ID, S as LaminarAttributes, T as SESSION_ID, U as TracingLevel, V as DEBUG_SESSION_DIR, W as errorMessage, _ as tryToOtelSpanContext, a as getFrontendUrl, b as ASSOCIATION_PROPERTIES_OVERRIDES, c as loadEnv, d as normalizeOtelSpanId, f as normalizeOtelTraceId, g as truncateSpanPayload, h as parseOtelHeaders, j as SPAN_LANGUAGE_VERSION, k as SPAN_INSTRUMENTATION_SCOPE_VERSION, l as metadataToAttributes, m as otelTraceIdToUUID, n as Semaphore, o as getOtelEnvVar, p as otelSpanIdToUUID, r as deserializeLaminarSpanContext, s as initializeLogger, t as NIL_UUID, u as newUUID, v as validateTracingConfig, w as PARENT_SPAN_PATH, x as HUMAN_EVALUATOR_OPTIONS, y as ASSOCIATION_PROPERTIES, z as version } from "./utils-Ch0R_9XE.mjs";
|
|
3
3
|
import { n as TRACEPARENT_HEADER, t as LAMINAR_SPAN_CONTEXT_HEADER } from "./consts-CTTu3jhI.mjs";
|
|
4
4
|
import * as path from "path";
|
|
5
5
|
import { DiagConsoleLogger, DiagLogLevel, ROOT_CONTEXT, SpanKind, SpanStatusCode, TraceFlags, context, createContextKey, diag, isSpanContextValid, trace } from "@opentelemetry/api";
|
|
@@ -176,7 +176,7 @@ var LaminarDataset = class extends EvaluationDataset {
|
|
|
176
176
|
*/
|
|
177
177
|
async push(paths, recursive = false) {
|
|
178
178
|
if (!this.client) throw new Error("Client not set");
|
|
179
|
-
const { loadFromPaths } = await import("./file-utils-
|
|
179
|
+
const { loadFromPaths } = await import("./file-utils-CVTUiqFU.mjs").then((n) => n.t);
|
|
180
180
|
const data = await loadFromPaths(Array.isArray(paths) ? paths : [paths], recursive);
|
|
181
181
|
if (data.length === 0) {
|
|
182
182
|
console.warn("No data to push. Skipping");
|
|
@@ -562,7 +562,8 @@ const handleStreamResult = (originalResult, streamInfo, span, ignoreOutput, seri
|
|
|
562
562
|
});
|
|
563
563
|
trackStreamProcessing(async () => {
|
|
564
564
|
try {
|
|
565
|
-
const
|
|
565
|
+
const tempResponse = new Response(stream2);
|
|
566
|
+
const { chunks, error } = await consumeResponse(tempResponse);
|
|
566
567
|
if (shouldSendTraces() && !ignoreOutput) try {
|
|
567
568
|
const output = {
|
|
568
569
|
type: "response",
|
|
@@ -661,7 +662,7 @@ const writeDebugSessionFile = (file, dir = process.cwd()) => {
|
|
|
661
662
|
* logic line-comparable with the Python `config.py`.
|
|
662
663
|
*/
|
|
663
664
|
const logger$24 = initializeLogger();
|
|
664
|
-
const TRUTHY = new Set([
|
|
665
|
+
const TRUTHY = /* @__PURE__ */ new Set([
|
|
665
666
|
"true",
|
|
666
667
|
"1",
|
|
667
668
|
"yes",
|
|
@@ -733,10 +734,12 @@ const buildDebugConfig = () => {
|
|
|
733
734
|
const providedSessionId = process.env.LMNR_DEBUG_SESSION_ID || existing?.session_id || void 0;
|
|
734
735
|
const sessionId = providedSessionId || randomUUID();
|
|
735
736
|
const sessionMinted = providedSessionId === void 0;
|
|
737
|
+
const replayTraceId = process.env.LMNR_DEBUG_REPLAY_TRACE_ID || existing?.replay_trace_id || null;
|
|
738
|
+
const cacheUntilValue = process.env.LMNR_DEBUG_CACHE_UNTIL ?? existing?.cache_until ?? void 0;
|
|
736
739
|
return {
|
|
737
740
|
sessionId,
|
|
738
|
-
replayTraceId
|
|
739
|
-
cacheUntilSpanId: parseCacheUntil(
|
|
741
|
+
replayTraceId,
|
|
742
|
+
cacheUntilSpanId: parseCacheUntil(cacheUntilValue),
|
|
740
743
|
localOrigin: true,
|
|
741
744
|
sessionMinted
|
|
742
745
|
};
|
|
@@ -957,14 +960,15 @@ var DebugRuntime = class {
|
|
|
957
960
|
if (!this._config.localOrigin) return;
|
|
958
961
|
if (this._emitted) return;
|
|
959
962
|
this._emitted = true;
|
|
960
|
-
|
|
963
|
+
const file = buildDebugSessionFile({
|
|
961
964
|
sessionId: this._config.sessionId,
|
|
962
965
|
traceId: this._traceId,
|
|
963
966
|
replayTraceId: this._config.replayTraceId,
|
|
964
967
|
cacheUntil: this._config.cacheUntilSpanId,
|
|
965
968
|
debuggerUrl: this.debuggerSessionUrl(),
|
|
966
969
|
startedAt: this._startedAt
|
|
967
|
-
})
|
|
970
|
+
});
|
|
971
|
+
emitPointer(file);
|
|
968
972
|
}
|
|
969
973
|
};
|
|
970
974
|
let runtime = null;
|
|
@@ -1173,7 +1177,7 @@ var LaminarSpan = class LaminarSpan {
|
|
|
1173
1177
|
}
|
|
1174
1178
|
addTags(tags) {
|
|
1175
1179
|
const currentTags = this.tags;
|
|
1176
|
-
this.setTags(Array.from(new Set([...currentTags, ...tags])));
|
|
1180
|
+
this.setTags(Array.from(/* @__PURE__ */ new Set([...currentTags, ...tags])));
|
|
1177
1181
|
}
|
|
1178
1182
|
getLaminarSpanContext() {
|
|
1179
1183
|
let spanPath = [];
|
|
@@ -1561,7 +1565,8 @@ const flagSettingsEnv = (existing, cwd) => {
|
|
|
1561
1565
|
} catch {}
|
|
1562
1566
|
return {};
|
|
1563
1567
|
}
|
|
1564
|
-
|
|
1568
|
+
const resolved = path.isAbsolute(trimmed) ? trimmed : path.join(cwd ?? process.cwd(), trimmed);
|
|
1569
|
+
return settingsEnvBlock(loadSettingsFile(resolved) ?? {});
|
|
1565
1570
|
};
|
|
1566
1571
|
/**
|
|
1567
1572
|
* Build the `--settings` value that forces the CLI through our proxy.
|
|
@@ -1595,7 +1600,8 @@ const buildProxyFlagSettings = (existing, proxyUrl, cwd, settingSources) => {
|
|
|
1595
1600
|
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return null;
|
|
1596
1601
|
settingsObj = parsed;
|
|
1597
1602
|
} else {
|
|
1598
|
-
const
|
|
1603
|
+
const resolved = path.isAbsolute(trimmed) ? trimmed : path.join(cwd ?? process.cwd(), trimmed);
|
|
1604
|
+
const loaded = loadSettingsFile(resolved);
|
|
1599
1605
|
if (loaded === null) return null;
|
|
1600
1606
|
settingsObj = loaded;
|
|
1601
1607
|
}
|
|
@@ -1686,7 +1692,10 @@ const resolveTargetUrlFromEnv = (envDict, fallback = DEFAULT_ANTHROPIC_BASE_URL,
|
|
|
1686
1692
|
const bedrockBaseUrl = getEnvValue(BEDROCK_BASE_URL_ENV);
|
|
1687
1693
|
if (bedrockBaseUrl) return bedrockBaseUrl.replace(/\/$/, "");
|
|
1688
1694
|
let region = getEnvValue(BEDROCK_AWS_REGION_ENV);
|
|
1689
|
-
if (!region)
|
|
1695
|
+
if (!region) {
|
|
1696
|
+
const awsProfile = getEnvValue("AWS_PROFILE") || "default";
|
|
1697
|
+
region = getRegionFromAwsConfig(awsProfile) ?? void 0;
|
|
1698
|
+
}
|
|
1690
1699
|
if (region) return `https://bedrock-runtime.${region}.amazonaws.com`;
|
|
1691
1700
|
logger$22.error(`${BEDROCK_USE_ENV} is set but could not determine AWS region. Set ${BEDROCK_AWS_REGION_ENV} or configure a region in ~/.aws/config for the active profile.`);
|
|
1692
1701
|
return null;
|
|
@@ -2225,10 +2234,13 @@ var Laminar = class Laminar {
|
|
|
2225
2234
|
runtime.recordProjectId(projectId);
|
|
2226
2235
|
const sessionUrl = runtime.debuggerSessionUrl();
|
|
2227
2236
|
logger$20.info(`Laminar debugger session: ${sessionUrl}`);
|
|
2228
|
-
if (runtime.shouldOpenBrowser)
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
|
|
2237
|
+
if (runtime.shouldOpenBrowser) {
|
|
2238
|
+
const opener = process.platform === "win32" ? "start" : process.platform === "darwin" ? "open" : "xdg-open";
|
|
2239
|
+
spawn(opener, [sessionUrl], {
|
|
2240
|
+
detached: true,
|
|
2241
|
+
stdio: "ignore"
|
|
2242
|
+
}).unref();
|
|
2243
|
+
}
|
|
2232
2244
|
}
|
|
2233
2245
|
}).catch((e) => {
|
|
2234
2246
|
logger$20.warn("Failed to register debug session: " + errorMessage(e));
|
|
@@ -2275,11 +2287,13 @@ var Laminar = class Laminar {
|
|
|
2275
2287
|
if (!debug || debug.enabled !== true || !debug.sessionId) return;
|
|
2276
2288
|
const existing = getRuntime();
|
|
2277
2289
|
if (existing !== null && existing.localOrigin) return;
|
|
2278
|
-
const
|
|
2290
|
+
const rolloutSessions = existing?.rolloutSessions ?? new LaminarClient({
|
|
2279
2291
|
baseUrl: this.baseUrlForDebug,
|
|
2280
2292
|
projectApiKey: this.projectApiKey,
|
|
2281
2293
|
port: this.httpPortForDebug
|
|
2282
|
-
}).rolloutSessions
|
|
2294
|
+
}).rolloutSessions;
|
|
2295
|
+
const debuggerUrl = process?.env?.LMNR_FRONTEND_URL ?? getFrontendUrl(this.baseUrlForDebug);
|
|
2296
|
+
const { runtime, configChanged } = initDebugRuntimeFromContext(debug, rolloutSessions, debuggerUrl);
|
|
2283
2297
|
if (runtime === null) return;
|
|
2284
2298
|
if (configChanged) {
|
|
2285
2299
|
this.debugRunLive = false;
|
|
@@ -2583,12 +2597,13 @@ var Laminar = class Laminar {
|
|
|
2583
2597
|
const sessionIdProperties = sessionIdValue ? { [SESSION_ID]: sessionIdValue } : {};
|
|
2584
2598
|
const traceTypeProperties = traceTypeValue ? { [TRACE_TYPE]: traceTypeValue } : {};
|
|
2585
2599
|
const tracingLevelProperties = tracingLevelValue ? { [`${ASSOCIATION_PROPERTIES}.tracing_level`]: tracingLevelValue } : {};
|
|
2586
|
-
const
|
|
2600
|
+
const mergedMetadata = {
|
|
2587
2601
|
...this.globalMetadata,
|
|
2588
2602
|
...ctxMetadata,
|
|
2589
2603
|
...parentMetadata,
|
|
2590
2604
|
...metadata
|
|
2591
|
-
}
|
|
2605
|
+
};
|
|
2606
|
+
const metadataProperties = metadataToAttributes(mergedMetadata);
|
|
2592
2607
|
const attributes = {
|
|
2593
2608
|
[SPAN_TYPE]: spanType ?? "DEFAULT",
|
|
2594
2609
|
...tagProperties,
|
|
@@ -2952,7 +2967,8 @@ const prettyPrintZodSchema = (schema, indent = 2) => {
|
|
|
2952
2967
|
if (typeof description === "string") result += `.describe('${description.replace(/'/g, "\\'")}')`;
|
|
2953
2968
|
return result;
|
|
2954
2969
|
};
|
|
2955
|
-
|
|
2970
|
+
const finalType = applyModifiers(value, getBaseType(value));
|
|
2971
|
+
return `${indentString}${key}: ${finalType},`;
|
|
2956
2972
|
}).join("\n")}\n})`;
|
|
2957
2973
|
};
|
|
2958
2974
|
const modelToProviderMap = {
|
|
@@ -3008,7 +3024,7 @@ const modelToProvider = (model) => {
|
|
|
3008
3024
|
const injectScript = (sessionRecordingOptions, stringifyCallbackArgs) => {
|
|
3009
3025
|
const BATCH_TIMEOUT = 2e3;
|
|
3010
3026
|
const HEARTBEAT_INTERVAL = 1e3;
|
|
3011
|
-
const CHUNK_SIZE =
|
|
3027
|
+
const CHUNK_SIZE = 262144;
|
|
3012
3028
|
window.lmnrRrwebEventsBatch = [];
|
|
3013
3029
|
window.lmnrChunkSequence = 0;
|
|
3014
3030
|
window.lmnrSendInProgress = false;
|
|
@@ -3069,7 +3085,9 @@ const injectScript = (sessionRecordingOptions, stringifyCallbackArgs) => {
|
|
|
3069
3085
|
try {
|
|
3070
3086
|
const compressedEvents = [];
|
|
3071
3087
|
for (const event of events) try {
|
|
3072
|
-
const
|
|
3088
|
+
const dataString = JSON.stringify(event.data);
|
|
3089
|
+
const compressed = await gzipCompress(dataString);
|
|
3090
|
+
const base64Data = await bufferToBase64(compressed);
|
|
3073
3091
|
compressedEvents.push({
|
|
3074
3092
|
...event,
|
|
3075
3093
|
data: base64Data
|
|
@@ -5319,15 +5337,17 @@ const applyGenerationSpanData = (lmnrSpan, spanData) => {
|
|
|
5319
5337
|
setGenAiInputMessages(lmnrSpan, inputData, getCurrentSystemInstructions());
|
|
5320
5338
|
const outputData = spanData?.output;
|
|
5321
5339
|
setGenAiOutputMessages(lmnrSpan, outputData);
|
|
5322
|
-
|
|
5340
|
+
const llmData = {
|
|
5323
5341
|
model: spanData?.model,
|
|
5324
5342
|
usage: spanData?.usage,
|
|
5325
5343
|
response_id: spanData?.response_id ?? spanData?.id
|
|
5326
|
-
}
|
|
5344
|
+
};
|
|
5345
|
+
applyLlmAttributes(lmnrSpan, llmData);
|
|
5327
5346
|
};
|
|
5328
5347
|
const applyResponseSpanData = (lmnrSpan, spanData) => {
|
|
5329
5348
|
const response = spanData?._response ?? spanData?.response;
|
|
5330
|
-
|
|
5349
|
+
const responseInput = spanData?._input ?? spanData?.input;
|
|
5350
|
+
setGenAiInputMessages(lmnrSpan, responseInput, getCurrentSystemInstructions());
|
|
5331
5351
|
if (response != null) {
|
|
5332
5352
|
setGenAiOutputMessagesFromResponse(lmnrSpan, response);
|
|
5333
5353
|
setToolDefinitionsFromResponse(lmnrSpan, response);
|
|
@@ -5368,8 +5388,10 @@ const applySpeechSpanData = (lmnrSpan, spanData) => {
|
|
|
5368
5388
|
const inputText = spanData?.input;
|
|
5369
5389
|
if (inputText) setGenAiInputMessages(lmnrSpan, inputText);
|
|
5370
5390
|
const outputData = spanData?.output;
|
|
5371
|
-
if (outputData)
|
|
5372
|
-
|
|
5391
|
+
if (outputData) {
|
|
5392
|
+
if (typeof outputData === "object" && !Array.isArray(outputData)) setGenAiOutputMessages(lmnrSpan, outputData.data);
|
|
5393
|
+
else setGenAiOutputMessages(lmnrSpan, outputData);
|
|
5394
|
+
}
|
|
5373
5395
|
};
|
|
5374
5396
|
const applyTranscriptionSpanData = (lmnrSpan, spanData) => {
|
|
5375
5397
|
const model = spanData?.model;
|
|
@@ -5379,8 +5401,10 @@ const applyTranscriptionSpanData = (lmnrSpan, spanData) => {
|
|
|
5379
5401
|
lmnrSpan.setAttribute(LaminarAttributes.PROVIDER, "openai");
|
|
5380
5402
|
}
|
|
5381
5403
|
const inputData = spanData?.input;
|
|
5382
|
-
if (inputData)
|
|
5383
|
-
|
|
5404
|
+
if (inputData) {
|
|
5405
|
+
if (typeof inputData === "object" && !Array.isArray(inputData)) setGenAiInputMessages(lmnrSpan, inputData.data);
|
|
5406
|
+
else setGenAiInputMessages(lmnrSpan, inputData);
|
|
5407
|
+
}
|
|
5384
5408
|
const outputText = spanData?.output;
|
|
5385
5409
|
if (outputText) setGenAiOutputMessages(lmnrSpan, outputText);
|
|
5386
5410
|
};
|
|
@@ -5603,7 +5627,8 @@ const wrapGetResponse = (original) => function(request, ...rest) {
|
|
|
5603
5627
|
};
|
|
5604
5628
|
const wrapGetStreamedResponse = (original) => function(request, ...rest) {
|
|
5605
5629
|
const systemInstructions = request?.systemInstructions;
|
|
5606
|
-
|
|
5630
|
+
const source = original.call(this, request, ...rest);
|
|
5631
|
+
return wrapStreamWithSystemInstructions(systemInstructions, source);
|
|
5607
5632
|
};
|
|
5608
5633
|
const MODEL_METHODS = ["getResponse", "getStreamedResponse"];
|
|
5609
5634
|
var OpenAIAgentsInstrumentation = class extends InstrumentationBase {
|
|
@@ -5748,12 +5773,12 @@ var OpencodeInstrumentation = class extends InstrumentationBase {
|
|
|
5748
5773
|
};
|
|
5749
5774
|
//#endregion
|
|
5750
5775
|
//#region src/opentelemetry-lib/instrumentation/openrouter/utils.ts
|
|
5751
|
-
const TERMINAL_RESPONSE_EVENTS = new Set([
|
|
5776
|
+
const TERMINAL_RESPONSE_EVENTS = /* @__PURE__ */ new Set([
|
|
5752
5777
|
"response.completed",
|
|
5753
5778
|
"response.incomplete",
|
|
5754
5779
|
"response.failed"
|
|
5755
5780
|
]);
|
|
5756
|
-
const ERROR_RESPONSE_STATUSES = new Set(["failed", "incomplete"]);
|
|
5781
|
+
const ERROR_RESPONSE_STATUSES = /* @__PURE__ */ new Set(["failed", "incomplete"]);
|
|
5757
5782
|
const safeSetAttribute$1 = (span, key, value) => {
|
|
5758
5783
|
if (value === null || value === void 0) return;
|
|
5759
5784
|
if (typeof value === "string" && value.length === 0) return;
|
|
@@ -5919,7 +5944,8 @@ const wrapStream = (span, kind, stream, traceContent) => {
|
|
|
5919
5944
|
throw error;
|
|
5920
5945
|
} finally {
|
|
5921
5946
|
try {
|
|
5922
|
-
|
|
5947
|
+
const response = kind === "chat" ? aggregateChatChunks(chunks) : responseFromStreamEvents(chunks);
|
|
5948
|
+
setResponseAttributes$1(span, kind, response, traceContent);
|
|
5923
5949
|
} finally {
|
|
5924
5950
|
span.end();
|
|
5925
5951
|
}
|
|
@@ -6485,7 +6511,9 @@ const buildHeaders = (existing) => {
|
|
|
6485
6511
|
const headers = { ...existing };
|
|
6486
6512
|
if (laminarCtx) {
|
|
6487
6513
|
headers[LAMINAR_SPAN_CONTEXT_HEADER] = encodePayload(JSON.stringify(laminarCtx));
|
|
6488
|
-
|
|
6514
|
+
const traceHex = laminarCtx.traceId.replace(/-/g, "");
|
|
6515
|
+
const spanHex = laminarCtx.spanId.replace(/-/g, "").slice(16);
|
|
6516
|
+
headers[TRACEPARENT_HEADER] = encodePayload(`00-${traceHex}-${spanHex}-01`);
|
|
6489
6517
|
} else {
|
|
6490
6518
|
const ctx = rawSpan.spanContext();
|
|
6491
6519
|
headers[TRACEPARENT_HEADER] = encodePayload(`00-${ctx.traceId}-${ctx.spanId}-01`);
|
|
@@ -6949,10 +6977,11 @@ const patchTemporalWorker = (workerModule, options = {}) => {
|
|
|
6949
6977
|
activity: [ActivityInterceptorFactory(options), ...rawOpts?.interceptors?.activity ?? []]
|
|
6950
6978
|
};
|
|
6951
6979
|
if (!usesWorkflowBundle) interceptors.workflowModules = [__require.resolve("@lmnr-ai/lmnr/temporal-workflow-interceptors"), ...rawOpts?.interceptors?.workflowModules ?? []];
|
|
6952
|
-
|
|
6980
|
+
const patched = {
|
|
6953
6981
|
...rawOpts,
|
|
6954
6982
|
interceptors
|
|
6955
|
-
}
|
|
6983
|
+
};
|
|
6984
|
+
return originalCreate(patched, ...rest);
|
|
6956
6985
|
};
|
|
6957
6986
|
};
|
|
6958
6987
|
/**
|
|
@@ -7161,8 +7190,9 @@ const initializeLaminarInstrumentations = (options = {}) => {
|
|
|
7161
7190
|
if (apiKey) {
|
|
7162
7191
|
const url = options.baseUrl ?? process?.env?.LMNR_BASE_URL ?? "https://api.lmnr.ai";
|
|
7163
7192
|
const port = options.httpPort ?? (url.match(/:\d{1,5}$/g) ? parseInt(url.match(/:\d{1,5}$/g)[0].slice(1)) : 443);
|
|
7193
|
+
const urlWithoutSlash = url.replace(/\/$/, "").replace(/:\d{1,5}$/g, "");
|
|
7164
7194
|
client = new LaminarClient({
|
|
7165
|
-
baseUrl: `${
|
|
7195
|
+
baseUrl: `${urlWithoutSlash}:${port}`,
|
|
7166
7196
|
projectApiKey: apiKey
|
|
7167
7197
|
});
|
|
7168
7198
|
}
|
|
@@ -7442,14 +7472,15 @@ const startTracing = (options) => {
|
|
|
7442
7472
|
});
|
|
7443
7473
|
};
|
|
7444
7474
|
const patchModules = (modules) => {
|
|
7475
|
+
const instrumentations = initializeLaminarInstrumentations({
|
|
7476
|
+
baseUrl: _baseHttpUrl,
|
|
7477
|
+
apiKey: _apiKey,
|
|
7478
|
+
instrumentModules: modules,
|
|
7479
|
+
suppressContentTracing: !shouldSendTraces(),
|
|
7480
|
+
sessionRecordingOptions: _configuration?.sessionRecordingOptions
|
|
7481
|
+
});
|
|
7445
7482
|
registerInstrumentations({
|
|
7446
|
-
instrumentations
|
|
7447
|
-
baseUrl: _baseHttpUrl,
|
|
7448
|
-
apiKey: _apiKey,
|
|
7449
|
-
instrumentModules: modules,
|
|
7450
|
-
suppressContentTracing: !shouldSendTraces(),
|
|
7451
|
-
sessionRecordingOptions: _configuration?.sessionRecordingOptions
|
|
7452
|
-
}),
|
|
7483
|
+
instrumentations,
|
|
7453
7484
|
tracerProvider
|
|
7454
7485
|
});
|
|
7455
7486
|
};
|
|
@@ -8296,165 +8327,10 @@ async function evaluate({ data, executor, evaluators, groupName, name, metadata,
|
|
|
8296
8327
|
//#region src/integrations/eve.ts
|
|
8297
8328
|
const logger$2 = initializeLogger();
|
|
8298
8329
|
const EVE_REPORTER_TRACER_NAME = "@lmnr-ai/lmnr/eve-reporter";
|
|
8299
|
-
const
|
|
8300
|
-
const
|
|
8301
|
-
|
|
8302
|
-
const
|
|
8303
|
-
/**
|
|
8304
|
-
* Root span name for every propagated eval, and it never changes. eve calls
|
|
8305
|
-
* `ClientSession.send` before any hook that knows which eval is in flight, and
|
|
8306
|
-
* eve exposes no `onEvalStart`, so the id simply is not knowable here. Renaming
|
|
8307
|
-
* the span later would invalidate the `lmnr.span.path` the processor already
|
|
8308
|
-
* stamped on it and on every descendant, and would make the path we propagate
|
|
8309
|
-
* to the agent stale. The eval id rides on `lmnr.eve.eval.id`, on the trace
|
|
8310
|
-
* metadata, and on the datapoint instead.
|
|
8311
|
-
*/
|
|
8312
|
-
const ROOT_SPAN_NAME = "eve eval";
|
|
8313
|
-
const EXECUTOR_SPAN_NAME = "executor";
|
|
8314
|
-
/** Prototypes already patched — `send` must never be wrapped twice. */
|
|
8315
|
-
const patchedSendPrototypes = /* @__PURE__ */ new WeakSet();
|
|
8316
|
-
/**
|
|
8317
|
-
* Per-eve-client-session state. Keyed off the `ClientSession` instance rather
|
|
8318
|
-
* than an AsyncLocalStorage frame or a "current eval" global: eve runs up to
|
|
8319
|
-
* `maxConcurrency` evals at once, and each eval builds its own `ClientSession`
|
|
8320
|
-
* (`EvalSessionManager` calls `client.session()` per `EvalSessionDriver`), so
|
|
8321
|
-
* the instance is the only correct correlation key available inside `send`.
|
|
8322
|
-
*/
|
|
8323
|
-
const traceByClientSession = /* @__PURE__ */ new WeakMap();
|
|
8324
|
-
/** eve session id -> trace, read by `onEvalComplete` via `result.sessionId`. */
|
|
8325
|
-
const traceByEveSessionId = /* @__PURE__ */ new Map();
|
|
8326
|
-
/** Set by the running reporter's `onRunStart`; cleared by `onRunComplete`. */
|
|
8327
|
-
let activeSessionTraceFactory = null;
|
|
8328
|
-
/** Mirrors eve's own `normalizeSendTurnInput`, then merges the trace headers. */
|
|
8329
|
-
const withTraceHeaders = (input, headers) => {
|
|
8330
|
-
const payload = typeof input === "string" ? { message: input } : { ...input };
|
|
8331
|
-
payload.headers = {
|
|
8332
|
-
...payload.headers ?? {},
|
|
8333
|
-
...headers
|
|
8334
|
-
};
|
|
8335
|
-
return payload;
|
|
8336
|
-
};
|
|
8337
|
-
const openSessionTrace = (clientSession) => {
|
|
8338
|
-
try {
|
|
8339
|
-
const existing = traceByClientSession.get(clientSession);
|
|
8340
|
-
if (existing) return existing;
|
|
8341
|
-
const minted = activeSessionTraceFactory?.();
|
|
8342
|
-
if (minted) traceByClientSession.set(clientSession, minted);
|
|
8343
|
-
return minted;
|
|
8344
|
-
} catch (error) {
|
|
8345
|
-
logger$2.warn(`Laminar eve reporter: failed to open a session trace: ` + errorMessage(error));
|
|
8346
|
-
return;
|
|
8347
|
-
}
|
|
8348
|
-
};
|
|
8349
|
-
/**
|
|
8350
|
-
* eve assigns the session id on the POST that `send` awaits, so the returned
|
|
8351
|
-
* `MessageResponse` is the earliest place the runner can bind the eve session
|
|
8352
|
-
* id to the trace it minted.
|
|
8353
|
-
*/
|
|
8354
|
-
const recordEveSessionId = (response, sessionTrace) => {
|
|
8355
|
-
const sessionId = response?.sessionId;
|
|
8356
|
-
if (typeof sessionId !== "string" || sessionId.length === 0) return;
|
|
8357
|
-
sessionTrace.sessionIds.add(sessionId);
|
|
8358
|
-
traceByEveSessionId.set(sessionId, sessionTrace);
|
|
8359
|
-
};
|
|
8360
|
-
/**
|
|
8361
|
-
* Build the propagation headers for the EXECUTOR span. Plain HTTP strings — eve
|
|
8362
|
-
* forwards `SendTurnPayload.headers` verbatim to `fetch`.
|
|
8363
|
-
*
|
|
8364
|
-
* The Laminar context carries the full path. Laminar nests spans by
|
|
8365
|
-
* `lmnr.span.ids_path`, not by OTel `parentSpanId`, so a receiver that adopts
|
|
8366
|
-
* only the ids lands its spans as a second root on our trace. The path is what
|
|
8367
|
-
* makes the agent's subtree render UNDER the executor. Both paths are constants
|
|
8368
|
-
* here — neither the root nor the executor span is ever renamed.
|
|
8369
|
-
*/
|
|
8370
|
-
const buildTraceHeaders = (rootSpan, executorSpan) => {
|
|
8371
|
-
const spanContext = executorSpan.spanContext();
|
|
8372
|
-
const laminarContext = {
|
|
8373
|
-
traceId: otelTraceIdToUUID(spanContext.traceId),
|
|
8374
|
-
spanId: otelSpanIdToUUID(spanContext.spanId),
|
|
8375
|
-
isRemote: true,
|
|
8376
|
-
spanPath: [ROOT_SPAN_NAME, EXECUTOR_SPAN_NAME],
|
|
8377
|
-
spanIdsPath: [otelSpanIdToUUID(rootSpan.spanContext().spanId), otelSpanIdToUUID(spanContext.spanId)]
|
|
8378
|
-
};
|
|
8379
|
-
return {
|
|
8380
|
-
[TRACEPARENT_HEADER$1]: `00-${spanContext.traceId}-${spanContext.spanId}-01`,
|
|
8381
|
-
[LAMINAR_SPAN_CONTEXT_HEADER$1]: JSON.stringify(laminarContext)
|
|
8382
|
-
};
|
|
8383
|
-
};
|
|
8384
|
-
/**
|
|
8385
|
-
* Bind this session's ROOT span into the calling eval's async context, so work
|
|
8386
|
-
* the eval does in the RUNNER process after `send` resolves lands on this trace
|
|
8387
|
-
* instead of opening its own. The concrete case is `t.judge.autoevals.*`: eve
|
|
8388
|
-
* runs the judge's model call in the runner, and Laminar's AI SDK integration
|
|
8389
|
-
* parents it to `LaminarContextManager.getContext()`.
|
|
8390
|
-
*
|
|
8391
|
-
* The root — not a pre-opened judge span — is the binding target on purpose.
|
|
8392
|
-
* An eval may record any number of judge assertions, and each one runs its own
|
|
8393
|
-
* model call. Binding the root gives every judge its own child of the root; a
|
|
8394
|
-
* single pre-opened parent would collect all of them under whichever judge
|
|
8395
|
-
* happened to run first.
|
|
8396
|
-
*
|
|
8397
|
-
* `pushContext` uses `AsyncLocalStorage.enterWith`, so this MUST run
|
|
8398
|
-
* synchronously before `send`'s first `await` — at that point we are still
|
|
8399
|
-
* executing in the caller's async context, so the store we set is the one
|
|
8400
|
-
* `test(t)` continues with. Each eval's `test(t)` runs in its own async chain,
|
|
8401
|
-
* so concurrent evals never see each other's root span; do NOT replace this
|
|
8402
|
-
* with the process-global context stack, which would cross-attribute.
|
|
8403
|
-
*
|
|
8404
|
-
* Judge spans reach Laminar only when the runner opts in — `Laminar.initialize()`
|
|
8405
|
-
* plus a globally registered AI SDK telemetry integration. Without that the
|
|
8406
|
-
* binding is inert, never wrong.
|
|
8407
|
-
*
|
|
8408
|
-
* Never popped: the store dies with the eval's async chain.
|
|
8409
|
-
*/
|
|
8410
|
-
const bindEvalContext = (sessionTrace) => {
|
|
8411
|
-
try {
|
|
8412
|
-
LaminarContextManager.pushContext(trace.setSpan(ROOT_CONTEXT, sessionTrace.rootSpan));
|
|
8413
|
-
} catch (error) {
|
|
8414
|
-
logger$2.warn(`Laminar eve reporter: failed to bind the eval context: ` + errorMessage(error));
|
|
8415
|
-
}
|
|
8416
|
-
};
|
|
8417
|
-
/**
|
|
8418
|
-
* Wrap `ClientSession.prototype.send` so every eve turn carries the trace
|
|
8419
|
-
* context of a runner-minted Laminar span. Idempotent per prototype.
|
|
8420
|
-
*
|
|
8421
|
-
* @internal Exported for tests, which drive it with a stub class so the suite
|
|
8422
|
-
* keeps no dependency on eve.
|
|
8423
|
-
*/
|
|
8424
|
-
const patchEveClientSession = (sessionClass) => {
|
|
8425
|
-
const prototype = sessionClass?.prototype;
|
|
8426
|
-
const original = prototype?.send;
|
|
8427
|
-
if (!prototype || typeof original !== "function") return false;
|
|
8428
|
-
if (patchedSendPrototypes.has(prototype)) return true;
|
|
8429
|
-
prototype.send = async function(input) {
|
|
8430
|
-
const sessionTrace = openSessionTrace(this);
|
|
8431
|
-
if (!sessionTrace) return original.call(this, input);
|
|
8432
|
-
bindEvalContext(sessionTrace);
|
|
8433
|
-
const response = await original.call(this, withTraceHeaders(input, sessionTrace.headers));
|
|
8434
|
-
try {
|
|
8435
|
-
recordEveSessionId(response, sessionTrace);
|
|
8436
|
-
} catch (error) {
|
|
8437
|
-
logger$2.warn(`Laminar eve reporter: failed to record the eve session id: ` + errorMessage(error));
|
|
8438
|
-
}
|
|
8439
|
-
return response;
|
|
8440
|
-
};
|
|
8441
|
-
patchedSendPrototypes.add(prototype);
|
|
8442
|
-
return true;
|
|
8443
|
-
};
|
|
8444
|
-
/**
|
|
8445
|
-
* Load eve's `ClientSession` without depending on eve. Mirrors the guarded
|
|
8446
|
-
* dynamic import in eve's own Braintrust reporter (`loadBraintrustSdk`), except
|
|
8447
|
-
* a missing eve degrades instead of throwing — the reporter must keep working
|
|
8448
|
-
* when it is registered outside an eve run.
|
|
8449
|
-
*/
|
|
8450
|
-
const loadEveClientSession = async () => {
|
|
8451
|
-
try {
|
|
8452
|
-
const sessionClass = (await import(EVE_CLIENT_MODULE)).ClientSession;
|
|
8453
|
-
return typeof sessionClass === "function" ? sessionClass : void 0;
|
|
8454
|
-
} catch {
|
|
8455
|
-
return;
|
|
8456
|
-
}
|
|
8457
|
-
};
|
|
8330
|
+
const OTEL_TRACE_ID_RE = /^[0-9a-f]{32}$/iu;
|
|
8331
|
+
const OTEL_SPAN_ID_RE = /^[0-9a-f]{16}$/iu;
|
|
8332
|
+
const ZERO_TRACE_ID = "0".repeat(32);
|
|
8333
|
+
const ZERO_SPAN_ID = "0".repeat(16);
|
|
8458
8334
|
const didAssertionPass = (assertion) => {
|
|
8459
8335
|
if (typeof assertion.passed === "boolean") return assertion.passed;
|
|
8460
8336
|
if (typeof assertion.score === "number") {
|
|
@@ -8463,14 +8339,8 @@ const didAssertionPass = (assertion) => {
|
|
|
8463
8339
|
}
|
|
8464
8340
|
return true;
|
|
8465
8341
|
};
|
|
8466
|
-
/**
|
|
8467
|
-
|
|
8468
|
-
* Detailed assertion outcomes live in metadata; sparse per-assertion score keys
|
|
8469
|
-
* make Laminar's eval UI look incomplete when different eval files assert
|
|
8470
|
-
* different things.
|
|
8471
|
-
*/
|
|
8472
|
-
const resultToScores = (result) => {
|
|
8473
|
-
const assertions = result.assertions ?? [];
|
|
8342
|
+
/** Keep Laminar's score columns stable across heterogeneous eve evals. */
|
|
8343
|
+
const resultToScores = (result, assertions) => {
|
|
8474
8344
|
const gateAssertions = assertions.filter((assertion) => assertion.severity === "gate");
|
|
8475
8345
|
const thresholdedSoftAssertions = assertions.filter((assertion) => assertion.severity === "soft" && typeof assertion.threshold === "number");
|
|
8476
8346
|
return {
|
|
@@ -8488,7 +8358,6 @@ const assertionsMetadata = (assertions) => assertions.map((assertion) => ({
|
|
|
8488
8358
|
message: assertion.message,
|
|
8489
8359
|
metadata: assertion.metadata
|
|
8490
8360
|
}));
|
|
8491
|
-
/** Surface failed assertions in metadata so a failing run is debuggable. */
|
|
8492
8361
|
const failedAssertionsMetadata = (assertions) => {
|
|
8493
8362
|
const failed = assertions.filter((assertion) => !didAssertionPass(assertion)).map((assertion) => ({
|
|
8494
8363
|
name: assertion.name,
|
|
@@ -8496,79 +8365,62 @@ const failedAssertionsMetadata = (assertions) => {
|
|
|
8496
8365
|
}));
|
|
8497
8366
|
return failed.length > 0 ? { failedAssertions: failed } : {};
|
|
8498
8367
|
};
|
|
8499
|
-
const
|
|
8368
|
+
const validTraceContext = (context) => context != null && typeof context.traceId === "string" && OTEL_TRACE_ID_RE.test(context.traceId) && context.traceId !== ZERO_TRACE_ID && typeof context.spanId === "string" && OTEL_SPAN_ID_RE.test(context.spanId) && context.spanId !== ZERO_SPAN_ID && typeof context.traceFlags === "number" && Number.isInteger(context.traceFlags) && context.traceFlags >= 0 && context.traceFlags <= 255;
|
|
8369
|
+
const normalizeTraceContexts = (contexts) => Array.isArray(contexts) ? contexts.filter(validTraceContext) : [];
|
|
8370
|
+
const isSampledTraceContext = (context) => (context.traceFlags & 1) === 1;
|
|
8371
|
+
const selectTraceContext = (contexts) => contexts.find((context) => context.primary === true && isSampledTraceContext(context)) ?? contexts.find(isSampledTraceContext);
|
|
8372
|
+
const traceContextMetadata = (contexts) => contexts.map((context) => ({
|
|
8373
|
+
traceId: context.traceId,
|
|
8374
|
+
spanId: context.spanId,
|
|
8375
|
+
traceFlags: context.traceFlags,
|
|
8376
|
+
primary: context.primary,
|
|
8377
|
+
sessionId: context.sessionId
|
|
8378
|
+
}));
|
|
8500
8379
|
/**
|
|
8501
|
-
*
|
|
8502
|
-
* or per-eval via the `reporters` field — eve runs and grades each eval, then
|
|
8503
|
-
* hands the graded result here, and this reporter ships it to Laminar as an
|
|
8504
|
-
* evaluation run with one datapoint per eval.
|
|
8505
|
-
*
|
|
8506
|
-
* The datapoint's trace id is never looked up — the reporter OWNS the trace.
|
|
8507
|
-
* `patchEveClientSession` wraps eve's `ClientSession.send` and mints the
|
|
8508
|
-
* EVALUATION root + EXECUTOR child on the first turn of each session, so the
|
|
8509
|
-
* runner already knows the trace id and pushes it to the agent as a
|
|
8510
|
-
* `traceparent` header. The eve session id is only the in-process key that ties
|
|
8511
|
-
* the send we instrumented to the graded result eve hands back later.
|
|
8512
|
-
*
|
|
8513
|
-
* Two outcomes, recorded on `metadata.traceResolution`:
|
|
8514
|
-
*
|
|
8515
|
-
* 1. `"propagated"` — the normal path, above. The agent joins the trace only
|
|
8516
|
-
* when `agent/instrumentation.ts` sets `traceChannelRequests: true` and the
|
|
8517
|
-
* agent process runs with `WORKFLOW_TRACE_MODE=continuous`.
|
|
8518
|
-
* 2. `"reporter-fallback"` — no session trace was minted (propagation disabled,
|
|
8519
|
-
* or eve produced no session id). The reporter opens its own EVALUATION span
|
|
8520
|
-
* so the grade still lands somewhere debuggable; it holds no agent work.
|
|
8521
|
-
*
|
|
8522
|
-
* `t.judge.autoevals.*` model calls run in the RUNNER process, and they reach
|
|
8523
|
-
* Laminar only when the runner registers an AI SDK telemetry integration. eve
|
|
8524
|
-
* imports `generateText` inside its own bundle, so `wrapAISDK` cannot reach it —
|
|
8525
|
-
* but AI SDK v7 reads registered integrations off `globalThis`, which does. One
|
|
8526
|
-
* line in `evals.config.ts`:
|
|
8380
|
+
* Reports eve eval results to Laminar.
|
|
8527
8381
|
*
|
|
8528
|
-
*
|
|
8529
|
-
*
|
|
8530
|
-
*
|
|
8382
|
+
* Eve 0.35+ gives reporters every native trace context observed during an eval.
|
|
8383
|
+
* The reporter links the datapoint to the first sampled primary trace (or the
|
|
8384
|
+
* first sampled trace when there is no sampled primary), records every valid
|
|
8385
|
+
* context in metadata, and adds
|
|
8386
|
+
* a small EVALUATION span to the selected native trace for the grade. When eve
|
|
8387
|
+
* reports no usable context, a reporter-owned fallback trace preserves the
|
|
8388
|
+
* result without pretending it contains agent work.
|
|
8531
8389
|
*
|
|
8532
|
-
*
|
|
8533
|
-
* the
|
|
8534
|
-
* Each judge gets its own span under the eval's root — see
|
|
8535
|
-
* {@link bindEvalContext}, and {@link LaminarReporter.flush} for how spans ship.
|
|
8536
|
-
* Without that line the reporter still records every assertion's score as an
|
|
8537
|
-
* EVALUATOR span; only the model call is missing.
|
|
8390
|
+
* The agent must export its native OpenTelemetry spans to the same Laminar
|
|
8391
|
+
* project. In the eve agent, declare a destination like:
|
|
8538
8392
|
*
|
|
8539
|
-
* @example
|
|
8540
8393
|
* ```ts
|
|
8541
|
-
* import {
|
|
8542
|
-
* import {
|
|
8394
|
+
* import { LaminarSpanProcessor } from "@lmnr-ai/lmnr";
|
|
8395
|
+
* import { otelIntegration } from "eve/instrumentation/otel";
|
|
8543
8396
|
*
|
|
8544
|
-
* export default
|
|
8545
|
-
*
|
|
8397
|
+
* export default otelIntegration({
|
|
8398
|
+
* spanProcessors: [new LaminarSpanProcessor()],
|
|
8546
8399
|
* });
|
|
8547
8400
|
* ```
|
|
8548
8401
|
*/
|
|
8549
8402
|
var LaminarReporter = class {
|
|
8550
8403
|
constructor(options = {}) {
|
|
8551
8404
|
this.evalsById = /* @__PURE__ */ new Map();
|
|
8552
|
-
this.
|
|
8553
|
-
this.
|
|
8554
|
-
this.sessionTraceFactory = () => this.mintSessionTrace();
|
|
8405
|
+
this.evalIndexById = /* @__PURE__ */ new Map();
|
|
8406
|
+
this.nextFallbackIndex = 0;
|
|
8555
8407
|
this.options = options;
|
|
8556
8408
|
}
|
|
8557
8409
|
async onRunStart(evaluations, target) {
|
|
8558
8410
|
this.evalId = void 0;
|
|
8559
|
-
|
|
8560
|
-
|
|
8561
|
-
|
|
8411
|
+
const identifiedEvaluations = evaluations.map((evaluation, index) => ({
|
|
8412
|
+
evaluation,
|
|
8413
|
+
index
|
|
8414
|
+
})).filter((item) => typeof item.evaluation.id === "string");
|
|
8415
|
+
this.evalsById = new Map(identifiedEvaluations.map(({ evaluation }) => [evaluation.id, evaluation]));
|
|
8416
|
+
this.evalIndexById = new Map(identifiedEvaluations.map(({ evaluation, index }) => [evaluation.id, index]));
|
|
8417
|
+
this.nextFallbackIndex = evaluations.length;
|
|
8562
8418
|
try {
|
|
8563
8419
|
this.client = this.options.client ?? new LaminarClient({
|
|
8564
8420
|
baseUrl: this.options.baseUrl,
|
|
8565
8421
|
projectApiKey: this.options.projectApiKey
|
|
8566
8422
|
});
|
|
8567
8423
|
this.ensureTracing();
|
|
8568
|
-
if (this.options.propagateTraceContext !== false) {
|
|
8569
|
-
activeSessionTraceFactory = this.sessionTraceFactory;
|
|
8570
|
-
patchEveClientSession(await loadEveClientSession());
|
|
8571
|
-
}
|
|
8572
8424
|
const evaluation = await this.client.evals.init(this.options.name, this.options.groupName, {
|
|
8573
8425
|
...this.options.metadata ?? {},
|
|
8574
8426
|
source: "eve",
|
|
@@ -8581,26 +8433,25 @@ var LaminarReporter = class {
|
|
|
8581
8433
|
process.stdout.write(`\nLaminar: check eve eval results at ${url}\n`);
|
|
8582
8434
|
} catch (error) {
|
|
8583
8435
|
logger$2.error(`Laminar eve reporter: failed to start run: ${errorMessage(error)}`);
|
|
8584
|
-
if (!this.evalId)
|
|
8585
|
-
this.client = void 0;
|
|
8586
|
-
if (activeSessionTraceFactory === this.sessionTraceFactory) activeSessionTraceFactory = null;
|
|
8587
|
-
}
|
|
8436
|
+
if (!this.evalId) this.client = void 0;
|
|
8588
8437
|
}
|
|
8589
8438
|
}
|
|
8590
|
-
async onEvalComplete(result) {
|
|
8439
|
+
async onEvalComplete(result, context) {
|
|
8591
8440
|
const client = this.client;
|
|
8592
8441
|
const evalId = this.evalId;
|
|
8593
8442
|
if (!client || !evalId) return;
|
|
8594
|
-
const index = this.
|
|
8443
|
+
const index = typeof result.id === "string" ? this.evalIndexById.get(result.id) ?? this.nextFallbackIndex++ : this.nextFallbackIndex++;
|
|
8595
8444
|
const task = result.result ?? {};
|
|
8596
8445
|
const derived = task.derived ?? {};
|
|
8597
|
-
const assertions = result.assertions
|
|
8598
|
-
const
|
|
8599
|
-
const
|
|
8600
|
-
|
|
8446
|
+
const assertions = Array.isArray(result.assertions) ? result.assertions.filter((assertion) => assertion != null && typeof assertion === "object") : [];
|
|
8447
|
+
const evalDefinition = typeof result.id === "string" ? this.evalsById.get(result.id) : void 0;
|
|
8448
|
+
const evalDescription = context?.evaluation?.description ?? evalDefinition?.description;
|
|
8449
|
+
const contexts = normalizeTraceContexts(context?.traceContexts ?? task.traceContexts ?? []);
|
|
8450
|
+
const gradeTrace = this.openGradeTrace({
|
|
8451
|
+
contexts,
|
|
8601
8452
|
description: evalDescription,
|
|
8602
|
-
|
|
8603
|
-
|
|
8453
|
+
evalId: result.id ?? String(index),
|
|
8454
|
+
verdict: result.verdict
|
|
8604
8455
|
});
|
|
8605
8456
|
try {
|
|
8606
8457
|
const datapointId = await client.evals.createDatapoint({
|
|
@@ -8608,17 +8459,19 @@ var LaminarReporter = class {
|
|
|
8608
8459
|
data: result.id ?? null,
|
|
8609
8460
|
target: task.finalMessage ?? null,
|
|
8610
8461
|
index,
|
|
8611
|
-
traceId:
|
|
8462
|
+
traceId: gradeTrace.traceId,
|
|
8612
8463
|
metadata: {
|
|
8613
8464
|
name: result.id,
|
|
8614
8465
|
verdict: result.verdict,
|
|
8615
8466
|
status: task.status,
|
|
8616
8467
|
description: evalDescription,
|
|
8617
8468
|
sessionId: task.sessionId,
|
|
8618
|
-
traceResolution:
|
|
8469
|
+
traceResolution: gradeTrace.source,
|
|
8470
|
+
selectedTraceContext: gradeTrace.selected ? traceContextMetadata([gradeTrace.selected])[0] : void 0,
|
|
8471
|
+
traceContexts: traceContextMetadata(gradeTrace.contexts),
|
|
8619
8472
|
skipReason: result.skipReason,
|
|
8620
8473
|
error: result.error,
|
|
8621
|
-
toolCalls: (derived.toolCalls
|
|
8474
|
+
toolCalls: (Array.isArray(derived.toolCalls) ? derived.toolCalls : []).map((call) => call?.name).filter((name) => typeof name === "string"),
|
|
8622
8475
|
toolCallCount: derived.toolCallCount,
|
|
8623
8476
|
parked: derived.parked,
|
|
8624
8477
|
failureCode: derived.failureCode,
|
|
@@ -8627,44 +8480,30 @@ var LaminarReporter = class {
|
|
|
8627
8480
|
...failedAssertionsMetadata(assertions)
|
|
8628
8481
|
}
|
|
8629
8482
|
});
|
|
8630
|
-
const executorOutput = task.output ?? task.finalMessage ?? null;
|
|
8631
8483
|
await client.evals.updateDatapoint({
|
|
8632
8484
|
evalId,
|
|
8633
8485
|
datapointId,
|
|
8634
|
-
scores: resultToScores(result),
|
|
8635
|
-
executorOutput
|
|
8486
|
+
scores: resultToScores(result, assertions),
|
|
8487
|
+
executorOutput: task.output ?? task.finalMessage ?? null
|
|
8636
8488
|
});
|
|
8637
8489
|
} catch (error) {
|
|
8638
|
-
|
|
8490
|
+
gradeTrace.span.recordException(error);
|
|
8639
8491
|
logger$2.error(`Laminar eve reporter: failed to report eval "${result.id ?? index}": ${errorMessage(error)}`);
|
|
8640
8492
|
} finally {
|
|
8641
|
-
|
|
8642
|
-
|
|
8643
|
-
evalId: result.id,
|
|
8644
|
-
description: evalDescription,
|
|
8645
|
-
verdict: result.verdict,
|
|
8646
|
-
status: task.status,
|
|
8493
|
+
this.finishGradeTrace(gradeTrace, {
|
|
8494
|
+
assertions,
|
|
8647
8495
|
output: task.output ?? task.finalMessage ?? null,
|
|
8648
|
-
|
|
8496
|
+
status: task.status
|
|
8649
8497
|
});
|
|
8650
8498
|
await this.flush();
|
|
8651
8499
|
}
|
|
8652
8500
|
}
|
|
8653
|
-
async onRunComplete() {
|
|
8654
|
-
this.finishOpenSessionTraces();
|
|
8655
|
-
if (activeSessionTraceFactory === this.sessionTraceFactory) activeSessionTraceFactory = null;
|
|
8501
|
+
async onRunComplete(_summary) {
|
|
8656
8502
|
await this.flush();
|
|
8657
8503
|
this.evalId = void 0;
|
|
8658
8504
|
this.evalsById.clear();
|
|
8505
|
+
this.evalIndexById.clear();
|
|
8659
8506
|
}
|
|
8660
|
-
/**
|
|
8661
|
-
* Point the process's Laminar tracing at this reporter's project, unless
|
|
8662
|
-
* something already initialized it. Mirrors `LaminarAiSdkTelemetry`, which
|
|
8663
|
-
* initializes the same way for judge spans — so eve eval spans and judge
|
|
8664
|
-
* spans share ONE tracer provider however the run was set up. Without this,
|
|
8665
|
-
* `getTracer()` returns a noop whenever nobody registered the AI SDK
|
|
8666
|
-
* integration, and every eve span would be dropped silently.
|
|
8667
|
-
*/
|
|
8668
8507
|
ensureTracing() {
|
|
8669
8508
|
if (Laminar.initialized()) {
|
|
8670
8509
|
const ourKey = this.client?.apiKey ?? this.options.projectApiKey;
|
|
@@ -8680,140 +8519,79 @@ var LaminarReporter = class {
|
|
|
8680
8519
|
disableBatch: true
|
|
8681
8520
|
});
|
|
8682
8521
|
}
|
|
8683
|
-
/**
|
|
8684
|
-
* Flush the process's Laminar pipeline — eve eval spans AND judge spans, which
|
|
8685
|
-
* now share one provider. eve ends `eve eval` with `process.exit()` (skipping
|
|
8686
|
-
* `beforeExit`), so these calls are the only points at which spans ship.
|
|
8687
|
-
* Never `shutdown()`: the pipeline may belong to the host, not to us.
|
|
8688
|
-
*/
|
|
8689
8522
|
async flush() {
|
|
8690
8523
|
try {
|
|
8691
8524
|
await Laminar.flush();
|
|
8692
8525
|
} catch (error) {
|
|
8693
|
-
logger$2.warn(`Laminar eve reporter: failed to flush spans:
|
|
8526
|
+
logger$2.warn(`Laminar eve reporter: failed to flush spans: ${errorMessage(error)}`);
|
|
8694
8527
|
}
|
|
8695
8528
|
}
|
|
8696
|
-
|
|
8697
|
-
|
|
8698
|
-
* root plus the EXECUTOR child the agent's turn is parented to. Called from
|
|
8699
|
-
* the patched `send`, so it must never throw into the user's eval.
|
|
8700
|
-
*/
|
|
8701
|
-
mintSessionTrace() {
|
|
8529
|
+
openGradeTrace({ contexts, description, evalId, verdict }) {
|
|
8530
|
+
const selected = selectTraceContext(contexts);
|
|
8702
8531
|
const tracer = getTracer();
|
|
8703
|
-
const
|
|
8704
|
-
|
|
8705
|
-
|
|
8706
|
-
"lmnr.eve.reporter": EVE_REPORTER_TRACER_NAME
|
|
8707
|
-
} }, ROOT_CONTEXT);
|
|
8708
|
-
const executorSpan = tracer.startSpan(EXECUTOR_SPAN_NAME, { attributes: {
|
|
8709
|
-
[SPAN_TYPE]: "EXECUTOR",
|
|
8710
|
-
[TRACE_TYPE]: "EVALUATION"
|
|
8711
|
-
} }, trace.setSpan(ROOT_CONTEXT, rootSpan));
|
|
8712
|
-
const sessionTrace = {
|
|
8713
|
-
traceId: otelTraceIdToUUID(rootSpan.spanContext().traceId),
|
|
8714
|
-
rootSpan,
|
|
8715
|
-
executorSpan,
|
|
8716
|
-
tracer,
|
|
8717
|
-
headers: buildTraceHeaders(rootSpan, executorSpan),
|
|
8718
|
-
sessionIds: /* @__PURE__ */ new Set(),
|
|
8719
|
-
ended: false
|
|
8720
|
-
};
|
|
8721
|
-
this.openSessionTraces.add(sessionTrace);
|
|
8722
|
-
return sessionTrace;
|
|
8723
|
-
}
|
|
8724
|
-
/** Grade, close and release one session trace. Idempotent. */
|
|
8725
|
-
finishSessionTrace(sessionTrace, grade) {
|
|
8726
|
-
if (sessionTrace.ended) return;
|
|
8727
|
-
sessionTrace.ended = true;
|
|
8728
|
-
this.openSessionTraces.delete(sessionTrace);
|
|
8729
|
-
for (const sessionId of sessionTrace.sessionIds) traceByEveSessionId.delete(sessionId);
|
|
8730
|
-
try {
|
|
8731
|
-
sessionTrace.rootSpan.setAttributes({
|
|
8732
|
-
"lmnr.eve.eval.id": grade.evalId ?? "",
|
|
8733
|
-
"lmnr.eve.eval.verdict": grade.verdict ?? "",
|
|
8734
|
-
"lmnr.eve.eval.status": grade.status ?? "",
|
|
8735
|
-
[SPAN_INPUT]: JSON.stringify({
|
|
8736
|
-
evalId: grade.evalId,
|
|
8737
|
-
description: grade.description
|
|
8738
|
-
}),
|
|
8739
|
-
[SPAN_OUTPUT]: JSON.stringify(grade.output ?? null),
|
|
8740
|
-
...metadataToAttributes({
|
|
8741
|
-
source: "eve",
|
|
8742
|
-
eveEvalId: grade.evalId ?? "",
|
|
8743
|
-
eveEvalDescription: grade.description ?? "",
|
|
8744
|
-
eveEvalVerdict: grade.verdict ?? "",
|
|
8745
|
-
eveSessionId: [...sessionTrace.sessionIds].join(",")
|
|
8746
|
-
})
|
|
8747
|
-
});
|
|
8748
|
-
sessionTrace.executorSpan.setAttribute(SPAN_OUTPUT, JSON.stringify(grade.output ?? null));
|
|
8749
|
-
this.recordEvaluatorSpans(sessionTrace, grade.assertions ?? []);
|
|
8750
|
-
sessionTrace.executorSpan.end();
|
|
8751
|
-
sessionTrace.rootSpan.end();
|
|
8752
|
-
} catch (error) {
|
|
8753
|
-
logger$2.warn(`Laminar eve reporter: failed to finish the session trace: ` + errorMessage(error));
|
|
8754
|
-
}
|
|
8755
|
-
}
|
|
8756
|
-
/**
|
|
8757
|
-
* One EVALUATOR span per eve assertion, judge or not. They are created at
|
|
8758
|
-
* grade time (the assertions only exist then) as short children of the
|
|
8759
|
-
* still-open root. No explicit parent path is needed: the root keeps the name
|
|
8760
|
-
* it was minted with, so the processor's cached path for it is still correct.
|
|
8761
|
-
*/
|
|
8762
|
-
recordEvaluatorSpans(sessionTrace, assertions) {
|
|
8763
|
-
const rootContext = trace.setSpan(ROOT_CONTEXT, sessionTrace.rootSpan);
|
|
8764
|
-
for (const assertion of assertions) {
|
|
8765
|
-
const name = assertion.name ?? "assertion";
|
|
8766
|
-
const output = JSON.stringify({
|
|
8767
|
-
score: assertion.score ?? null,
|
|
8768
|
-
passed: didAssertionPass(assertion),
|
|
8769
|
-
severity: assertion.severity,
|
|
8770
|
-
threshold: assertion.threshold,
|
|
8771
|
-
message: assertion.message
|
|
8772
|
-
});
|
|
8773
|
-
sessionTrace.tracer.startSpan(name, { attributes: {
|
|
8774
|
-
[SPAN_TYPE]: "EVALUATOR",
|
|
8775
|
-
[TRACE_TYPE]: "EVALUATION",
|
|
8776
|
-
[SPAN_OUTPUT]: output
|
|
8777
|
-
} }, rootContext).end();
|
|
8778
|
-
}
|
|
8779
|
-
}
|
|
8780
|
-
finishOpenSessionTraces() {
|
|
8781
|
-
for (const sessionTrace of [...this.openSessionTraces]) this.finishSessionTrace(sessionTrace, {});
|
|
8782
|
-
}
|
|
8783
|
-
/**
|
|
8784
|
-
* Resolve the trace this datapoint links to. There is no lookup: either
|
|
8785
|
-
* `patchEveClientSession` already minted the session's trace (and we know its
|
|
8786
|
-
* id), or we open a reporter-owned EVALUATION span to hold the grade.
|
|
8787
|
-
*/
|
|
8788
|
-
resolveDatapointTrace({ evalId, description, verdict, sessionId }) {
|
|
8789
|
-
const sessionTrace = sessionId ? traceByEveSessionId.get(sessionId) : void 0;
|
|
8790
|
-
if (sessionTrace) return {
|
|
8791
|
-
traceId: sessionTrace.traceId,
|
|
8792
|
-
source: "propagated",
|
|
8793
|
-
sessionTrace
|
|
8794
|
-
};
|
|
8795
|
-
const span = getTracer().startSpan(`eve eval ${evalId}`, { attributes: {
|
|
8532
|
+
const parentContext = selected ? trace.setSpanContext(ROOT_CONTEXT, this.toSpanContext(selected)) : ROOT_CONTEXT;
|
|
8533
|
+
const source = selected ? "eve-native" : "reporter-fallback";
|
|
8534
|
+
const span = tracer.startSpan(`eve eval ${evalId}`, { attributes: {
|
|
8796
8535
|
[SPAN_TYPE]: "EVALUATION",
|
|
8797
8536
|
[TRACE_TYPE]: "EVALUATION",
|
|
8798
8537
|
"lmnr.eve.reporter": EVE_REPORTER_TRACER_NAME,
|
|
8799
8538
|
"lmnr.eve.eval.id": evalId,
|
|
8800
8539
|
"lmnr.eve.eval.verdict": verdict ?? "",
|
|
8801
|
-
|
|
8802
|
-
"lmnr.eve.trace_resolution": "reporter-fallback",
|
|
8540
|
+
"lmnr.eve.trace_resolution": source,
|
|
8803
8541
|
...metadataToAttributes({
|
|
8804
8542
|
source: "eve",
|
|
8805
8543
|
eveEvalId: evalId,
|
|
8806
8544
|
eveEvalDescription: description ?? "",
|
|
8807
8545
|
eveEvalVerdict: verdict ?? "",
|
|
8808
|
-
eveSessionId: sessionId ?? ""
|
|
8546
|
+
eveSessionId: selected?.sessionId ?? "",
|
|
8547
|
+
eveTraceCount: contexts.length
|
|
8548
|
+
}),
|
|
8549
|
+
[SPAN_INPUT]: JSON.stringify({
|
|
8550
|
+
evalId,
|
|
8551
|
+
description
|
|
8809
8552
|
})
|
|
8810
|
-
} },
|
|
8553
|
+
} }, parentContext);
|
|
8554
|
+
return {
|
|
8555
|
+
traceId: otelTraceIdToUUID(span.spanContext().traceId),
|
|
8556
|
+
source,
|
|
8557
|
+
span,
|
|
8558
|
+
tracer,
|
|
8559
|
+
selected,
|
|
8560
|
+
contexts
|
|
8561
|
+
};
|
|
8562
|
+
}
|
|
8563
|
+
toSpanContext(context) {
|
|
8811
8564
|
return {
|
|
8812
|
-
traceId:
|
|
8813
|
-
|
|
8814
|
-
|
|
8565
|
+
traceId: context.traceId.toLowerCase(),
|
|
8566
|
+
spanId: context.spanId.toLowerCase(),
|
|
8567
|
+
traceFlags: context.traceFlags,
|
|
8568
|
+
isRemote: true
|
|
8815
8569
|
};
|
|
8816
8570
|
}
|
|
8571
|
+
finishGradeTrace(gradeTrace, grade) {
|
|
8572
|
+
try {
|
|
8573
|
+
gradeTrace.span.setAttributes({
|
|
8574
|
+
"lmnr.eve.eval.status": grade.status ?? "",
|
|
8575
|
+
[SPAN_OUTPUT]: JSON.stringify(grade.output ?? null)
|
|
8576
|
+
});
|
|
8577
|
+
const parentContext = trace.setSpan(ROOT_CONTEXT, gradeTrace.span);
|
|
8578
|
+
for (const assertion of grade.assertions) gradeTrace.tracer.startSpan(assertion.name ?? "assertion", { attributes: {
|
|
8579
|
+
[SPAN_TYPE]: "EVALUATOR",
|
|
8580
|
+
[TRACE_TYPE]: "EVALUATION",
|
|
8581
|
+
[SPAN_OUTPUT]: JSON.stringify({
|
|
8582
|
+
score: assertion.score ?? null,
|
|
8583
|
+
passed: didAssertionPass(assertion),
|
|
8584
|
+
severity: assertion.severity,
|
|
8585
|
+
threshold: assertion.threshold,
|
|
8586
|
+
message: assertion.message
|
|
8587
|
+
})
|
|
8588
|
+
} }, parentContext).end();
|
|
8589
|
+
} catch (error) {
|
|
8590
|
+
logger$2.warn(`Laminar eve reporter: failed to finish the grade trace: ${errorMessage(error)}`);
|
|
8591
|
+
} finally {
|
|
8592
|
+
gradeTrace.span.end();
|
|
8593
|
+
}
|
|
8594
|
+
}
|
|
8817
8595
|
};
|
|
8818
8596
|
//#endregion
|
|
8819
8597
|
//#region src/debug/hash.ts
|
|
@@ -8868,8 +8646,10 @@ const stripSystemMessages = (messages) => messages.filter((m) => !(m !== null &&
|
|
|
8868
8646
|
* hex-encoded (64 chars).
|
|
8869
8647
|
*/
|
|
8870
8648
|
const debugInputHash = (messages) => {
|
|
8871
|
-
const
|
|
8872
|
-
|
|
8649
|
+
const stripped = stripSystemMessages(messages);
|
|
8650
|
+
const canonical = canonicalJson(stripped);
|
|
8651
|
+
const digest = blake3(new TextEncoder().encode(canonical));
|
|
8652
|
+
return bytesToHex(digest);
|
|
8873
8653
|
};
|
|
8874
8654
|
//#endregion
|
|
8875
8655
|
//#region src/debug/replay.ts
|
|
@@ -8985,6 +8765,28 @@ const applyUsageToSpan = (span, usage) => {
|
|
|
8985
8765
|
applyUsage(attrs, usage);
|
|
8986
8766
|
for (const [k, v] of Object.entries(attrs)) if (v !== void 0) span.setAttribute(k, v);
|
|
8987
8767
|
};
|
|
8768
|
+
const resolveAssociationProperties = (event, parent) => {
|
|
8769
|
+
const { userId, sessionId, tags, ...metadata } = {
|
|
8770
|
+
...event?.toolsContext ?? {},
|
|
8771
|
+
...event?.runtimeContext ?? {}
|
|
8772
|
+
};
|
|
8773
|
+
return {
|
|
8774
|
+
associationProperties: {
|
|
8775
|
+
userId: typeof userId === "string" && userId.length > 0 ? userId : parent.userId,
|
|
8776
|
+
sessionId: typeof sessionId === "string" && sessionId.length > 0 ? sessionId : parent.sessionId,
|
|
8777
|
+
metadata: {
|
|
8778
|
+
...parent.metadata ?? {},
|
|
8779
|
+
...metadata
|
|
8780
|
+
}
|
|
8781
|
+
},
|
|
8782
|
+
tags: Array.isArray(tags) && tags.every((tag) => typeof tag === "string") ? tags : void 0
|
|
8783
|
+
};
|
|
8784
|
+
};
|
|
8785
|
+
const applyAssociationProperties = (span, properties) => {
|
|
8786
|
+
if (properties.userId) span.setAttribute(USER_ID, properties.userId);
|
|
8787
|
+
if (properties.sessionId) span.setAttribute(SESSION_ID, properties.sessionId);
|
|
8788
|
+
span.setAttributes(metadataToAttributes(properties.metadata));
|
|
8789
|
+
};
|
|
8988
8790
|
const applyFinishReason = (span, finishReason) => {
|
|
8989
8791
|
span.setAttribute("gen_ai.response.finish_reason", finishReason);
|
|
8990
8792
|
span.setAttribute("ai.response.finishReason", finishReason);
|
|
@@ -9018,6 +8820,17 @@ const verbatimPromptString = (prompt) => {
|
|
|
9018
8820
|
return null;
|
|
9019
8821
|
}
|
|
9020
8822
|
};
|
|
8823
|
+
const buildDecisionInputMessages = (state) => {
|
|
8824
|
+
if (state === void 0 || state === null) return void 0;
|
|
8825
|
+
if (Array.isArray(state)) return verbatimPromptString([{
|
|
8826
|
+
role: "user",
|
|
8827
|
+
content: state
|
|
8828
|
+
}]) ?? void 0;
|
|
8829
|
+
return serializeJSON$1([{
|
|
8830
|
+
role: "user",
|
|
8831
|
+
content: typeof state === "string" ? state : serializeJSON$1(state)
|
|
8832
|
+
}]);
|
|
8833
|
+
};
|
|
9021
8834
|
const verbatimPromptMessages = (prompt) => {
|
|
9022
8835
|
const serialized = verbatimPromptString(prompt);
|
|
9023
8836
|
if (serialized === null) return null;
|
|
@@ -9673,7 +9486,6 @@ var LaminarAiSdkTelemetry = class {
|
|
|
9673
9486
|
} : {}
|
|
9674
9487
|
}
|
|
9675
9488
|
}, parentCtx);
|
|
9676
|
-
const spanCtx = trace.setSpan(parentCtx, span);
|
|
9677
9489
|
span.setAttribute(SPAN_TYPE, "DEFAULT");
|
|
9678
9490
|
if (typeof event.operationId === "string") span.setAttribute("ai.operation", event.operationId);
|
|
9679
9491
|
if (typeof event.provider === "string") {
|
|
@@ -9686,14 +9498,14 @@ var LaminarAiSdkTelemetry = class {
|
|
|
9686
9498
|
}
|
|
9687
9499
|
if (typeof event.functionId === "string") span.setAttribute("ai.functionId", event.functionId);
|
|
9688
9500
|
if (event.operationId === "ai.embed" || event.operationId === "ai.embedMany" || event.operationId === "ai.rerank") span.setAttribute(SPAN_TYPE, "LLM");
|
|
9689
|
-
const
|
|
9690
|
-
|
|
9691
|
-
|
|
9692
|
-
};
|
|
9693
|
-
|
|
9694
|
-
|
|
9695
|
-
|
|
9696
|
-
span
|
|
9501
|
+
const parentAssociationProperties = parentCtx.getValue(ASSOCIATION_PROPERTIES_KEY) ?? {};
|
|
9502
|
+
const { associationProperties, tags } = resolveAssociationProperties(event, parentAssociationProperties);
|
|
9503
|
+
applyAssociationProperties(span, associationProperties);
|
|
9504
|
+
if (tags) span.setAttribute(`${ASSOCIATION_PROPERTIES}.tags`, tags);
|
|
9505
|
+
const spanCtx = trace.setSpan(parentCtx.setValue(ASSOCIATION_PROPERTIES_KEY, {
|
|
9506
|
+
...parentAssociationProperties,
|
|
9507
|
+
...Object.fromEntries(Object.entries(associationProperties).filter(([, value]) => value !== void 0))
|
|
9508
|
+
}), span);
|
|
9697
9509
|
if (this.recordInputs) {
|
|
9698
9510
|
if (Array.isArray(event.messages) || event.system !== void 0 || event.instructions !== void 0) {
|
|
9699
9511
|
const msgs = verbatimStandardizedMessages(event);
|
|
@@ -9709,7 +9521,8 @@ var LaminarAiSdkTelemetry = class {
|
|
|
9709
9521
|
ctx: spanCtx,
|
|
9710
9522
|
operationId: event.operationId ?? "unknown",
|
|
9711
9523
|
provider: event.provider,
|
|
9712
|
-
modelId: event.modelId
|
|
9524
|
+
modelId: event.modelId,
|
|
9525
|
+
associationProperties
|
|
9713
9526
|
});
|
|
9714
9527
|
};
|
|
9715
9528
|
this.onStepStart = (event) => {
|
|
@@ -9727,6 +9540,7 @@ var LaminarAiSdkTelemetry = class {
|
|
|
9727
9540
|
const spanCtx = trace.setSpan(op.ctx, span);
|
|
9728
9541
|
span.setAttribute(SPAN_TYPE, "DEFAULT");
|
|
9729
9542
|
span.setAttributes(buildAiSdkInstrumentationAttributes());
|
|
9543
|
+
applyAssociationProperties(span, op.associationProperties);
|
|
9730
9544
|
span.setAttribute("ai.step.number", stepNumber);
|
|
9731
9545
|
if (this.recordInputs) {
|
|
9732
9546
|
const msgs = verbatimStandardizedMessages(event);
|
|
@@ -9743,11 +9557,13 @@ var LaminarAiSdkTelemetry = class {
|
|
|
9743
9557
|
if (!callId) return;
|
|
9744
9558
|
const step = this.findLatestStep(callId);
|
|
9745
9559
|
const stepNumber = step?.stepNumber ?? event?.stepNumber ?? 0;
|
|
9746
|
-
const
|
|
9747
|
-
|
|
9560
|
+
const op = this.operationByCallId.get(callId);
|
|
9561
|
+
const parentCtx = step?.ctx ?? op?.ctx;
|
|
9562
|
+
if (!op || !parentCtx) return;
|
|
9748
9563
|
const span = getTracer().startSpan(`ai.llm ${event.provider ?? ""}:${event.modelId ?? ""}`, { kind: SpanKind.CLIENT }, parentCtx);
|
|
9749
9564
|
span.setAttribute(SPAN_TYPE, "LLM");
|
|
9750
9565
|
span.setAttributes(buildAiSdkInstrumentationAttributes());
|
|
9566
|
+
applyAssociationProperties(span, op.associationProperties);
|
|
9751
9567
|
applyRequestModelAttributes(span, event);
|
|
9752
9568
|
if (this.recordInputs) {
|
|
9753
9569
|
if (event.tools) try {
|
|
@@ -9844,6 +9660,7 @@ var LaminarAiSdkTelemetry = class {
|
|
|
9844
9660
|
const span = getTracer().startSpan(`ai.llm ${event.provider ?? ""}:${event.modelId ?? ""}`, { kind: SpanKind.CLIENT }, op.ctx);
|
|
9845
9661
|
span.setAttribute(SPAN_TYPE, "LLM");
|
|
9846
9662
|
span.setAttributes(buildAiSdkInstrumentationAttributes());
|
|
9663
|
+
applyAssociationProperties(span, op.associationProperties);
|
|
9847
9664
|
applyRequestModelAttributes(span, event);
|
|
9848
9665
|
if (this.recordInputs && Array.isArray(event?.promptMessages)) {
|
|
9849
9666
|
const serialized = verbatimPromptString(event.promptMessages);
|
|
@@ -9882,12 +9699,15 @@ var LaminarAiSdkTelemetry = class {
|
|
|
9882
9699
|
const toolCallId = toolCall?.toolCallId;
|
|
9883
9700
|
const toolName = toolCall?.toolName;
|
|
9884
9701
|
if (!callId || !toolCallId) return;
|
|
9885
|
-
const
|
|
9886
|
-
|
|
9702
|
+
const step = this.findLatestStep(callId);
|
|
9703
|
+
const op = this.operationByCallId.get(callId);
|
|
9704
|
+
const parentCtx = step?.ctx ?? op?.ctx;
|
|
9705
|
+
if (!op || !parentCtx) return;
|
|
9887
9706
|
const span = getTracer().startSpan(`ai.tool ${toolName ?? "unknown"}`, { kind: SpanKind.CLIENT }, parentCtx);
|
|
9888
9707
|
const spanCtx = trace.setSpan(parentCtx, span);
|
|
9889
9708
|
span.setAttribute(SPAN_TYPE, "TOOL");
|
|
9890
9709
|
span.setAttributes(buildAiSdkInstrumentationAttributes());
|
|
9710
|
+
applyAssociationProperties(span, op.associationProperties);
|
|
9891
9711
|
if (toolName) span.setAttribute("ai.toolCall.name", toolName);
|
|
9892
9712
|
span.setAttribute("ai.toolCall.id", toolCallId);
|
|
9893
9713
|
if (this.recordInputs && toolCall?.input !== void 0) {
|
|
@@ -9957,6 +9777,35 @@ var LaminarAiSdkTelemetry = class {
|
|
|
9957
9777
|
};
|
|
9958
9778
|
this.onRerankStart = () => {};
|
|
9959
9779
|
this.onRerankEnd = () => {};
|
|
9780
|
+
this.experimental_onDecideStart = (event) => {
|
|
9781
|
+
const callId = event?.callId;
|
|
9782
|
+
if (!callId) return;
|
|
9783
|
+
this.onStart(event);
|
|
9784
|
+
const op = this.operationByCallId.get(callId);
|
|
9785
|
+
if (!op) return;
|
|
9786
|
+
op.span.setAttribute(SPAN_TYPE, "LLM");
|
|
9787
|
+
applyRequestModelAttributes(op.span, event);
|
|
9788
|
+
if (this.recordInputs) {
|
|
9789
|
+
const input = buildDecisionInputMessages(event.state);
|
|
9790
|
+
if (input !== void 0) op.span.setAttribute("gen_ai.input.messages", input);
|
|
9791
|
+
if (event.questions !== void 0) op.span.setAttribute("gen_ai.request.structured_output_schema", serializeJSON$1(event.questions));
|
|
9792
|
+
}
|
|
9793
|
+
};
|
|
9794
|
+
this.experimental_onDecideEnd = (event) => {
|
|
9795
|
+
const callId = event?.callId;
|
|
9796
|
+
if (!callId) return;
|
|
9797
|
+
const op = this.operationByCallId.get(callId);
|
|
9798
|
+
if (!op) return;
|
|
9799
|
+
if (typeof event.response?.modelId === "string") op.span.setAttribute(LaminarAttributes.RESPONSE_MODEL, event.response.modelId);
|
|
9800
|
+
if (typeof event.response?.id === "string") op.span.setAttribute("gen_ai.response.id", event.response.id);
|
|
9801
|
+
applyUsageToSpan(op.span, event.usage);
|
|
9802
|
+
if (this.recordOutputs && event.answers !== void 0) op.span.setAttribute("gen_ai.output.messages", serializeJSON$1([{
|
|
9803
|
+
role: "assistant",
|
|
9804
|
+
content: serializeJSON$1(event.answers)
|
|
9805
|
+
}]));
|
|
9806
|
+
op.span.setStatus({ code: SpanStatusCode.OK });
|
|
9807
|
+
this.closeOperation(callId, op);
|
|
9808
|
+
};
|
|
9960
9809
|
this.onEnd = (event) => {
|
|
9961
9810
|
const callId = event?.callId;
|
|
9962
9811
|
if (!callId) return;
|