@lmnr-ai/lmnr 0.8.49 → 0.8.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.cjs +19 -14
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.d.cts +2 -2
- package/dist/cli.d.mts +2 -2
- package/dist/cli.mjs +16 -11
- package/dist/cli.mjs.map +1 -1
- package/dist/{evaluations-4bQ4UuTj.d.mts → evaluations-BVaPPf0j.d.mts} +1679 -1305
- package/dist/{evaluations-CbHnDCAY.d.cts → evaluations-C8cI_XIE.d.cts} +1682 -1307
- package/dist/{file-utils-Ct8PiWkV.mjs → file-utils-CVTUiqFU.mjs} +47 -18
- package/dist/file-utils-CVTUiqFU.mjs.map +1 -0
- package/dist/{file-utils-N65ihgSU.cjs → file-utils-Cyx-2kqE.cjs} +47 -18
- package/dist/file-utils-Cyx-2kqE.cjs.map +1 -0
- package/dist/index.cjs +280 -431
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +6938 -4632
- package/dist/index.d.mts +6938 -4632
- package/dist/index.mjs +280 -431
- package/dist/index.mjs.map +1 -1
- package/dist/opentelemetry-lib/instrumentation/temporal/workflow-interceptors.cjs +114 -102
- package/dist/opentelemetry-lib/instrumentation/temporal/workflow-interceptors.cjs.map +1 -1
- package/dist/opentelemetry-lib/instrumentation/temporal/workflow-interceptors.d.cts +776 -372
- package/dist/opentelemetry-lib/instrumentation/temporal/workflow-interceptors.d.mts +776 -372
- package/dist/opentelemetry-lib/instrumentation/temporal/workflow-interceptors.mjs +114 -102
- package/dist/opentelemetry-lib/instrumentation/temporal/workflow-interceptors.mjs.map +1 -1
- package/dist/{rolldown-runtime-2rV9d50f.mjs → rolldown-runtime-B4iAMlE-.mjs} +2 -2
- package/dist/{rolldown-runtime-CVvi-lCc.cjs → rolldown-runtime-Ctb5zv51.cjs} +1 -1
- package/dist/{utils-CbvnF9B9.mjs → utils-Ch0R_9XE.mjs} +9 -8
- package/dist/{utils-CbvnF9B9.mjs.map → utils-Ch0R_9XE.mjs.map} +1 -1
- package/dist/{utils-D8kgcFxb.cjs → utils-Cq24YsD_.cjs} +10 -9
- package/dist/utils-Cq24YsD_.cjs.map +1 -0
- package/package.json +15 -17
- package/dist/file-utils-Ct8PiWkV.mjs.map +0 -1
- package/dist/file-utils-N65ihgSU.cjs.map +0 -1
- package/dist/utils-D8kgcFxb.cjs.map +0 -1
package/dist/index.cjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
-
const require_rolldown_runtime = require("./rolldown-runtime-
|
|
3
|
-
const require_utils = require("./utils-
|
|
2
|
+
const require_rolldown_runtime = require("./rolldown-runtime-Ctb5zv51.cjs");
|
|
3
|
+
const require_utils = require("./utils-Cq24YsD_.cjs");
|
|
4
4
|
const require_consts = require("./consts-9rSS0oeR.cjs");
|
|
5
5
|
let path = require("path");
|
|
6
6
|
path = require_rolldown_runtime.__toESM(path);
|
|
@@ -183,7 +183,7 @@ var LaminarDataset = class extends EvaluationDataset {
|
|
|
183
183
|
*/
|
|
184
184
|
async push(paths, recursive = false) {
|
|
185
185
|
if (!this.client) throw new Error("Client not set");
|
|
186
|
-
const { loadFromPaths } = await Promise.resolve().then(() => require("./file-utils-
|
|
186
|
+
const { loadFromPaths } = await Promise.resolve().then(() => require("./file-utils-Cyx-2kqE.cjs")).then((n) => n.file_utils_exports);
|
|
187
187
|
const data = await loadFromPaths(Array.isArray(paths) ? paths : [paths], recursive);
|
|
188
188
|
if (data.length === 0) {
|
|
189
189
|
console.warn("No data to push. Skipping");
|
|
@@ -569,7 +569,8 @@ const handleStreamResult = (originalResult, streamInfo, span, ignoreOutput, seri
|
|
|
569
569
|
});
|
|
570
570
|
trackStreamProcessing(async () => {
|
|
571
571
|
try {
|
|
572
|
-
const
|
|
572
|
+
const tempResponse = new Response(stream2);
|
|
573
|
+
const { chunks, error } = await consumeResponse(tempResponse);
|
|
573
574
|
if (shouldSendTraces() && !ignoreOutput) try {
|
|
574
575
|
const output = {
|
|
575
576
|
type: "response",
|
|
@@ -668,7 +669,7 @@ const writeDebugSessionFile = (file, dir = process.cwd()) => {
|
|
|
668
669
|
* logic line-comparable with the Python `config.py`.
|
|
669
670
|
*/
|
|
670
671
|
const logger$24 = require_utils.initializeLogger();
|
|
671
|
-
const TRUTHY = new Set([
|
|
672
|
+
const TRUTHY = /* @__PURE__ */ new Set([
|
|
672
673
|
"true",
|
|
673
674
|
"1",
|
|
674
675
|
"yes",
|
|
@@ -740,10 +741,12 @@ const buildDebugConfig = () => {
|
|
|
740
741
|
const providedSessionId = process.env.LMNR_DEBUG_SESSION_ID || existing?.session_id || void 0;
|
|
741
742
|
const sessionId = providedSessionId || (0, node_crypto.randomUUID)();
|
|
742
743
|
const sessionMinted = providedSessionId === void 0;
|
|
744
|
+
const replayTraceId = process.env.LMNR_DEBUG_REPLAY_TRACE_ID || existing?.replay_trace_id || null;
|
|
745
|
+
const cacheUntilValue = process.env.LMNR_DEBUG_CACHE_UNTIL ?? existing?.cache_until ?? void 0;
|
|
743
746
|
return {
|
|
744
747
|
sessionId,
|
|
745
|
-
replayTraceId
|
|
746
|
-
cacheUntilSpanId: parseCacheUntil(
|
|
748
|
+
replayTraceId,
|
|
749
|
+
cacheUntilSpanId: parseCacheUntil(cacheUntilValue),
|
|
747
750
|
localOrigin: true,
|
|
748
751
|
sessionMinted
|
|
749
752
|
};
|
|
@@ -964,14 +967,15 @@ var DebugRuntime = class {
|
|
|
964
967
|
if (!this._config.localOrigin) return;
|
|
965
968
|
if (this._emitted) return;
|
|
966
969
|
this._emitted = true;
|
|
967
|
-
|
|
970
|
+
const file = buildDebugSessionFile({
|
|
968
971
|
sessionId: this._config.sessionId,
|
|
969
972
|
traceId: this._traceId,
|
|
970
973
|
replayTraceId: this._config.replayTraceId,
|
|
971
974
|
cacheUntil: this._config.cacheUntilSpanId,
|
|
972
975
|
debuggerUrl: this.debuggerSessionUrl(),
|
|
973
976
|
startedAt: this._startedAt
|
|
974
|
-
})
|
|
977
|
+
});
|
|
978
|
+
emitPointer(file);
|
|
975
979
|
}
|
|
976
980
|
};
|
|
977
981
|
let runtime = null;
|
|
@@ -1180,7 +1184,7 @@ var LaminarSpan = class LaminarSpan {
|
|
|
1180
1184
|
}
|
|
1181
1185
|
addTags(tags) {
|
|
1182
1186
|
const currentTags = this.tags;
|
|
1183
|
-
this.setTags(Array.from(new Set([...currentTags, ...tags])));
|
|
1187
|
+
this.setTags(Array.from(/* @__PURE__ */ new Set([...currentTags, ...tags])));
|
|
1184
1188
|
}
|
|
1185
1189
|
getLaminarSpanContext() {
|
|
1186
1190
|
let spanPath = [];
|
|
@@ -1568,7 +1572,8 @@ const flagSettingsEnv = (existing, cwd) => {
|
|
|
1568
1572
|
} catch {}
|
|
1569
1573
|
return {};
|
|
1570
1574
|
}
|
|
1571
|
-
|
|
1575
|
+
const resolved = path.isAbsolute(trimmed) ? trimmed : path.join(cwd ?? process.cwd(), trimmed);
|
|
1576
|
+
return settingsEnvBlock(loadSettingsFile(resolved) ?? {});
|
|
1572
1577
|
};
|
|
1573
1578
|
/**
|
|
1574
1579
|
* Build the `--settings` value that forces the CLI through our proxy.
|
|
@@ -1602,7 +1607,8 @@ const buildProxyFlagSettings = (existing, proxyUrl, cwd, settingSources) => {
|
|
|
1602
1607
|
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return null;
|
|
1603
1608
|
settingsObj = parsed;
|
|
1604
1609
|
} else {
|
|
1605
|
-
const
|
|
1610
|
+
const resolved = path.isAbsolute(trimmed) ? trimmed : path.join(cwd ?? process.cwd(), trimmed);
|
|
1611
|
+
const loaded = loadSettingsFile(resolved);
|
|
1606
1612
|
if (loaded === null) return null;
|
|
1607
1613
|
settingsObj = loaded;
|
|
1608
1614
|
}
|
|
@@ -1693,7 +1699,10 @@ const resolveTargetUrlFromEnv = (envDict, fallback = DEFAULT_ANTHROPIC_BASE_URL,
|
|
|
1693
1699
|
const bedrockBaseUrl = getEnvValue(BEDROCK_BASE_URL_ENV);
|
|
1694
1700
|
if (bedrockBaseUrl) return bedrockBaseUrl.replace(/\/$/, "");
|
|
1695
1701
|
let region = getEnvValue(BEDROCK_AWS_REGION_ENV);
|
|
1696
|
-
if (!region)
|
|
1702
|
+
if (!region) {
|
|
1703
|
+
const awsProfile = getEnvValue("AWS_PROFILE") || "default";
|
|
1704
|
+
region = getRegionFromAwsConfig(awsProfile) ?? void 0;
|
|
1705
|
+
}
|
|
1697
1706
|
if (region) return `https://bedrock-runtime.${region}.amazonaws.com`;
|
|
1698
1707
|
logger$22.error(`${BEDROCK_USE_ENV} is set but could not determine AWS region. Set ${BEDROCK_AWS_REGION_ENV} or configure a region in ~/.aws/config for the active profile.`);
|
|
1699
1708
|
return null;
|
|
@@ -2232,10 +2241,13 @@ var Laminar = class Laminar {
|
|
|
2232
2241
|
runtime.recordProjectId(projectId);
|
|
2233
2242
|
const sessionUrl = runtime.debuggerSessionUrl();
|
|
2234
2243
|
logger$20.info(`Laminar debugger session: ${sessionUrl}`);
|
|
2235
|
-
if (runtime.shouldOpenBrowser)
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2244
|
+
if (runtime.shouldOpenBrowser) {
|
|
2245
|
+
const opener = process.platform === "win32" ? "start" : process.platform === "darwin" ? "open" : "xdg-open";
|
|
2246
|
+
(0, node_child_process.spawn)(opener, [sessionUrl], {
|
|
2247
|
+
detached: true,
|
|
2248
|
+
stdio: "ignore"
|
|
2249
|
+
}).unref();
|
|
2250
|
+
}
|
|
2239
2251
|
}
|
|
2240
2252
|
}).catch((e) => {
|
|
2241
2253
|
logger$20.warn("Failed to register debug session: " + require_utils.errorMessage(e));
|
|
@@ -2282,11 +2294,13 @@ var Laminar = class Laminar {
|
|
|
2282
2294
|
if (!debug || debug.enabled !== true || !debug.sessionId) return;
|
|
2283
2295
|
const existing = getRuntime();
|
|
2284
2296
|
if (existing !== null && existing.localOrigin) return;
|
|
2285
|
-
const
|
|
2297
|
+
const rolloutSessions = existing?.rolloutSessions ?? new require_utils.LaminarClient({
|
|
2286
2298
|
baseUrl: this.baseUrlForDebug,
|
|
2287
2299
|
projectApiKey: this.projectApiKey,
|
|
2288
2300
|
port: this.httpPortForDebug
|
|
2289
|
-
}).rolloutSessions
|
|
2301
|
+
}).rolloutSessions;
|
|
2302
|
+
const debuggerUrl = process?.env?.LMNR_FRONTEND_URL ?? require_utils.getFrontendUrl(this.baseUrlForDebug);
|
|
2303
|
+
const { runtime, configChanged } = initDebugRuntimeFromContext(debug, rolloutSessions, debuggerUrl);
|
|
2290
2304
|
if (runtime === null) return;
|
|
2291
2305
|
if (configChanged) {
|
|
2292
2306
|
this.debugRunLive = false;
|
|
@@ -2590,12 +2604,13 @@ var Laminar = class Laminar {
|
|
|
2590
2604
|
const sessionIdProperties = sessionIdValue ? { [require_utils.SESSION_ID]: sessionIdValue } : {};
|
|
2591
2605
|
const traceTypeProperties = traceTypeValue ? { [require_utils.TRACE_TYPE]: traceTypeValue } : {};
|
|
2592
2606
|
const tracingLevelProperties = tracingLevelValue ? { [`${require_utils.ASSOCIATION_PROPERTIES}.tracing_level`]: tracingLevelValue } : {};
|
|
2593
|
-
const
|
|
2607
|
+
const mergedMetadata = {
|
|
2594
2608
|
...this.globalMetadata,
|
|
2595
2609
|
...ctxMetadata,
|
|
2596
2610
|
...parentMetadata,
|
|
2597
2611
|
...metadata
|
|
2598
|
-
}
|
|
2612
|
+
};
|
|
2613
|
+
const metadataProperties = require_utils.metadataToAttributes(mergedMetadata);
|
|
2599
2614
|
const attributes = {
|
|
2600
2615
|
[require_utils.SPAN_TYPE]: spanType ?? "DEFAULT",
|
|
2601
2616
|
...tagProperties,
|
|
@@ -2959,7 +2974,8 @@ const prettyPrintZodSchema = (schema, indent = 2) => {
|
|
|
2959
2974
|
if (typeof description === "string") result += `.describe('${description.replace(/'/g, "\\'")}')`;
|
|
2960
2975
|
return result;
|
|
2961
2976
|
};
|
|
2962
|
-
|
|
2977
|
+
const finalType = applyModifiers(value, getBaseType(value));
|
|
2978
|
+
return `${indentString}${key}: ${finalType},`;
|
|
2963
2979
|
}).join("\n")}\n})`;
|
|
2964
2980
|
};
|
|
2965
2981
|
const modelToProviderMap = {
|
|
@@ -3015,7 +3031,7 @@ const modelToProvider = (model) => {
|
|
|
3015
3031
|
const injectScript = (sessionRecordingOptions, stringifyCallbackArgs) => {
|
|
3016
3032
|
const BATCH_TIMEOUT = 2e3;
|
|
3017
3033
|
const HEARTBEAT_INTERVAL = 1e3;
|
|
3018
|
-
const CHUNK_SIZE =
|
|
3034
|
+
const CHUNK_SIZE = 262144;
|
|
3019
3035
|
window.lmnrRrwebEventsBatch = [];
|
|
3020
3036
|
window.lmnrChunkSequence = 0;
|
|
3021
3037
|
window.lmnrSendInProgress = false;
|
|
@@ -3076,7 +3092,9 @@ const injectScript = (sessionRecordingOptions, stringifyCallbackArgs) => {
|
|
|
3076
3092
|
try {
|
|
3077
3093
|
const compressedEvents = [];
|
|
3078
3094
|
for (const event of events) try {
|
|
3079
|
-
const
|
|
3095
|
+
const dataString = JSON.stringify(event.data);
|
|
3096
|
+
const compressed = await gzipCompress(dataString);
|
|
3097
|
+
const base64Data = await bufferToBase64(compressed);
|
|
3080
3098
|
compressedEvents.push({
|
|
3081
3099
|
...event,
|
|
3082
3100
|
data: base64Data
|
|
@@ -5326,15 +5344,17 @@ const applyGenerationSpanData = (lmnrSpan, spanData) => {
|
|
|
5326
5344
|
setGenAiInputMessages(lmnrSpan, inputData, getCurrentSystemInstructions());
|
|
5327
5345
|
const outputData = spanData?.output;
|
|
5328
5346
|
setGenAiOutputMessages(lmnrSpan, outputData);
|
|
5329
|
-
|
|
5347
|
+
const llmData = {
|
|
5330
5348
|
model: spanData?.model,
|
|
5331
5349
|
usage: spanData?.usage,
|
|
5332
5350
|
response_id: spanData?.response_id ?? spanData?.id
|
|
5333
|
-
}
|
|
5351
|
+
};
|
|
5352
|
+
applyLlmAttributes(lmnrSpan, llmData);
|
|
5334
5353
|
};
|
|
5335
5354
|
const applyResponseSpanData = (lmnrSpan, spanData) => {
|
|
5336
5355
|
const response = spanData?._response ?? spanData?.response;
|
|
5337
|
-
|
|
5356
|
+
const responseInput = spanData?._input ?? spanData?.input;
|
|
5357
|
+
setGenAiInputMessages(lmnrSpan, responseInput, getCurrentSystemInstructions());
|
|
5338
5358
|
if (response != null) {
|
|
5339
5359
|
setGenAiOutputMessagesFromResponse(lmnrSpan, response);
|
|
5340
5360
|
setToolDefinitionsFromResponse(lmnrSpan, response);
|
|
@@ -5375,8 +5395,10 @@ const applySpeechSpanData = (lmnrSpan, spanData) => {
|
|
|
5375
5395
|
const inputText = spanData?.input;
|
|
5376
5396
|
if (inputText) setGenAiInputMessages(lmnrSpan, inputText);
|
|
5377
5397
|
const outputData = spanData?.output;
|
|
5378
|
-
if (outputData)
|
|
5379
|
-
|
|
5398
|
+
if (outputData) {
|
|
5399
|
+
if (typeof outputData === "object" && !Array.isArray(outputData)) setGenAiOutputMessages(lmnrSpan, outputData.data);
|
|
5400
|
+
else setGenAiOutputMessages(lmnrSpan, outputData);
|
|
5401
|
+
}
|
|
5380
5402
|
};
|
|
5381
5403
|
const applyTranscriptionSpanData = (lmnrSpan, spanData) => {
|
|
5382
5404
|
const model = spanData?.model;
|
|
@@ -5386,8 +5408,10 @@ const applyTranscriptionSpanData = (lmnrSpan, spanData) => {
|
|
|
5386
5408
|
lmnrSpan.setAttribute(require_utils.LaminarAttributes.PROVIDER, "openai");
|
|
5387
5409
|
}
|
|
5388
5410
|
const inputData = spanData?.input;
|
|
5389
|
-
if (inputData)
|
|
5390
|
-
|
|
5411
|
+
if (inputData) {
|
|
5412
|
+
if (typeof inputData === "object" && !Array.isArray(inputData)) setGenAiInputMessages(lmnrSpan, inputData.data);
|
|
5413
|
+
else setGenAiInputMessages(lmnrSpan, inputData);
|
|
5414
|
+
}
|
|
5391
5415
|
const outputText = spanData?.output;
|
|
5392
5416
|
if (outputText) setGenAiOutputMessages(lmnrSpan, outputText);
|
|
5393
5417
|
};
|
|
@@ -5610,7 +5634,8 @@ const wrapGetResponse = (original) => function(request, ...rest) {
|
|
|
5610
5634
|
};
|
|
5611
5635
|
const wrapGetStreamedResponse = (original) => function(request, ...rest) {
|
|
5612
5636
|
const systemInstructions = request?.systemInstructions;
|
|
5613
|
-
|
|
5637
|
+
const source = original.call(this, request, ...rest);
|
|
5638
|
+
return wrapStreamWithSystemInstructions(systemInstructions, source);
|
|
5614
5639
|
};
|
|
5615
5640
|
const MODEL_METHODS = ["getResponse", "getStreamedResponse"];
|
|
5616
5641
|
var OpenAIAgentsInstrumentation = class extends _opentelemetry_instrumentation.InstrumentationBase {
|
|
@@ -5755,12 +5780,12 @@ var OpencodeInstrumentation = class extends _opentelemetry_instrumentation.Instr
|
|
|
5755
5780
|
};
|
|
5756
5781
|
//#endregion
|
|
5757
5782
|
//#region src/opentelemetry-lib/instrumentation/openrouter/utils.ts
|
|
5758
|
-
const TERMINAL_RESPONSE_EVENTS = new Set([
|
|
5783
|
+
const TERMINAL_RESPONSE_EVENTS = /* @__PURE__ */ new Set([
|
|
5759
5784
|
"response.completed",
|
|
5760
5785
|
"response.incomplete",
|
|
5761
5786
|
"response.failed"
|
|
5762
5787
|
]);
|
|
5763
|
-
const ERROR_RESPONSE_STATUSES = new Set(["failed", "incomplete"]);
|
|
5788
|
+
const ERROR_RESPONSE_STATUSES = /* @__PURE__ */ new Set(["failed", "incomplete"]);
|
|
5764
5789
|
const safeSetAttribute$1 = (span, key, value) => {
|
|
5765
5790
|
if (value === null || value === void 0) return;
|
|
5766
5791
|
if (typeof value === "string" && value.length === 0) return;
|
|
@@ -5926,7 +5951,8 @@ const wrapStream = (span, kind, stream, traceContent) => {
|
|
|
5926
5951
|
throw error;
|
|
5927
5952
|
} finally {
|
|
5928
5953
|
try {
|
|
5929
|
-
|
|
5954
|
+
const response = kind === "chat" ? aggregateChatChunks(chunks) : responseFromStreamEvents(chunks);
|
|
5955
|
+
setResponseAttributes$1(span, kind, response, traceContent);
|
|
5930
5956
|
} finally {
|
|
5931
5957
|
span.end();
|
|
5932
5958
|
}
|
|
@@ -6492,7 +6518,9 @@ const buildHeaders = (existing) => {
|
|
|
6492
6518
|
const headers = { ...existing };
|
|
6493
6519
|
if (laminarCtx) {
|
|
6494
6520
|
headers[require_consts.LAMINAR_SPAN_CONTEXT_HEADER] = encodePayload(JSON.stringify(laminarCtx));
|
|
6495
|
-
|
|
6521
|
+
const traceHex = laminarCtx.traceId.replace(/-/g, "");
|
|
6522
|
+
const spanHex = laminarCtx.spanId.replace(/-/g, "").slice(16);
|
|
6523
|
+
headers[require_consts.TRACEPARENT_HEADER] = encodePayload(`00-${traceHex}-${spanHex}-01`);
|
|
6496
6524
|
} else {
|
|
6497
6525
|
const ctx = rawSpan.spanContext();
|
|
6498
6526
|
headers[require_consts.TRACEPARENT_HEADER] = encodePayload(`00-${ctx.traceId}-${ctx.spanId}-01`);
|
|
@@ -6956,10 +6984,11 @@ const patchTemporalWorker = (workerModule, options = {}) => {
|
|
|
6956
6984
|
activity: [ActivityInterceptorFactory(options), ...rawOpts?.interceptors?.activity ?? []]
|
|
6957
6985
|
};
|
|
6958
6986
|
if (!usesWorkflowBundle) interceptors.workflowModules = [require.resolve("@lmnr-ai/lmnr/temporal-workflow-interceptors"), ...rawOpts?.interceptors?.workflowModules ?? []];
|
|
6959
|
-
|
|
6987
|
+
const patched = {
|
|
6960
6988
|
...rawOpts,
|
|
6961
6989
|
interceptors
|
|
6962
|
-
}
|
|
6990
|
+
};
|
|
6991
|
+
return originalCreate(patched, ...rest);
|
|
6963
6992
|
};
|
|
6964
6993
|
};
|
|
6965
6994
|
/**
|
|
@@ -7168,8 +7197,9 @@ const initializeLaminarInstrumentations = (options = {}) => {
|
|
|
7168
7197
|
if (apiKey) {
|
|
7169
7198
|
const url = options.baseUrl ?? process?.env?.LMNR_BASE_URL ?? "https://api.lmnr.ai";
|
|
7170
7199
|
const port = options.httpPort ?? (url.match(/:\d{1,5}$/g) ? parseInt(url.match(/:\d{1,5}$/g)[0].slice(1)) : 443);
|
|
7200
|
+
const urlWithoutSlash = url.replace(/\/$/, "").replace(/:\d{1,5}$/g, "");
|
|
7171
7201
|
client = new require_utils.LaminarClient({
|
|
7172
|
-
baseUrl: `${
|
|
7202
|
+
baseUrl: `${urlWithoutSlash}:${port}`,
|
|
7173
7203
|
projectApiKey: apiKey
|
|
7174
7204
|
});
|
|
7175
7205
|
}
|
|
@@ -7449,14 +7479,15 @@ const startTracing = (options) => {
|
|
|
7449
7479
|
});
|
|
7450
7480
|
};
|
|
7451
7481
|
const patchModules = (modules) => {
|
|
7482
|
+
const instrumentations = initializeLaminarInstrumentations({
|
|
7483
|
+
baseUrl: _baseHttpUrl,
|
|
7484
|
+
apiKey: _apiKey,
|
|
7485
|
+
instrumentModules: modules,
|
|
7486
|
+
suppressContentTracing: !shouldSendTraces(),
|
|
7487
|
+
sessionRecordingOptions: _configuration?.sessionRecordingOptions
|
|
7488
|
+
});
|
|
7452
7489
|
(0, _opentelemetry_instrumentation.registerInstrumentations)({
|
|
7453
|
-
instrumentations
|
|
7454
|
-
baseUrl: _baseHttpUrl,
|
|
7455
|
-
apiKey: _apiKey,
|
|
7456
|
-
instrumentModules: modules,
|
|
7457
|
-
suppressContentTracing: !shouldSendTraces(),
|
|
7458
|
-
sessionRecordingOptions: _configuration?.sessionRecordingOptions
|
|
7459
|
-
}),
|
|
7490
|
+
instrumentations,
|
|
7460
7491
|
tracerProvider
|
|
7461
7492
|
});
|
|
7462
7493
|
};
|
|
@@ -8303,165 +8334,10 @@ async function evaluate({ data, executor, evaluators, groupName, name, metadata,
|
|
|
8303
8334
|
//#region src/integrations/eve.ts
|
|
8304
8335
|
const logger$2 = require_utils.initializeLogger();
|
|
8305
8336
|
const EVE_REPORTER_TRACER_NAME = "@lmnr-ai/lmnr/eve-reporter";
|
|
8306
|
-
const
|
|
8307
|
-
const
|
|
8308
|
-
|
|
8309
|
-
const
|
|
8310
|
-
/**
|
|
8311
|
-
* Root span name for every propagated eval, and it never changes. eve calls
|
|
8312
|
-
* `ClientSession.send` before any hook that knows which eval is in flight, and
|
|
8313
|
-
* eve exposes no `onEvalStart`, so the id simply is not knowable here. Renaming
|
|
8314
|
-
* the span later would invalidate the `lmnr.span.path` the processor already
|
|
8315
|
-
* stamped on it and on every descendant, and would make the path we propagate
|
|
8316
|
-
* to the agent stale. The eval id rides on `lmnr.eve.eval.id`, on the trace
|
|
8317
|
-
* metadata, and on the datapoint instead.
|
|
8318
|
-
*/
|
|
8319
|
-
const ROOT_SPAN_NAME = "eve eval";
|
|
8320
|
-
const EXECUTOR_SPAN_NAME = "executor";
|
|
8321
|
-
/** Prototypes already patched — `send` must never be wrapped twice. */
|
|
8322
|
-
const patchedSendPrototypes = /* @__PURE__ */ new WeakSet();
|
|
8323
|
-
/**
|
|
8324
|
-
* Per-eve-client-session state. Keyed off the `ClientSession` instance rather
|
|
8325
|
-
* than an AsyncLocalStorage frame or a "current eval" global: eve runs up to
|
|
8326
|
-
* `maxConcurrency` evals at once, and each eval builds its own `ClientSession`
|
|
8327
|
-
* (`EvalSessionManager` calls `client.session()` per `EvalSessionDriver`), so
|
|
8328
|
-
* the instance is the only correct correlation key available inside `send`.
|
|
8329
|
-
*/
|
|
8330
|
-
const traceByClientSession = /* @__PURE__ */ new WeakMap();
|
|
8331
|
-
/** eve session id -> trace, read by `onEvalComplete` via `result.sessionId`. */
|
|
8332
|
-
const traceByEveSessionId = /* @__PURE__ */ new Map();
|
|
8333
|
-
/** Set by the running reporter's `onRunStart`; cleared by `onRunComplete`. */
|
|
8334
|
-
let activeSessionTraceFactory = null;
|
|
8335
|
-
/** Mirrors eve's own `normalizeSendTurnInput`, then merges the trace headers. */
|
|
8336
|
-
const withTraceHeaders = (input, headers) => {
|
|
8337
|
-
const payload = typeof input === "string" ? { message: input } : { ...input };
|
|
8338
|
-
payload.headers = {
|
|
8339
|
-
...payload.headers ?? {},
|
|
8340
|
-
...headers
|
|
8341
|
-
};
|
|
8342
|
-
return payload;
|
|
8343
|
-
};
|
|
8344
|
-
const openSessionTrace = (clientSession) => {
|
|
8345
|
-
try {
|
|
8346
|
-
const existing = traceByClientSession.get(clientSession);
|
|
8347
|
-
if (existing) return existing;
|
|
8348
|
-
const minted = activeSessionTraceFactory?.();
|
|
8349
|
-
if (minted) traceByClientSession.set(clientSession, minted);
|
|
8350
|
-
return minted;
|
|
8351
|
-
} catch (error) {
|
|
8352
|
-
logger$2.warn(`Laminar eve reporter: failed to open a session trace: ` + require_utils.errorMessage(error));
|
|
8353
|
-
return;
|
|
8354
|
-
}
|
|
8355
|
-
};
|
|
8356
|
-
/**
|
|
8357
|
-
* eve assigns the session id on the POST that `send` awaits, so the returned
|
|
8358
|
-
* `MessageResponse` is the earliest place the runner can bind the eve session
|
|
8359
|
-
* id to the trace it minted.
|
|
8360
|
-
*/
|
|
8361
|
-
const recordEveSessionId = (response, sessionTrace) => {
|
|
8362
|
-
const sessionId = response?.sessionId;
|
|
8363
|
-
if (typeof sessionId !== "string" || sessionId.length === 0) return;
|
|
8364
|
-
sessionTrace.sessionIds.add(sessionId);
|
|
8365
|
-
traceByEveSessionId.set(sessionId, sessionTrace);
|
|
8366
|
-
};
|
|
8367
|
-
/**
|
|
8368
|
-
* Build the propagation headers for the EXECUTOR span. Plain HTTP strings — eve
|
|
8369
|
-
* forwards `SendTurnPayload.headers` verbatim to `fetch`.
|
|
8370
|
-
*
|
|
8371
|
-
* The Laminar context carries the full path. Laminar nests spans by
|
|
8372
|
-
* `lmnr.span.ids_path`, not by OTel `parentSpanId`, so a receiver that adopts
|
|
8373
|
-
* only the ids lands its spans as a second root on our trace. The path is what
|
|
8374
|
-
* makes the agent's subtree render UNDER the executor. Both paths are constants
|
|
8375
|
-
* here — neither the root nor the executor span is ever renamed.
|
|
8376
|
-
*/
|
|
8377
|
-
const buildTraceHeaders = (rootSpan, executorSpan) => {
|
|
8378
|
-
const spanContext = executorSpan.spanContext();
|
|
8379
|
-
const laminarContext = {
|
|
8380
|
-
traceId: require_utils.otelTraceIdToUUID(spanContext.traceId),
|
|
8381
|
-
spanId: require_utils.otelSpanIdToUUID(spanContext.spanId),
|
|
8382
|
-
isRemote: true,
|
|
8383
|
-
spanPath: [ROOT_SPAN_NAME, EXECUTOR_SPAN_NAME],
|
|
8384
|
-
spanIdsPath: [require_utils.otelSpanIdToUUID(rootSpan.spanContext().spanId), require_utils.otelSpanIdToUUID(spanContext.spanId)]
|
|
8385
|
-
};
|
|
8386
|
-
return {
|
|
8387
|
-
[TRACEPARENT_HEADER$1]: `00-${spanContext.traceId}-${spanContext.spanId}-01`,
|
|
8388
|
-
[LAMINAR_SPAN_CONTEXT_HEADER$1]: JSON.stringify(laminarContext)
|
|
8389
|
-
};
|
|
8390
|
-
};
|
|
8391
|
-
/**
|
|
8392
|
-
* Bind this session's ROOT span into the calling eval's async context, so work
|
|
8393
|
-
* the eval does in the RUNNER process after `send` resolves lands on this trace
|
|
8394
|
-
* instead of opening its own. The concrete case is `t.judge.autoevals.*`: eve
|
|
8395
|
-
* runs the judge's model call in the runner, and Laminar's AI SDK integration
|
|
8396
|
-
* parents it to `LaminarContextManager.getContext()`.
|
|
8397
|
-
*
|
|
8398
|
-
* The root — not a pre-opened judge span — is the binding target on purpose.
|
|
8399
|
-
* An eval may record any number of judge assertions, and each one runs its own
|
|
8400
|
-
* model call. Binding the root gives every judge its own child of the root; a
|
|
8401
|
-
* single pre-opened parent would collect all of them under whichever judge
|
|
8402
|
-
* happened to run first.
|
|
8403
|
-
*
|
|
8404
|
-
* `pushContext` uses `AsyncLocalStorage.enterWith`, so this MUST run
|
|
8405
|
-
* synchronously before `send`'s first `await` — at that point we are still
|
|
8406
|
-
* executing in the caller's async context, so the store we set is the one
|
|
8407
|
-
* `test(t)` continues with. Each eval's `test(t)` runs in its own async chain,
|
|
8408
|
-
* so concurrent evals never see each other's root span; do NOT replace this
|
|
8409
|
-
* with the process-global context stack, which would cross-attribute.
|
|
8410
|
-
*
|
|
8411
|
-
* Judge spans reach Laminar only when the runner opts in — `Laminar.initialize()`
|
|
8412
|
-
* plus a globally registered AI SDK telemetry integration. Without that the
|
|
8413
|
-
* binding is inert, never wrong.
|
|
8414
|
-
*
|
|
8415
|
-
* Never popped: the store dies with the eval's async chain.
|
|
8416
|
-
*/
|
|
8417
|
-
const bindEvalContext = (sessionTrace) => {
|
|
8418
|
-
try {
|
|
8419
|
-
LaminarContextManager.pushContext(_opentelemetry_api.trace.setSpan(_opentelemetry_api.ROOT_CONTEXT, sessionTrace.rootSpan));
|
|
8420
|
-
} catch (error) {
|
|
8421
|
-
logger$2.warn(`Laminar eve reporter: failed to bind the eval context: ` + require_utils.errorMessage(error));
|
|
8422
|
-
}
|
|
8423
|
-
};
|
|
8424
|
-
/**
|
|
8425
|
-
* Wrap `ClientSession.prototype.send` so every eve turn carries the trace
|
|
8426
|
-
* context of a runner-minted Laminar span. Idempotent per prototype.
|
|
8427
|
-
*
|
|
8428
|
-
* @internal Exported for tests, which drive it with a stub class so the suite
|
|
8429
|
-
* keeps no dependency on eve.
|
|
8430
|
-
*/
|
|
8431
|
-
const patchEveClientSession = (sessionClass) => {
|
|
8432
|
-
const prototype = sessionClass?.prototype;
|
|
8433
|
-
const original = prototype?.send;
|
|
8434
|
-
if (!prototype || typeof original !== "function") return false;
|
|
8435
|
-
if (patchedSendPrototypes.has(prototype)) return true;
|
|
8436
|
-
prototype.send = async function(input) {
|
|
8437
|
-
const sessionTrace = openSessionTrace(this);
|
|
8438
|
-
if (!sessionTrace) return original.call(this, input);
|
|
8439
|
-
bindEvalContext(sessionTrace);
|
|
8440
|
-
const response = await original.call(this, withTraceHeaders(input, sessionTrace.headers));
|
|
8441
|
-
try {
|
|
8442
|
-
recordEveSessionId(response, sessionTrace);
|
|
8443
|
-
} catch (error) {
|
|
8444
|
-
logger$2.warn(`Laminar eve reporter: failed to record the eve session id: ` + require_utils.errorMessage(error));
|
|
8445
|
-
}
|
|
8446
|
-
return response;
|
|
8447
|
-
};
|
|
8448
|
-
patchedSendPrototypes.add(prototype);
|
|
8449
|
-
return true;
|
|
8450
|
-
};
|
|
8451
|
-
/**
|
|
8452
|
-
* Load eve's `ClientSession` without depending on eve. Mirrors the guarded
|
|
8453
|
-
* dynamic import in eve's own Braintrust reporter (`loadBraintrustSdk`), except
|
|
8454
|
-
* a missing eve degrades instead of throwing — the reporter must keep working
|
|
8455
|
-
* when it is registered outside an eve run.
|
|
8456
|
-
*/
|
|
8457
|
-
const loadEveClientSession = async () => {
|
|
8458
|
-
try {
|
|
8459
|
-
const sessionClass = (await import(EVE_CLIENT_MODULE)).ClientSession;
|
|
8460
|
-
return typeof sessionClass === "function" ? sessionClass : void 0;
|
|
8461
|
-
} catch {
|
|
8462
|
-
return;
|
|
8463
|
-
}
|
|
8464
|
-
};
|
|
8337
|
+
const OTEL_TRACE_ID_RE = /^[0-9a-f]{32}$/iu;
|
|
8338
|
+
const OTEL_SPAN_ID_RE = /^[0-9a-f]{16}$/iu;
|
|
8339
|
+
const ZERO_TRACE_ID = "0".repeat(32);
|
|
8340
|
+
const ZERO_SPAN_ID = "0".repeat(16);
|
|
8465
8341
|
const didAssertionPass = (assertion) => {
|
|
8466
8342
|
if (typeof assertion.passed === "boolean") return assertion.passed;
|
|
8467
8343
|
if (typeof assertion.score === "number") {
|
|
@@ -8470,14 +8346,8 @@ const didAssertionPass = (assertion) => {
|
|
|
8470
8346
|
}
|
|
8471
8347
|
return true;
|
|
8472
8348
|
};
|
|
8473
|
-
/**
|
|
8474
|
-
|
|
8475
|
-
* Detailed assertion outcomes live in metadata; sparse per-assertion score keys
|
|
8476
|
-
* make Laminar's eval UI look incomplete when different eval files assert
|
|
8477
|
-
* different things.
|
|
8478
|
-
*/
|
|
8479
|
-
const resultToScores = (result) => {
|
|
8480
|
-
const assertions = result.assertions ?? [];
|
|
8349
|
+
/** Keep Laminar's score columns stable across heterogeneous eve evals. */
|
|
8350
|
+
const resultToScores = (result, assertions) => {
|
|
8481
8351
|
const gateAssertions = assertions.filter((assertion) => assertion.severity === "gate");
|
|
8482
8352
|
const thresholdedSoftAssertions = assertions.filter((assertion) => assertion.severity === "soft" && typeof assertion.threshold === "number");
|
|
8483
8353
|
return {
|
|
@@ -8495,7 +8365,6 @@ const assertionsMetadata = (assertions) => assertions.map((assertion) => ({
|
|
|
8495
8365
|
message: assertion.message,
|
|
8496
8366
|
metadata: assertion.metadata
|
|
8497
8367
|
}));
|
|
8498
|
-
/** Surface failed assertions in metadata so a failing run is debuggable. */
|
|
8499
8368
|
const failedAssertionsMetadata = (assertions) => {
|
|
8500
8369
|
const failed = assertions.filter((assertion) => !didAssertionPass(assertion)).map((assertion) => ({
|
|
8501
8370
|
name: assertion.name,
|
|
@@ -8503,79 +8372,62 @@ const failedAssertionsMetadata = (assertions) => {
|
|
|
8503
8372
|
}));
|
|
8504
8373
|
return failed.length > 0 ? { failedAssertions: failed } : {};
|
|
8505
8374
|
};
|
|
8506
|
-
const
|
|
8375
|
+
const validTraceContext = (context) => context != null && typeof context.traceId === "string" && OTEL_TRACE_ID_RE.test(context.traceId) && context.traceId !== ZERO_TRACE_ID && typeof context.spanId === "string" && OTEL_SPAN_ID_RE.test(context.spanId) && context.spanId !== ZERO_SPAN_ID && typeof context.traceFlags === "number" && Number.isInteger(context.traceFlags) && context.traceFlags >= 0 && context.traceFlags <= 255;
|
|
8376
|
+
const normalizeTraceContexts = (contexts) => Array.isArray(contexts) ? contexts.filter(validTraceContext) : [];
|
|
8377
|
+
const isSampledTraceContext = (context) => (context.traceFlags & 1) === 1;
|
|
8378
|
+
const selectTraceContext = (contexts) => contexts.find((context) => context.primary === true && isSampledTraceContext(context)) ?? contexts.find(isSampledTraceContext);
|
|
8379
|
+
const traceContextMetadata = (contexts) => contexts.map((context) => ({
|
|
8380
|
+
traceId: context.traceId,
|
|
8381
|
+
spanId: context.spanId,
|
|
8382
|
+
traceFlags: context.traceFlags,
|
|
8383
|
+
primary: context.primary,
|
|
8384
|
+
sessionId: context.sessionId
|
|
8385
|
+
}));
|
|
8507
8386
|
/**
|
|
8508
|
-
*
|
|
8509
|
-
* or per-eval via the `reporters` field — eve runs and grades each eval, then
|
|
8510
|
-
* hands the graded result here, and this reporter ships it to Laminar as an
|
|
8511
|
-
* evaluation run with one datapoint per eval.
|
|
8512
|
-
*
|
|
8513
|
-
* The datapoint's trace id is never looked up — the reporter OWNS the trace.
|
|
8514
|
-
* `patchEveClientSession` wraps eve's `ClientSession.send` and mints the
|
|
8515
|
-
* EVALUATION root + EXECUTOR child on the first turn of each session, so the
|
|
8516
|
-
* runner already knows the trace id and pushes it to the agent as a
|
|
8517
|
-
* `traceparent` header. The eve session id is only the in-process key that ties
|
|
8518
|
-
* the send we instrumented to the graded result eve hands back later.
|
|
8519
|
-
*
|
|
8520
|
-
* Two outcomes, recorded on `metadata.traceResolution`:
|
|
8521
|
-
*
|
|
8522
|
-
* 1. `"propagated"` — the normal path, above. The agent joins the trace only
|
|
8523
|
-
* when `agent/instrumentation.ts` sets `traceChannelRequests: true` and the
|
|
8524
|
-
* agent process runs with `WORKFLOW_TRACE_MODE=continuous`.
|
|
8525
|
-
* 2. `"reporter-fallback"` — no session trace was minted (propagation disabled,
|
|
8526
|
-
* or eve produced no session id). The reporter opens its own EVALUATION span
|
|
8527
|
-
* so the grade still lands somewhere debuggable; it holds no agent work.
|
|
8528
|
-
*
|
|
8529
|
-
* `t.judge.autoevals.*` model calls run in the RUNNER process, and they reach
|
|
8530
|
-
* Laminar only when the runner registers an AI SDK telemetry integration. eve
|
|
8531
|
-
* imports `generateText` inside its own bundle, so `wrapAISDK` cannot reach it —
|
|
8532
|
-
* but AI SDK v7 reads registered integrations off `globalThis`, which does. One
|
|
8533
|
-
* line in `evals.config.ts`:
|
|
8387
|
+
* Reports eve eval results to Laminar.
|
|
8534
8388
|
*
|
|
8535
|
-
*
|
|
8536
|
-
*
|
|
8537
|
-
*
|
|
8389
|
+
* Eve 0.35+ gives reporters every native trace context observed during an eval.
|
|
8390
|
+
* The reporter links the datapoint to the first sampled primary trace (or the
|
|
8391
|
+
* first sampled trace when there is no sampled primary), records every valid
|
|
8392
|
+
* context in metadata, and adds
|
|
8393
|
+
* a small EVALUATION span to the selected native trace for the grade. When eve
|
|
8394
|
+
* reports no usable context, a reporter-owned fallback trace preserves the
|
|
8395
|
+
* result without pretending it contains agent work.
|
|
8538
8396
|
*
|
|
8539
|
-
*
|
|
8540
|
-
* the
|
|
8541
|
-
* Each judge gets its own span under the eval's root — see
|
|
8542
|
-
* {@link bindEvalContext}, and {@link LaminarReporter.flush} for how spans ship.
|
|
8543
|
-
* Without that line the reporter still records every assertion's score as an
|
|
8544
|
-
* EVALUATOR span; only the model call is missing.
|
|
8397
|
+
* The agent must export its native OpenTelemetry spans to the same Laminar
|
|
8398
|
+
* project. In the eve agent, declare a destination like:
|
|
8545
8399
|
*
|
|
8546
|
-
* @example
|
|
8547
8400
|
* ```ts
|
|
8548
|
-
* import {
|
|
8549
|
-
* import {
|
|
8401
|
+
* import { LaminarSpanProcessor } from "@lmnr-ai/lmnr";
|
|
8402
|
+
* import { otelIntegration } from "eve/instrumentation/otel";
|
|
8550
8403
|
*
|
|
8551
|
-
* export default
|
|
8552
|
-
*
|
|
8404
|
+
* export default otelIntegration({
|
|
8405
|
+
* spanProcessors: [new LaminarSpanProcessor()],
|
|
8553
8406
|
* });
|
|
8554
8407
|
* ```
|
|
8555
8408
|
*/
|
|
8556
8409
|
var LaminarReporter = class {
|
|
8557
8410
|
constructor(options = {}) {
|
|
8558
8411
|
this.evalsById = /* @__PURE__ */ new Map();
|
|
8559
|
-
this.
|
|
8560
|
-
this.
|
|
8561
|
-
this.sessionTraceFactory = () => this.mintSessionTrace();
|
|
8412
|
+
this.evalIndexById = /* @__PURE__ */ new Map();
|
|
8413
|
+
this.nextFallbackIndex = 0;
|
|
8562
8414
|
this.options = options;
|
|
8563
8415
|
}
|
|
8564
8416
|
async onRunStart(evaluations, target) {
|
|
8565
8417
|
this.evalId = void 0;
|
|
8566
|
-
|
|
8567
|
-
|
|
8568
|
-
|
|
8418
|
+
const identifiedEvaluations = evaluations.map((evaluation, index) => ({
|
|
8419
|
+
evaluation,
|
|
8420
|
+
index
|
|
8421
|
+
})).filter((item) => typeof item.evaluation.id === "string");
|
|
8422
|
+
this.evalsById = new Map(identifiedEvaluations.map(({ evaluation }) => [evaluation.id, evaluation]));
|
|
8423
|
+
this.evalIndexById = new Map(identifiedEvaluations.map(({ evaluation, index }) => [evaluation.id, index]));
|
|
8424
|
+
this.nextFallbackIndex = evaluations.length;
|
|
8569
8425
|
try {
|
|
8570
8426
|
this.client = this.options.client ?? new require_utils.LaminarClient({
|
|
8571
8427
|
baseUrl: this.options.baseUrl,
|
|
8572
8428
|
projectApiKey: this.options.projectApiKey
|
|
8573
8429
|
});
|
|
8574
8430
|
this.ensureTracing();
|
|
8575
|
-
if (this.options.propagateTraceContext !== false) {
|
|
8576
|
-
activeSessionTraceFactory = this.sessionTraceFactory;
|
|
8577
|
-
patchEveClientSession(await loadEveClientSession());
|
|
8578
|
-
}
|
|
8579
8431
|
const evaluation = await this.client.evals.init(this.options.name, this.options.groupName, {
|
|
8580
8432
|
...this.options.metadata ?? {},
|
|
8581
8433
|
source: "eve",
|
|
@@ -8588,26 +8440,25 @@ var LaminarReporter = class {
|
|
|
8588
8440
|
process.stdout.write(`\nLaminar: check eve eval results at ${url}\n`);
|
|
8589
8441
|
} catch (error) {
|
|
8590
8442
|
logger$2.error(`Laminar eve reporter: failed to start run: ${require_utils.errorMessage(error)}`);
|
|
8591
|
-
if (!this.evalId)
|
|
8592
|
-
this.client = void 0;
|
|
8593
|
-
if (activeSessionTraceFactory === this.sessionTraceFactory) activeSessionTraceFactory = null;
|
|
8594
|
-
}
|
|
8443
|
+
if (!this.evalId) this.client = void 0;
|
|
8595
8444
|
}
|
|
8596
8445
|
}
|
|
8597
|
-
async onEvalComplete(result) {
|
|
8446
|
+
async onEvalComplete(result, context) {
|
|
8598
8447
|
const client = this.client;
|
|
8599
8448
|
const evalId = this.evalId;
|
|
8600
8449
|
if (!client || !evalId) return;
|
|
8601
|
-
const index = this.
|
|
8450
|
+
const index = typeof result.id === "string" ? this.evalIndexById.get(result.id) ?? this.nextFallbackIndex++ : this.nextFallbackIndex++;
|
|
8602
8451
|
const task = result.result ?? {};
|
|
8603
8452
|
const derived = task.derived ?? {};
|
|
8604
|
-
const assertions = result.assertions
|
|
8605
|
-
const
|
|
8606
|
-
const
|
|
8607
|
-
|
|
8453
|
+
const assertions = Array.isArray(result.assertions) ? result.assertions.filter((assertion) => assertion != null && typeof assertion === "object") : [];
|
|
8454
|
+
const evalDefinition = typeof result.id === "string" ? this.evalsById.get(result.id) : void 0;
|
|
8455
|
+
const evalDescription = context?.evaluation?.description ?? evalDefinition?.description;
|
|
8456
|
+
const contexts = normalizeTraceContexts(context?.traceContexts ?? task.traceContexts ?? []);
|
|
8457
|
+
const gradeTrace = this.openGradeTrace({
|
|
8458
|
+
contexts,
|
|
8608
8459
|
description: evalDescription,
|
|
8609
|
-
|
|
8610
|
-
|
|
8460
|
+
evalId: result.id ?? String(index),
|
|
8461
|
+
verdict: result.verdict
|
|
8611
8462
|
});
|
|
8612
8463
|
try {
|
|
8613
8464
|
const datapointId = await client.evals.createDatapoint({
|
|
@@ -8615,17 +8466,19 @@ var LaminarReporter = class {
|
|
|
8615
8466
|
data: result.id ?? null,
|
|
8616
8467
|
target: task.finalMessage ?? null,
|
|
8617
8468
|
index,
|
|
8618
|
-
traceId:
|
|
8469
|
+
traceId: gradeTrace.traceId,
|
|
8619
8470
|
metadata: {
|
|
8620
8471
|
name: result.id,
|
|
8621
8472
|
verdict: result.verdict,
|
|
8622
8473
|
status: task.status,
|
|
8623
8474
|
description: evalDescription,
|
|
8624
8475
|
sessionId: task.sessionId,
|
|
8625
|
-
traceResolution:
|
|
8476
|
+
traceResolution: gradeTrace.source,
|
|
8477
|
+
selectedTraceContext: gradeTrace.selected ? traceContextMetadata([gradeTrace.selected])[0] : void 0,
|
|
8478
|
+
traceContexts: traceContextMetadata(gradeTrace.contexts),
|
|
8626
8479
|
skipReason: result.skipReason,
|
|
8627
8480
|
error: result.error,
|
|
8628
|
-
toolCalls: (derived.toolCalls
|
|
8481
|
+
toolCalls: (Array.isArray(derived.toolCalls) ? derived.toolCalls : []).map((call) => call?.name).filter((name) => typeof name === "string"),
|
|
8629
8482
|
toolCallCount: derived.toolCallCount,
|
|
8630
8483
|
parked: derived.parked,
|
|
8631
8484
|
failureCode: derived.failureCode,
|
|
@@ -8634,44 +8487,30 @@ var LaminarReporter = class {
|
|
|
8634
8487
|
...failedAssertionsMetadata(assertions)
|
|
8635
8488
|
}
|
|
8636
8489
|
});
|
|
8637
|
-
const executorOutput = task.output ?? task.finalMessage ?? null;
|
|
8638
8490
|
await client.evals.updateDatapoint({
|
|
8639
8491
|
evalId,
|
|
8640
8492
|
datapointId,
|
|
8641
|
-
scores: resultToScores(result),
|
|
8642
|
-
executorOutput
|
|
8493
|
+
scores: resultToScores(result, assertions),
|
|
8494
|
+
executorOutput: task.output ?? task.finalMessage ?? null
|
|
8643
8495
|
});
|
|
8644
8496
|
} catch (error) {
|
|
8645
|
-
|
|
8497
|
+
gradeTrace.span.recordException(error);
|
|
8646
8498
|
logger$2.error(`Laminar eve reporter: failed to report eval "${result.id ?? index}": ${require_utils.errorMessage(error)}`);
|
|
8647
8499
|
} finally {
|
|
8648
|
-
|
|
8649
|
-
|
|
8650
|
-
evalId: result.id,
|
|
8651
|
-
description: evalDescription,
|
|
8652
|
-
verdict: result.verdict,
|
|
8653
|
-
status: task.status,
|
|
8500
|
+
this.finishGradeTrace(gradeTrace, {
|
|
8501
|
+
assertions,
|
|
8654
8502
|
output: task.output ?? task.finalMessage ?? null,
|
|
8655
|
-
|
|
8503
|
+
status: task.status
|
|
8656
8504
|
});
|
|
8657
8505
|
await this.flush();
|
|
8658
8506
|
}
|
|
8659
8507
|
}
|
|
8660
|
-
async onRunComplete() {
|
|
8661
|
-
this.finishOpenSessionTraces();
|
|
8662
|
-
if (activeSessionTraceFactory === this.sessionTraceFactory) activeSessionTraceFactory = null;
|
|
8508
|
+
async onRunComplete(_summary) {
|
|
8663
8509
|
await this.flush();
|
|
8664
8510
|
this.evalId = void 0;
|
|
8665
8511
|
this.evalsById.clear();
|
|
8512
|
+
this.evalIndexById.clear();
|
|
8666
8513
|
}
|
|
8667
|
-
/**
|
|
8668
|
-
* Point the process's Laminar tracing at this reporter's project, unless
|
|
8669
|
-
* something already initialized it. Mirrors `LaminarAiSdkTelemetry`, which
|
|
8670
|
-
* initializes the same way for judge spans — so eve eval spans and judge
|
|
8671
|
-
* spans share ONE tracer provider however the run was set up. Without this,
|
|
8672
|
-
* `getTracer()` returns a noop whenever nobody registered the AI SDK
|
|
8673
|
-
* integration, and every eve span would be dropped silently.
|
|
8674
|
-
*/
|
|
8675
8514
|
ensureTracing() {
|
|
8676
8515
|
if (Laminar.initialized()) {
|
|
8677
8516
|
const ourKey = this.client?.apiKey ?? this.options.projectApiKey;
|
|
@@ -8687,140 +8526,79 @@ var LaminarReporter = class {
|
|
|
8687
8526
|
disableBatch: true
|
|
8688
8527
|
});
|
|
8689
8528
|
}
|
|
8690
|
-
/**
|
|
8691
|
-
* Flush the process's Laminar pipeline — eve eval spans AND judge spans, which
|
|
8692
|
-
* now share one provider. eve ends `eve eval` with `process.exit()` (skipping
|
|
8693
|
-
* `beforeExit`), so these calls are the only points at which spans ship.
|
|
8694
|
-
* Never `shutdown()`: the pipeline may belong to the host, not to us.
|
|
8695
|
-
*/
|
|
8696
8529
|
async flush() {
|
|
8697
8530
|
try {
|
|
8698
8531
|
await Laminar.flush();
|
|
8699
8532
|
} catch (error) {
|
|
8700
|
-
logger$2.warn(`Laminar eve reporter: failed to flush spans:
|
|
8533
|
+
logger$2.warn(`Laminar eve reporter: failed to flush spans: ${require_utils.errorMessage(error)}`);
|
|
8701
8534
|
}
|
|
8702
8535
|
}
|
|
8703
|
-
|
|
8704
|
-
|
|
8705
|
-
* root plus the EXECUTOR child the agent's turn is parented to. Called from
|
|
8706
|
-
* the patched `send`, so it must never throw into the user's eval.
|
|
8707
|
-
*/
|
|
8708
|
-
mintSessionTrace() {
|
|
8536
|
+
openGradeTrace({ contexts, description, evalId, verdict }) {
|
|
8537
|
+
const selected = selectTraceContext(contexts);
|
|
8709
8538
|
const tracer = getTracer();
|
|
8710
|
-
const
|
|
8711
|
-
|
|
8712
|
-
|
|
8713
|
-
"lmnr.eve.reporter": EVE_REPORTER_TRACER_NAME
|
|
8714
|
-
} }, _opentelemetry_api.ROOT_CONTEXT);
|
|
8715
|
-
const executorSpan = tracer.startSpan(EXECUTOR_SPAN_NAME, { attributes: {
|
|
8716
|
-
[require_utils.SPAN_TYPE]: "EXECUTOR",
|
|
8717
|
-
[require_utils.TRACE_TYPE]: "EVALUATION"
|
|
8718
|
-
} }, _opentelemetry_api.trace.setSpan(_opentelemetry_api.ROOT_CONTEXT, rootSpan));
|
|
8719
|
-
const sessionTrace = {
|
|
8720
|
-
traceId: require_utils.otelTraceIdToUUID(rootSpan.spanContext().traceId),
|
|
8721
|
-
rootSpan,
|
|
8722
|
-
executorSpan,
|
|
8723
|
-
tracer,
|
|
8724
|
-
headers: buildTraceHeaders(rootSpan, executorSpan),
|
|
8725
|
-
sessionIds: /* @__PURE__ */ new Set(),
|
|
8726
|
-
ended: false
|
|
8727
|
-
};
|
|
8728
|
-
this.openSessionTraces.add(sessionTrace);
|
|
8729
|
-
return sessionTrace;
|
|
8730
|
-
}
|
|
8731
|
-
/** Grade, close and release one session trace. Idempotent. */
|
|
8732
|
-
finishSessionTrace(sessionTrace, grade) {
|
|
8733
|
-
if (sessionTrace.ended) return;
|
|
8734
|
-
sessionTrace.ended = true;
|
|
8735
|
-
this.openSessionTraces.delete(sessionTrace);
|
|
8736
|
-
for (const sessionId of sessionTrace.sessionIds) traceByEveSessionId.delete(sessionId);
|
|
8737
|
-
try {
|
|
8738
|
-
sessionTrace.rootSpan.setAttributes({
|
|
8739
|
-
"lmnr.eve.eval.id": grade.evalId ?? "",
|
|
8740
|
-
"lmnr.eve.eval.verdict": grade.verdict ?? "",
|
|
8741
|
-
"lmnr.eve.eval.status": grade.status ?? "",
|
|
8742
|
-
[require_utils.SPAN_INPUT]: JSON.stringify({
|
|
8743
|
-
evalId: grade.evalId,
|
|
8744
|
-
description: grade.description
|
|
8745
|
-
}),
|
|
8746
|
-
[require_utils.SPAN_OUTPUT]: JSON.stringify(grade.output ?? null),
|
|
8747
|
-
...require_utils.metadataToAttributes({
|
|
8748
|
-
source: "eve",
|
|
8749
|
-
eveEvalId: grade.evalId ?? "",
|
|
8750
|
-
eveEvalDescription: grade.description ?? "",
|
|
8751
|
-
eveEvalVerdict: grade.verdict ?? "",
|
|
8752
|
-
eveSessionId: [...sessionTrace.sessionIds].join(",")
|
|
8753
|
-
})
|
|
8754
|
-
});
|
|
8755
|
-
sessionTrace.executorSpan.setAttribute(require_utils.SPAN_OUTPUT, JSON.stringify(grade.output ?? null));
|
|
8756
|
-
this.recordEvaluatorSpans(sessionTrace, grade.assertions ?? []);
|
|
8757
|
-
sessionTrace.executorSpan.end();
|
|
8758
|
-
sessionTrace.rootSpan.end();
|
|
8759
|
-
} catch (error) {
|
|
8760
|
-
logger$2.warn(`Laminar eve reporter: failed to finish the session trace: ` + require_utils.errorMessage(error));
|
|
8761
|
-
}
|
|
8762
|
-
}
|
|
8763
|
-
/**
|
|
8764
|
-
* One EVALUATOR span per eve assertion, judge or not. They are created at
|
|
8765
|
-
* grade time (the assertions only exist then) as short children of the
|
|
8766
|
-
* still-open root. No explicit parent path is needed: the root keeps the name
|
|
8767
|
-
* it was minted with, so the processor's cached path for it is still correct.
|
|
8768
|
-
*/
|
|
8769
|
-
recordEvaluatorSpans(sessionTrace, assertions) {
|
|
8770
|
-
const rootContext = _opentelemetry_api.trace.setSpan(_opentelemetry_api.ROOT_CONTEXT, sessionTrace.rootSpan);
|
|
8771
|
-
for (const assertion of assertions) {
|
|
8772
|
-
const name = assertion.name ?? "assertion";
|
|
8773
|
-
const output = JSON.stringify({
|
|
8774
|
-
score: assertion.score ?? null,
|
|
8775
|
-
passed: didAssertionPass(assertion),
|
|
8776
|
-
severity: assertion.severity,
|
|
8777
|
-
threshold: assertion.threshold,
|
|
8778
|
-
message: assertion.message
|
|
8779
|
-
});
|
|
8780
|
-
sessionTrace.tracer.startSpan(name, { attributes: {
|
|
8781
|
-
[require_utils.SPAN_TYPE]: "EVALUATOR",
|
|
8782
|
-
[require_utils.TRACE_TYPE]: "EVALUATION",
|
|
8783
|
-
[require_utils.SPAN_OUTPUT]: output
|
|
8784
|
-
} }, rootContext).end();
|
|
8785
|
-
}
|
|
8786
|
-
}
|
|
8787
|
-
finishOpenSessionTraces() {
|
|
8788
|
-
for (const sessionTrace of [...this.openSessionTraces]) this.finishSessionTrace(sessionTrace, {});
|
|
8789
|
-
}
|
|
8790
|
-
/**
|
|
8791
|
-
* Resolve the trace this datapoint links to. There is no lookup: either
|
|
8792
|
-
* `patchEveClientSession` already minted the session's trace (and we know its
|
|
8793
|
-
* id), or we open a reporter-owned EVALUATION span to hold the grade.
|
|
8794
|
-
*/
|
|
8795
|
-
resolveDatapointTrace({ evalId, description, verdict, sessionId }) {
|
|
8796
|
-
const sessionTrace = sessionId ? traceByEveSessionId.get(sessionId) : void 0;
|
|
8797
|
-
if (sessionTrace) return {
|
|
8798
|
-
traceId: sessionTrace.traceId,
|
|
8799
|
-
source: "propagated",
|
|
8800
|
-
sessionTrace
|
|
8801
|
-
};
|
|
8802
|
-
const span = getTracer().startSpan(`eve eval ${evalId}`, { attributes: {
|
|
8539
|
+
const parentContext = selected ? _opentelemetry_api.trace.setSpanContext(_opentelemetry_api.ROOT_CONTEXT, this.toSpanContext(selected)) : _opentelemetry_api.ROOT_CONTEXT;
|
|
8540
|
+
const source = selected ? "eve-native" : "reporter-fallback";
|
|
8541
|
+
const span = tracer.startSpan(`eve eval ${evalId}`, { attributes: {
|
|
8803
8542
|
[require_utils.SPAN_TYPE]: "EVALUATION",
|
|
8804
8543
|
[require_utils.TRACE_TYPE]: "EVALUATION",
|
|
8805
8544
|
"lmnr.eve.reporter": EVE_REPORTER_TRACER_NAME,
|
|
8806
8545
|
"lmnr.eve.eval.id": evalId,
|
|
8807
8546
|
"lmnr.eve.eval.verdict": verdict ?? "",
|
|
8808
|
-
|
|
8809
|
-
"lmnr.eve.trace_resolution": "reporter-fallback",
|
|
8547
|
+
"lmnr.eve.trace_resolution": source,
|
|
8810
8548
|
...require_utils.metadataToAttributes({
|
|
8811
8549
|
source: "eve",
|
|
8812
8550
|
eveEvalId: evalId,
|
|
8813
8551
|
eveEvalDescription: description ?? "",
|
|
8814
8552
|
eveEvalVerdict: verdict ?? "",
|
|
8815
|
-
eveSessionId: sessionId ?? ""
|
|
8553
|
+
eveSessionId: selected?.sessionId ?? "",
|
|
8554
|
+
eveTraceCount: contexts.length
|
|
8555
|
+
}),
|
|
8556
|
+
[require_utils.SPAN_INPUT]: JSON.stringify({
|
|
8557
|
+
evalId,
|
|
8558
|
+
description
|
|
8816
8559
|
})
|
|
8817
|
-
} },
|
|
8560
|
+
} }, parentContext);
|
|
8561
|
+
return {
|
|
8562
|
+
traceId: require_utils.otelTraceIdToUUID(span.spanContext().traceId),
|
|
8563
|
+
source,
|
|
8564
|
+
span,
|
|
8565
|
+
tracer,
|
|
8566
|
+
selected,
|
|
8567
|
+
contexts
|
|
8568
|
+
};
|
|
8569
|
+
}
|
|
8570
|
+
toSpanContext(context) {
|
|
8818
8571
|
return {
|
|
8819
|
-
traceId:
|
|
8820
|
-
|
|
8821
|
-
|
|
8572
|
+
traceId: context.traceId.toLowerCase(),
|
|
8573
|
+
spanId: context.spanId.toLowerCase(),
|
|
8574
|
+
traceFlags: context.traceFlags,
|
|
8575
|
+
isRemote: true
|
|
8822
8576
|
};
|
|
8823
8577
|
}
|
|
8578
|
+
finishGradeTrace(gradeTrace, grade) {
|
|
8579
|
+
try {
|
|
8580
|
+
gradeTrace.span.setAttributes({
|
|
8581
|
+
"lmnr.eve.eval.status": grade.status ?? "",
|
|
8582
|
+
[require_utils.SPAN_OUTPUT]: JSON.stringify(grade.output ?? null)
|
|
8583
|
+
});
|
|
8584
|
+
const parentContext = _opentelemetry_api.trace.setSpan(_opentelemetry_api.ROOT_CONTEXT, gradeTrace.span);
|
|
8585
|
+
for (const assertion of grade.assertions) gradeTrace.tracer.startSpan(assertion.name ?? "assertion", { attributes: {
|
|
8586
|
+
[require_utils.SPAN_TYPE]: "EVALUATOR",
|
|
8587
|
+
[require_utils.TRACE_TYPE]: "EVALUATION",
|
|
8588
|
+
[require_utils.SPAN_OUTPUT]: JSON.stringify({
|
|
8589
|
+
score: assertion.score ?? null,
|
|
8590
|
+
passed: didAssertionPass(assertion),
|
|
8591
|
+
severity: assertion.severity,
|
|
8592
|
+
threshold: assertion.threshold,
|
|
8593
|
+
message: assertion.message
|
|
8594
|
+
})
|
|
8595
|
+
} }, parentContext).end();
|
|
8596
|
+
} catch (error) {
|
|
8597
|
+
logger$2.warn(`Laminar eve reporter: failed to finish the grade trace: ${require_utils.errorMessage(error)}`);
|
|
8598
|
+
} finally {
|
|
8599
|
+
gradeTrace.span.end();
|
|
8600
|
+
}
|
|
8601
|
+
}
|
|
8824
8602
|
};
|
|
8825
8603
|
//#endregion
|
|
8826
8604
|
//#region src/debug/hash.ts
|
|
@@ -8875,8 +8653,10 @@ const stripSystemMessages = (messages) => messages.filter((m) => !(m !== null &&
|
|
|
8875
8653
|
* hex-encoded (64 chars).
|
|
8876
8654
|
*/
|
|
8877
8655
|
const debugInputHash = (messages) => {
|
|
8878
|
-
const
|
|
8879
|
-
|
|
8656
|
+
const stripped = stripSystemMessages(messages);
|
|
8657
|
+
const canonical = canonicalJson(stripped);
|
|
8658
|
+
const digest = (0, _noble_hashes_blake3_js.blake3)(new TextEncoder().encode(canonical));
|
|
8659
|
+
return (0, _noble_hashes_utils_js.bytesToHex)(digest);
|
|
8880
8660
|
};
|
|
8881
8661
|
//#endregion
|
|
8882
8662
|
//#region src/debug/replay.ts
|
|
@@ -8992,6 +8772,28 @@ const applyUsageToSpan = (span, usage) => {
|
|
|
8992
8772
|
applyUsage(attrs, usage);
|
|
8993
8773
|
for (const [k, v] of Object.entries(attrs)) if (v !== void 0) span.setAttribute(k, v);
|
|
8994
8774
|
};
|
|
8775
|
+
const resolveAssociationProperties = (event, parent) => {
|
|
8776
|
+
const { userId, sessionId, tags, ...metadata } = {
|
|
8777
|
+
...event?.toolsContext ?? {},
|
|
8778
|
+
...event?.runtimeContext ?? {}
|
|
8779
|
+
};
|
|
8780
|
+
return {
|
|
8781
|
+
associationProperties: {
|
|
8782
|
+
userId: typeof userId === "string" && userId.length > 0 ? userId : parent.userId,
|
|
8783
|
+
sessionId: typeof sessionId === "string" && sessionId.length > 0 ? sessionId : parent.sessionId,
|
|
8784
|
+
metadata: {
|
|
8785
|
+
...parent.metadata ?? {},
|
|
8786
|
+
...metadata
|
|
8787
|
+
}
|
|
8788
|
+
},
|
|
8789
|
+
tags: Array.isArray(tags) && tags.every((tag) => typeof tag === "string") ? tags : void 0
|
|
8790
|
+
};
|
|
8791
|
+
};
|
|
8792
|
+
const applyAssociationProperties = (span, properties) => {
|
|
8793
|
+
if (properties.userId) span.setAttribute(require_utils.USER_ID, properties.userId);
|
|
8794
|
+
if (properties.sessionId) span.setAttribute(require_utils.SESSION_ID, properties.sessionId);
|
|
8795
|
+
span.setAttributes(require_utils.metadataToAttributes(properties.metadata));
|
|
8796
|
+
};
|
|
8995
8797
|
const applyFinishReason = (span, finishReason) => {
|
|
8996
8798
|
span.setAttribute("gen_ai.response.finish_reason", finishReason);
|
|
8997
8799
|
span.setAttribute("ai.response.finishReason", finishReason);
|
|
@@ -9025,6 +8827,17 @@ const verbatimPromptString = (prompt) => {
|
|
|
9025
8827
|
return null;
|
|
9026
8828
|
}
|
|
9027
8829
|
};
|
|
8830
|
+
const buildDecisionInputMessages = (state) => {
|
|
8831
|
+
if (state === void 0 || state === null) return void 0;
|
|
8832
|
+
if (Array.isArray(state)) return verbatimPromptString([{
|
|
8833
|
+
role: "user",
|
|
8834
|
+
content: state
|
|
8835
|
+
}]) ?? void 0;
|
|
8836
|
+
return serializeJSON$1([{
|
|
8837
|
+
role: "user",
|
|
8838
|
+
content: typeof state === "string" ? state : serializeJSON$1(state)
|
|
8839
|
+
}]);
|
|
8840
|
+
};
|
|
9028
8841
|
const verbatimPromptMessages = (prompt) => {
|
|
9029
8842
|
const serialized = verbatimPromptString(prompt);
|
|
9030
8843
|
if (serialized === null) return null;
|
|
@@ -9680,7 +9493,6 @@ var LaminarAiSdkTelemetry = class {
|
|
|
9680
9493
|
} : {}
|
|
9681
9494
|
}
|
|
9682
9495
|
}, parentCtx);
|
|
9683
|
-
const spanCtx = _opentelemetry_api.trace.setSpan(parentCtx, span);
|
|
9684
9496
|
span.setAttribute(require_utils.SPAN_TYPE, "DEFAULT");
|
|
9685
9497
|
if (typeof event.operationId === "string") span.setAttribute("ai.operation", event.operationId);
|
|
9686
9498
|
if (typeof event.provider === "string") {
|
|
@@ -9693,14 +9505,14 @@ var LaminarAiSdkTelemetry = class {
|
|
|
9693
9505
|
}
|
|
9694
9506
|
if (typeof event.functionId === "string") span.setAttribute("ai.functionId", event.functionId);
|
|
9695
9507
|
if (event.operationId === "ai.embed" || event.operationId === "ai.embedMany" || event.operationId === "ai.rerank") span.setAttribute(require_utils.SPAN_TYPE, "LLM");
|
|
9696
|
-
const
|
|
9697
|
-
|
|
9698
|
-
|
|
9699
|
-
};
|
|
9700
|
-
|
|
9701
|
-
|
|
9702
|
-
|
|
9703
|
-
span
|
|
9508
|
+
const parentAssociationProperties = parentCtx.getValue(ASSOCIATION_PROPERTIES_KEY) ?? {};
|
|
9509
|
+
const { associationProperties, tags } = resolveAssociationProperties(event, parentAssociationProperties);
|
|
9510
|
+
applyAssociationProperties(span, associationProperties);
|
|
9511
|
+
if (tags) span.setAttribute(`${require_utils.ASSOCIATION_PROPERTIES}.tags`, tags);
|
|
9512
|
+
const spanCtx = _opentelemetry_api.trace.setSpan(parentCtx.setValue(ASSOCIATION_PROPERTIES_KEY, {
|
|
9513
|
+
...parentAssociationProperties,
|
|
9514
|
+
...Object.fromEntries(Object.entries(associationProperties).filter(([, value]) => value !== void 0))
|
|
9515
|
+
}), span);
|
|
9704
9516
|
if (this.recordInputs) {
|
|
9705
9517
|
if (Array.isArray(event.messages) || event.system !== void 0 || event.instructions !== void 0) {
|
|
9706
9518
|
const msgs = verbatimStandardizedMessages(event);
|
|
@@ -9716,7 +9528,8 @@ var LaminarAiSdkTelemetry = class {
|
|
|
9716
9528
|
ctx: spanCtx,
|
|
9717
9529
|
operationId: event.operationId ?? "unknown",
|
|
9718
9530
|
provider: event.provider,
|
|
9719
|
-
modelId: event.modelId
|
|
9531
|
+
modelId: event.modelId,
|
|
9532
|
+
associationProperties
|
|
9720
9533
|
});
|
|
9721
9534
|
};
|
|
9722
9535
|
this.onStepStart = (event) => {
|
|
@@ -9734,6 +9547,7 @@ var LaminarAiSdkTelemetry = class {
|
|
|
9734
9547
|
const spanCtx = _opentelemetry_api.trace.setSpan(op.ctx, span);
|
|
9735
9548
|
span.setAttribute(require_utils.SPAN_TYPE, "DEFAULT");
|
|
9736
9549
|
span.setAttributes(buildAiSdkInstrumentationAttributes());
|
|
9550
|
+
applyAssociationProperties(span, op.associationProperties);
|
|
9737
9551
|
span.setAttribute("ai.step.number", stepNumber);
|
|
9738
9552
|
if (this.recordInputs) {
|
|
9739
9553
|
const msgs = verbatimStandardizedMessages(event);
|
|
@@ -9750,11 +9564,13 @@ var LaminarAiSdkTelemetry = class {
|
|
|
9750
9564
|
if (!callId) return;
|
|
9751
9565
|
const step = this.findLatestStep(callId);
|
|
9752
9566
|
const stepNumber = step?.stepNumber ?? event?.stepNumber ?? 0;
|
|
9753
|
-
const
|
|
9754
|
-
|
|
9567
|
+
const op = this.operationByCallId.get(callId);
|
|
9568
|
+
const parentCtx = step?.ctx ?? op?.ctx;
|
|
9569
|
+
if (!op || !parentCtx) return;
|
|
9755
9570
|
const span = getTracer().startSpan(`ai.llm ${event.provider ?? ""}:${event.modelId ?? ""}`, { kind: _opentelemetry_api.SpanKind.CLIENT }, parentCtx);
|
|
9756
9571
|
span.setAttribute(require_utils.SPAN_TYPE, "LLM");
|
|
9757
9572
|
span.setAttributes(buildAiSdkInstrumentationAttributes());
|
|
9573
|
+
applyAssociationProperties(span, op.associationProperties);
|
|
9758
9574
|
applyRequestModelAttributes(span, event);
|
|
9759
9575
|
if (this.recordInputs) {
|
|
9760
9576
|
if (event.tools) try {
|
|
@@ -9851,6 +9667,7 @@ var LaminarAiSdkTelemetry = class {
|
|
|
9851
9667
|
const span = getTracer().startSpan(`ai.llm ${event.provider ?? ""}:${event.modelId ?? ""}`, { kind: _opentelemetry_api.SpanKind.CLIENT }, op.ctx);
|
|
9852
9668
|
span.setAttribute(require_utils.SPAN_TYPE, "LLM");
|
|
9853
9669
|
span.setAttributes(buildAiSdkInstrumentationAttributes());
|
|
9670
|
+
applyAssociationProperties(span, op.associationProperties);
|
|
9854
9671
|
applyRequestModelAttributes(span, event);
|
|
9855
9672
|
if (this.recordInputs && Array.isArray(event?.promptMessages)) {
|
|
9856
9673
|
const serialized = verbatimPromptString(event.promptMessages);
|
|
@@ -9889,12 +9706,15 @@ var LaminarAiSdkTelemetry = class {
|
|
|
9889
9706
|
const toolCallId = toolCall?.toolCallId;
|
|
9890
9707
|
const toolName = toolCall?.toolName;
|
|
9891
9708
|
if (!callId || !toolCallId) return;
|
|
9892
|
-
const
|
|
9893
|
-
|
|
9709
|
+
const step = this.findLatestStep(callId);
|
|
9710
|
+
const op = this.operationByCallId.get(callId);
|
|
9711
|
+
const parentCtx = step?.ctx ?? op?.ctx;
|
|
9712
|
+
if (!op || !parentCtx) return;
|
|
9894
9713
|
const span = getTracer().startSpan(`ai.tool ${toolName ?? "unknown"}`, { kind: _opentelemetry_api.SpanKind.CLIENT }, parentCtx);
|
|
9895
9714
|
const spanCtx = _opentelemetry_api.trace.setSpan(parentCtx, span);
|
|
9896
9715
|
span.setAttribute(require_utils.SPAN_TYPE, "TOOL");
|
|
9897
9716
|
span.setAttributes(buildAiSdkInstrumentationAttributes());
|
|
9717
|
+
applyAssociationProperties(span, op.associationProperties);
|
|
9898
9718
|
if (toolName) span.setAttribute("ai.toolCall.name", toolName);
|
|
9899
9719
|
span.setAttribute("ai.toolCall.id", toolCallId);
|
|
9900
9720
|
if (this.recordInputs && toolCall?.input !== void 0) {
|
|
@@ -9964,6 +9784,35 @@ var LaminarAiSdkTelemetry = class {
|
|
|
9964
9784
|
};
|
|
9965
9785
|
this.onRerankStart = () => {};
|
|
9966
9786
|
this.onRerankEnd = () => {};
|
|
9787
|
+
this.experimental_onDecideStart = (event) => {
|
|
9788
|
+
const callId = event?.callId;
|
|
9789
|
+
if (!callId) return;
|
|
9790
|
+
this.onStart(event);
|
|
9791
|
+
const op = this.operationByCallId.get(callId);
|
|
9792
|
+
if (!op) return;
|
|
9793
|
+
op.span.setAttribute(require_utils.SPAN_TYPE, "LLM");
|
|
9794
|
+
applyRequestModelAttributes(op.span, event);
|
|
9795
|
+
if (this.recordInputs) {
|
|
9796
|
+
const input = buildDecisionInputMessages(event.state);
|
|
9797
|
+
if (input !== void 0) op.span.setAttribute("gen_ai.input.messages", input);
|
|
9798
|
+
if (event.questions !== void 0) op.span.setAttribute("gen_ai.request.structured_output_schema", serializeJSON$1(event.questions));
|
|
9799
|
+
}
|
|
9800
|
+
};
|
|
9801
|
+
this.experimental_onDecideEnd = (event) => {
|
|
9802
|
+
const callId = event?.callId;
|
|
9803
|
+
if (!callId) return;
|
|
9804
|
+
const op = this.operationByCallId.get(callId);
|
|
9805
|
+
if (!op) return;
|
|
9806
|
+
if (typeof event.response?.modelId === "string") op.span.setAttribute(require_utils.LaminarAttributes.RESPONSE_MODEL, event.response.modelId);
|
|
9807
|
+
if (typeof event.response?.id === "string") op.span.setAttribute("gen_ai.response.id", event.response.id);
|
|
9808
|
+
applyUsageToSpan(op.span, event.usage);
|
|
9809
|
+
if (this.recordOutputs && event.answers !== void 0) op.span.setAttribute("gen_ai.output.messages", serializeJSON$1([{
|
|
9810
|
+
role: "assistant",
|
|
9811
|
+
content: serializeJSON$1(event.answers)
|
|
9812
|
+
}]));
|
|
9813
|
+
op.span.setStatus({ code: _opentelemetry_api.SpanStatusCode.OK });
|
|
9814
|
+
this.closeOperation(callId, op);
|
|
9815
|
+
};
|
|
9967
9816
|
this.onEnd = (event) => {
|
|
9968
9817
|
const callId = event?.callId;
|
|
9969
9818
|
if (!callId) return;
|