@lmnr-ai/lmnr 0.8.50 → 0.8.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.cjs +2 -2
- package/dist/cli.mjs +2 -2
- package/dist/{file-utils-DOEv6ccL.mjs → file-utils-CVTUiqFU.mjs} +2 -2
- package/dist/{file-utils-DOEv6ccL.mjs.map → file-utils-CVTUiqFU.mjs.map} +1 -1
- package/dist/{file-utils-RXJzJyiI.cjs → file-utils-Cyx-2kqE.cjs} +2 -2
- package/dist/{file-utils-RXJzJyiI.cjs.map → file-utils-Cyx-2kqE.cjs.map} +1 -1
- package/dist/index.cjs +118 -371
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +12112 -10284
- package/dist/index.d.mts +12112 -10284
- package/dist/index.mjs +118 -371
- package/dist/index.mjs.map +1 -1
- package/dist/{utils-DRrXCfun.mjs → utils-Ch0R_9XE.mjs} +3 -3
- package/dist/{utils-DRrXCfun.mjs.map → utils-Ch0R_9XE.mjs.map} +1 -1
- package/dist/{utils-CNR1tzh5.cjs → utils-Cq24YsD_.cjs} +3 -3
- package/dist/{utils-CNR1tzh5.cjs.map → utils-Cq24YsD_.cjs.map} +1 -1
- package/package.json +12 -12
package/dist/index.cjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
2
|
const require_rolldown_runtime = require("./rolldown-runtime-Ctb5zv51.cjs");
|
|
3
|
-
const require_utils = require("./utils-
|
|
3
|
+
const require_utils = require("./utils-Cq24YsD_.cjs");
|
|
4
4
|
const require_consts = require("./consts-9rSS0oeR.cjs");
|
|
5
5
|
let path = require("path");
|
|
6
6
|
path = require_rolldown_runtime.__toESM(path);
|
|
@@ -183,7 +183,7 @@ var LaminarDataset = class extends EvaluationDataset {
|
|
|
183
183
|
*/
|
|
184
184
|
async push(paths, recursive = false) {
|
|
185
185
|
if (!this.client) throw new Error("Client not set");
|
|
186
|
-
const { loadFromPaths } = await Promise.resolve().then(() => require("./file-utils-
|
|
186
|
+
const { loadFromPaths } = await Promise.resolve().then(() => require("./file-utils-Cyx-2kqE.cjs")).then((n) => n.file_utils_exports);
|
|
187
187
|
const data = await loadFromPaths(Array.isArray(paths) ? paths : [paths], recursive);
|
|
188
188
|
if (data.length === 0) {
|
|
189
189
|
console.warn("No data to push. Skipping");
|
|
@@ -8334,165 +8334,10 @@ async function evaluate({ data, executor, evaluators, groupName, name, metadata,
|
|
|
8334
8334
|
//#region src/integrations/eve.ts
|
|
8335
8335
|
const logger$2 = require_utils.initializeLogger();
|
|
8336
8336
|
const EVE_REPORTER_TRACER_NAME = "@lmnr-ai/lmnr/eve-reporter";
|
|
8337
|
-
const
|
|
8338
|
-
const
|
|
8339
|
-
|
|
8340
|
-
const
|
|
8341
|
-
/**
|
|
8342
|
-
* Root span name for every propagated eval, and it never changes. eve calls
|
|
8343
|
-
* `ClientSession.send` before any hook that knows which eval is in flight, and
|
|
8344
|
-
* eve exposes no `onEvalStart`, so the id simply is not knowable here. Renaming
|
|
8345
|
-
* the span later would invalidate the `lmnr.span.path` the processor already
|
|
8346
|
-
* stamped on it and on every descendant, and would make the path we propagate
|
|
8347
|
-
* to the agent stale. The eval id rides on `lmnr.eve.eval.id`, on the trace
|
|
8348
|
-
* metadata, and on the datapoint instead.
|
|
8349
|
-
*/
|
|
8350
|
-
const ROOT_SPAN_NAME = "eve eval";
|
|
8351
|
-
const EXECUTOR_SPAN_NAME = "executor";
|
|
8352
|
-
/** Prototypes already patched — `send` must never be wrapped twice. */
|
|
8353
|
-
const patchedSendPrototypes = /* @__PURE__ */ new WeakSet();
|
|
8354
|
-
/**
|
|
8355
|
-
* Per-eve-client-session state. Keyed off the `ClientSession` instance rather
|
|
8356
|
-
* than an AsyncLocalStorage frame or a "current eval" global: eve runs up to
|
|
8357
|
-
* `maxConcurrency` evals at once, and each eval builds its own `ClientSession`
|
|
8358
|
-
* (`EvalSessionManager` calls `client.session()` per `EvalSessionDriver`), so
|
|
8359
|
-
* the instance is the only correct correlation key available inside `send`.
|
|
8360
|
-
*/
|
|
8361
|
-
const traceByClientSession = /* @__PURE__ */ new WeakMap();
|
|
8362
|
-
/** eve session id -> trace, read by `onEvalComplete` via `result.sessionId`. */
|
|
8363
|
-
const traceByEveSessionId = /* @__PURE__ */ new Map();
|
|
8364
|
-
/** Set by the running reporter's `onRunStart`; cleared by `onRunComplete`. */
|
|
8365
|
-
let activeSessionTraceFactory = null;
|
|
8366
|
-
/** Mirrors eve's own `normalizeSendTurnInput`, then merges the trace headers. */
|
|
8367
|
-
const withTraceHeaders = (input, headers) => {
|
|
8368
|
-
const payload = typeof input === "string" ? { message: input } : { ...input };
|
|
8369
|
-
payload.headers = {
|
|
8370
|
-
...payload.headers ?? {},
|
|
8371
|
-
...headers
|
|
8372
|
-
};
|
|
8373
|
-
return payload;
|
|
8374
|
-
};
|
|
8375
|
-
const openSessionTrace = (clientSession) => {
|
|
8376
|
-
try {
|
|
8377
|
-
const existing = traceByClientSession.get(clientSession);
|
|
8378
|
-
if (existing) return existing;
|
|
8379
|
-
const minted = activeSessionTraceFactory?.();
|
|
8380
|
-
if (minted) traceByClientSession.set(clientSession, minted);
|
|
8381
|
-
return minted;
|
|
8382
|
-
} catch (error) {
|
|
8383
|
-
logger$2.warn(`Laminar eve reporter: failed to open a session trace: ` + require_utils.errorMessage(error));
|
|
8384
|
-
return;
|
|
8385
|
-
}
|
|
8386
|
-
};
|
|
8387
|
-
/**
|
|
8388
|
-
* eve assigns the session id on the POST that `send` awaits, so the returned
|
|
8389
|
-
* `MessageResponse` is the earliest place the runner can bind the eve session
|
|
8390
|
-
* id to the trace it minted.
|
|
8391
|
-
*/
|
|
8392
|
-
const recordEveSessionId = (response, sessionTrace) => {
|
|
8393
|
-
const sessionId = response?.sessionId;
|
|
8394
|
-
if (typeof sessionId !== "string" || sessionId.length === 0) return;
|
|
8395
|
-
sessionTrace.sessionIds.add(sessionId);
|
|
8396
|
-
traceByEveSessionId.set(sessionId, sessionTrace);
|
|
8397
|
-
};
|
|
8398
|
-
/**
|
|
8399
|
-
* Build the propagation headers for the EXECUTOR span. Plain HTTP strings — eve
|
|
8400
|
-
* forwards `SendTurnPayload.headers` verbatim to `fetch`.
|
|
8401
|
-
*
|
|
8402
|
-
* The Laminar context carries the full path. Laminar nests spans by
|
|
8403
|
-
* `lmnr.span.ids_path`, not by OTel `parentSpanId`, so a receiver that adopts
|
|
8404
|
-
* only the ids lands its spans as a second root on our trace. The path is what
|
|
8405
|
-
* makes the agent's subtree render UNDER the executor. Both paths are constants
|
|
8406
|
-
* here — neither the root nor the executor span is ever renamed.
|
|
8407
|
-
*/
|
|
8408
|
-
const buildTraceHeaders = (rootSpan, executorSpan) => {
|
|
8409
|
-
const spanContext = executorSpan.spanContext();
|
|
8410
|
-
const laminarContext = {
|
|
8411
|
-
traceId: require_utils.otelTraceIdToUUID(spanContext.traceId),
|
|
8412
|
-
spanId: require_utils.otelSpanIdToUUID(spanContext.spanId),
|
|
8413
|
-
isRemote: true,
|
|
8414
|
-
spanPath: [ROOT_SPAN_NAME, EXECUTOR_SPAN_NAME],
|
|
8415
|
-
spanIdsPath: [require_utils.otelSpanIdToUUID(rootSpan.spanContext().spanId), require_utils.otelSpanIdToUUID(spanContext.spanId)]
|
|
8416
|
-
};
|
|
8417
|
-
return {
|
|
8418
|
-
[TRACEPARENT_HEADER$1]: `00-${spanContext.traceId}-${spanContext.spanId}-01`,
|
|
8419
|
-
[LAMINAR_SPAN_CONTEXT_HEADER$1]: JSON.stringify(laminarContext)
|
|
8420
|
-
};
|
|
8421
|
-
};
|
|
8422
|
-
/**
|
|
8423
|
-
* Bind this session's ROOT span into the calling eval's async context, so work
|
|
8424
|
-
* the eval does in the RUNNER process after `send` resolves lands on this trace
|
|
8425
|
-
* instead of opening its own. The concrete case is `t.judge.autoevals.*`: eve
|
|
8426
|
-
* runs the judge's model call in the runner, and Laminar's AI SDK integration
|
|
8427
|
-
* parents it to `LaminarContextManager.getContext()`.
|
|
8428
|
-
*
|
|
8429
|
-
* The root — not a pre-opened judge span — is the binding target on purpose.
|
|
8430
|
-
* An eval may record any number of judge assertions, and each one runs its own
|
|
8431
|
-
* model call. Binding the root gives every judge its own child of the root; a
|
|
8432
|
-
* single pre-opened parent would collect all of them under whichever judge
|
|
8433
|
-
* happened to run first.
|
|
8434
|
-
*
|
|
8435
|
-
* `pushContext` uses `AsyncLocalStorage.enterWith`, so this MUST run
|
|
8436
|
-
* synchronously before `send`'s first `await` — at that point we are still
|
|
8437
|
-
* executing in the caller's async context, so the store we set is the one
|
|
8438
|
-
* `test(t)` continues with. Each eval's `test(t)` runs in its own async chain,
|
|
8439
|
-
* so concurrent evals never see each other's root span; do NOT replace this
|
|
8440
|
-
* with the process-global context stack, which would cross-attribute.
|
|
8441
|
-
*
|
|
8442
|
-
* Judge spans reach Laminar only when the runner opts in — `Laminar.initialize()`
|
|
8443
|
-
* plus a globally registered AI SDK telemetry integration. Without that the
|
|
8444
|
-
* binding is inert, never wrong.
|
|
8445
|
-
*
|
|
8446
|
-
* Never popped: the store dies with the eval's async chain.
|
|
8447
|
-
*/
|
|
8448
|
-
const bindEvalContext = (sessionTrace) => {
|
|
8449
|
-
try {
|
|
8450
|
-
LaminarContextManager.pushContext(_opentelemetry_api.trace.setSpan(_opentelemetry_api.ROOT_CONTEXT, sessionTrace.rootSpan));
|
|
8451
|
-
} catch (error) {
|
|
8452
|
-
logger$2.warn(`Laminar eve reporter: failed to bind the eval context: ` + require_utils.errorMessage(error));
|
|
8453
|
-
}
|
|
8454
|
-
};
|
|
8455
|
-
/**
|
|
8456
|
-
* Wrap `ClientSession.prototype.send` so every eve turn carries the trace
|
|
8457
|
-
* context of a runner-minted Laminar span. Idempotent per prototype.
|
|
8458
|
-
*
|
|
8459
|
-
* @internal Exported for tests, which drive it with a stub class so the suite
|
|
8460
|
-
* keeps no dependency on eve.
|
|
8461
|
-
*/
|
|
8462
|
-
const patchEveClientSession = (sessionClass) => {
|
|
8463
|
-
const prototype = sessionClass?.prototype;
|
|
8464
|
-
const original = prototype?.send;
|
|
8465
|
-
if (!prototype || typeof original !== "function") return false;
|
|
8466
|
-
if (patchedSendPrototypes.has(prototype)) return true;
|
|
8467
|
-
prototype.send = async function(input) {
|
|
8468
|
-
const sessionTrace = openSessionTrace(this);
|
|
8469
|
-
if (!sessionTrace) return original.call(this, input);
|
|
8470
|
-
bindEvalContext(sessionTrace);
|
|
8471
|
-
const response = await original.call(this, withTraceHeaders(input, sessionTrace.headers));
|
|
8472
|
-
try {
|
|
8473
|
-
recordEveSessionId(response, sessionTrace);
|
|
8474
|
-
} catch (error) {
|
|
8475
|
-
logger$2.warn(`Laminar eve reporter: failed to record the eve session id: ` + require_utils.errorMessage(error));
|
|
8476
|
-
}
|
|
8477
|
-
return response;
|
|
8478
|
-
};
|
|
8479
|
-
patchedSendPrototypes.add(prototype);
|
|
8480
|
-
return true;
|
|
8481
|
-
};
|
|
8482
|
-
/**
|
|
8483
|
-
* Load eve's `ClientSession` without depending on eve. Mirrors the guarded
|
|
8484
|
-
* dynamic import in eve's own Braintrust reporter (`loadBraintrustSdk`), except
|
|
8485
|
-
* a missing eve degrades instead of throwing — the reporter must keep working
|
|
8486
|
-
* when it is registered outside an eve run.
|
|
8487
|
-
*/
|
|
8488
|
-
const loadEveClientSession = async () => {
|
|
8489
|
-
try {
|
|
8490
|
-
const sessionClass = (await import(EVE_CLIENT_MODULE)).ClientSession;
|
|
8491
|
-
return typeof sessionClass === "function" ? sessionClass : void 0;
|
|
8492
|
-
} catch {
|
|
8493
|
-
return;
|
|
8494
|
-
}
|
|
8495
|
-
};
|
|
8337
|
+
const OTEL_TRACE_ID_RE = /^[0-9a-f]{32}$/iu;
|
|
8338
|
+
const OTEL_SPAN_ID_RE = /^[0-9a-f]{16}$/iu;
|
|
8339
|
+
const ZERO_TRACE_ID = "0".repeat(32);
|
|
8340
|
+
const ZERO_SPAN_ID = "0".repeat(16);
|
|
8496
8341
|
const didAssertionPass = (assertion) => {
|
|
8497
8342
|
if (typeof assertion.passed === "boolean") return assertion.passed;
|
|
8498
8343
|
if (typeof assertion.score === "number") {
|
|
@@ -8501,14 +8346,8 @@ const didAssertionPass = (assertion) => {
|
|
|
8501
8346
|
}
|
|
8502
8347
|
return true;
|
|
8503
8348
|
};
|
|
8504
|
-
/**
|
|
8505
|
-
|
|
8506
|
-
* Detailed assertion outcomes live in metadata; sparse per-assertion score keys
|
|
8507
|
-
* make Laminar's eval UI look incomplete when different eval files assert
|
|
8508
|
-
* different things.
|
|
8509
|
-
*/
|
|
8510
|
-
const resultToScores = (result) => {
|
|
8511
|
-
const assertions = result.assertions ?? [];
|
|
8349
|
+
/** Keep Laminar's score columns stable across heterogeneous eve evals. */
|
|
8350
|
+
const resultToScores = (result, assertions) => {
|
|
8512
8351
|
const gateAssertions = assertions.filter((assertion) => assertion.severity === "gate");
|
|
8513
8352
|
const thresholdedSoftAssertions = assertions.filter((assertion) => assertion.severity === "soft" && typeof assertion.threshold === "number");
|
|
8514
8353
|
return {
|
|
@@ -8526,7 +8365,6 @@ const assertionsMetadata = (assertions) => assertions.map((assertion) => ({
|
|
|
8526
8365
|
message: assertion.message,
|
|
8527
8366
|
metadata: assertion.metadata
|
|
8528
8367
|
}));
|
|
8529
|
-
/** Surface failed assertions in metadata so a failing run is debuggable. */
|
|
8530
8368
|
const failedAssertionsMetadata = (assertions) => {
|
|
8531
8369
|
const failed = assertions.filter((assertion) => !didAssertionPass(assertion)).map((assertion) => ({
|
|
8532
8370
|
name: assertion.name,
|
|
@@ -8534,79 +8372,62 @@ const failedAssertionsMetadata = (assertions) => {
|
|
|
8534
8372
|
}));
|
|
8535
8373
|
return failed.length > 0 ? { failedAssertions: failed } : {};
|
|
8536
8374
|
};
|
|
8537
|
-
const
|
|
8375
|
+
const validTraceContext = (context) => context != null && typeof context.traceId === "string" && OTEL_TRACE_ID_RE.test(context.traceId) && context.traceId !== ZERO_TRACE_ID && typeof context.spanId === "string" && OTEL_SPAN_ID_RE.test(context.spanId) && context.spanId !== ZERO_SPAN_ID && typeof context.traceFlags === "number" && Number.isInteger(context.traceFlags) && context.traceFlags >= 0 && context.traceFlags <= 255;
|
|
8376
|
+
const normalizeTraceContexts = (contexts) => Array.isArray(contexts) ? contexts.filter(validTraceContext) : [];
|
|
8377
|
+
const isSampledTraceContext = (context) => (context.traceFlags & 1) === 1;
|
|
8378
|
+
const selectTraceContext = (contexts) => contexts.find((context) => context.primary === true && isSampledTraceContext(context)) ?? contexts.find(isSampledTraceContext);
|
|
8379
|
+
const traceContextMetadata = (contexts) => contexts.map((context) => ({
|
|
8380
|
+
traceId: context.traceId,
|
|
8381
|
+
spanId: context.spanId,
|
|
8382
|
+
traceFlags: context.traceFlags,
|
|
8383
|
+
primary: context.primary,
|
|
8384
|
+
sessionId: context.sessionId
|
|
8385
|
+
}));
|
|
8538
8386
|
/**
|
|
8539
|
-
*
|
|
8540
|
-
* or per-eval via the `reporters` field — eve runs and grades each eval, then
|
|
8541
|
-
* hands the graded result here, and this reporter ships it to Laminar as an
|
|
8542
|
-
* evaluation run with one datapoint per eval.
|
|
8543
|
-
*
|
|
8544
|
-
* The datapoint's trace id is never looked up — the reporter OWNS the trace.
|
|
8545
|
-
* `patchEveClientSession` wraps eve's `ClientSession.send` and mints the
|
|
8546
|
-
* EVALUATION root + EXECUTOR child on the first turn of each session, so the
|
|
8547
|
-
* runner already knows the trace id and pushes it to the agent as a
|
|
8548
|
-
* `traceparent` header. The eve session id is only the in-process key that ties
|
|
8549
|
-
* the send we instrumented to the graded result eve hands back later.
|
|
8550
|
-
*
|
|
8551
|
-
* Two outcomes, recorded on `metadata.traceResolution`:
|
|
8552
|
-
*
|
|
8553
|
-
* 1. `"propagated"` — the normal path, above. The agent joins the trace only
|
|
8554
|
-
* when `agent/instrumentation.ts` sets `traceChannelRequests: true` and the
|
|
8555
|
-
* agent process runs with `WORKFLOW_TRACE_MODE=continuous`.
|
|
8556
|
-
* 2. `"reporter-fallback"` — no session trace was minted (propagation disabled,
|
|
8557
|
-
* or eve produced no session id). The reporter opens its own EVALUATION span
|
|
8558
|
-
* so the grade still lands somewhere debuggable; it holds no agent work.
|
|
8559
|
-
*
|
|
8560
|
-
* `t.judge.autoevals.*` model calls run in the RUNNER process, and they reach
|
|
8561
|
-
* Laminar only when the runner registers an AI SDK telemetry integration. eve
|
|
8562
|
-
* imports `generateText` inside its own bundle, so `wrapAISDK` cannot reach it —
|
|
8563
|
-
* but AI SDK v7 reads registered integrations off `globalThis`, which does. One
|
|
8564
|
-
* line in `evals.config.ts`:
|
|
8387
|
+
* Reports eve eval results to Laminar.
|
|
8565
8388
|
*
|
|
8566
|
-
*
|
|
8567
|
-
*
|
|
8568
|
-
*
|
|
8389
|
+
* Eve 0.35+ gives reporters every native trace context observed during an eval.
|
|
8390
|
+
* The reporter links the datapoint to the first sampled primary trace (or the
|
|
8391
|
+
* first sampled trace when there is no sampled primary), records every valid
|
|
8392
|
+
* context in metadata, and adds
|
|
8393
|
+
* a small EVALUATION span to the selected native trace for the grade. When eve
|
|
8394
|
+
* reports no usable context, a reporter-owned fallback trace preserves the
|
|
8395
|
+
* result without pretending it contains agent work.
|
|
8569
8396
|
*
|
|
8570
|
-
*
|
|
8571
|
-
* the
|
|
8572
|
-
* Each judge gets its own span under the eval's root — see
|
|
8573
|
-
* {@link bindEvalContext}, and {@link LaminarReporter.flush} for how spans ship.
|
|
8574
|
-
* Without that line the reporter still records every assertion's score as an
|
|
8575
|
-
* EVALUATOR span; only the model call is missing.
|
|
8397
|
+
* The agent must export its native OpenTelemetry spans to the same Laminar
|
|
8398
|
+
* project. In the eve agent, declare a destination like:
|
|
8576
8399
|
*
|
|
8577
|
-
* @example
|
|
8578
8400
|
* ```ts
|
|
8579
|
-
* import {
|
|
8580
|
-
* import {
|
|
8401
|
+
* import { LaminarSpanProcessor } from "@lmnr-ai/lmnr";
|
|
8402
|
+
* import { otelIntegration } from "eve/instrumentation/otel";
|
|
8581
8403
|
*
|
|
8582
|
-
* export default
|
|
8583
|
-
*
|
|
8404
|
+
* export default otelIntegration({
|
|
8405
|
+
* spanProcessors: [new LaminarSpanProcessor()],
|
|
8584
8406
|
* });
|
|
8585
8407
|
* ```
|
|
8586
8408
|
*/
|
|
8587
8409
|
var LaminarReporter = class {
|
|
8588
8410
|
constructor(options = {}) {
|
|
8589
8411
|
this.evalsById = /* @__PURE__ */ new Map();
|
|
8590
|
-
this.
|
|
8591
|
-
this.
|
|
8592
|
-
this.sessionTraceFactory = () => this.mintSessionTrace();
|
|
8412
|
+
this.evalIndexById = /* @__PURE__ */ new Map();
|
|
8413
|
+
this.nextFallbackIndex = 0;
|
|
8593
8414
|
this.options = options;
|
|
8594
8415
|
}
|
|
8595
8416
|
async onRunStart(evaluations, target) {
|
|
8596
8417
|
this.evalId = void 0;
|
|
8597
|
-
|
|
8598
|
-
|
|
8599
|
-
|
|
8418
|
+
const identifiedEvaluations = evaluations.map((evaluation, index) => ({
|
|
8419
|
+
evaluation,
|
|
8420
|
+
index
|
|
8421
|
+
})).filter((item) => typeof item.evaluation.id === "string");
|
|
8422
|
+
this.evalsById = new Map(identifiedEvaluations.map(({ evaluation }) => [evaluation.id, evaluation]));
|
|
8423
|
+
this.evalIndexById = new Map(identifiedEvaluations.map(({ evaluation, index }) => [evaluation.id, index]));
|
|
8424
|
+
this.nextFallbackIndex = evaluations.length;
|
|
8600
8425
|
try {
|
|
8601
8426
|
this.client = this.options.client ?? new require_utils.LaminarClient({
|
|
8602
8427
|
baseUrl: this.options.baseUrl,
|
|
8603
8428
|
projectApiKey: this.options.projectApiKey
|
|
8604
8429
|
});
|
|
8605
8430
|
this.ensureTracing();
|
|
8606
|
-
if (this.options.propagateTraceContext !== false) {
|
|
8607
|
-
activeSessionTraceFactory = this.sessionTraceFactory;
|
|
8608
|
-
patchEveClientSession(await loadEveClientSession());
|
|
8609
|
-
}
|
|
8610
8431
|
const evaluation = await this.client.evals.init(this.options.name, this.options.groupName, {
|
|
8611
8432
|
...this.options.metadata ?? {},
|
|
8612
8433
|
source: "eve",
|
|
@@ -8619,26 +8440,25 @@ var LaminarReporter = class {
|
|
|
8619
8440
|
process.stdout.write(`\nLaminar: check eve eval results at ${url}\n`);
|
|
8620
8441
|
} catch (error) {
|
|
8621
8442
|
logger$2.error(`Laminar eve reporter: failed to start run: ${require_utils.errorMessage(error)}`);
|
|
8622
|
-
if (!this.evalId)
|
|
8623
|
-
this.client = void 0;
|
|
8624
|
-
if (activeSessionTraceFactory === this.sessionTraceFactory) activeSessionTraceFactory = null;
|
|
8625
|
-
}
|
|
8443
|
+
if (!this.evalId) this.client = void 0;
|
|
8626
8444
|
}
|
|
8627
8445
|
}
|
|
8628
|
-
async onEvalComplete(result) {
|
|
8446
|
+
async onEvalComplete(result, context) {
|
|
8629
8447
|
const client = this.client;
|
|
8630
8448
|
const evalId = this.evalId;
|
|
8631
8449
|
if (!client || !evalId) return;
|
|
8632
|
-
const index = this.
|
|
8450
|
+
const index = typeof result.id === "string" ? this.evalIndexById.get(result.id) ?? this.nextFallbackIndex++ : this.nextFallbackIndex++;
|
|
8633
8451
|
const task = result.result ?? {};
|
|
8634
8452
|
const derived = task.derived ?? {};
|
|
8635
|
-
const assertions = result.assertions
|
|
8636
|
-
const
|
|
8637
|
-
const
|
|
8638
|
-
|
|
8453
|
+
const assertions = Array.isArray(result.assertions) ? result.assertions.filter((assertion) => assertion != null && typeof assertion === "object") : [];
|
|
8454
|
+
const evalDefinition = typeof result.id === "string" ? this.evalsById.get(result.id) : void 0;
|
|
8455
|
+
const evalDescription = context?.evaluation?.description ?? evalDefinition?.description;
|
|
8456
|
+
const contexts = normalizeTraceContexts(context?.traceContexts ?? task.traceContexts ?? []);
|
|
8457
|
+
const gradeTrace = this.openGradeTrace({
|
|
8458
|
+
contexts,
|
|
8639
8459
|
description: evalDescription,
|
|
8640
|
-
|
|
8641
|
-
|
|
8460
|
+
evalId: result.id ?? String(index),
|
|
8461
|
+
verdict: result.verdict
|
|
8642
8462
|
});
|
|
8643
8463
|
try {
|
|
8644
8464
|
const datapointId = await client.evals.createDatapoint({
|
|
@@ -8646,17 +8466,19 @@ var LaminarReporter = class {
|
|
|
8646
8466
|
data: result.id ?? null,
|
|
8647
8467
|
target: task.finalMessage ?? null,
|
|
8648
8468
|
index,
|
|
8649
|
-
traceId:
|
|
8469
|
+
traceId: gradeTrace.traceId,
|
|
8650
8470
|
metadata: {
|
|
8651
8471
|
name: result.id,
|
|
8652
8472
|
verdict: result.verdict,
|
|
8653
8473
|
status: task.status,
|
|
8654
8474
|
description: evalDescription,
|
|
8655
8475
|
sessionId: task.sessionId,
|
|
8656
|
-
traceResolution:
|
|
8476
|
+
traceResolution: gradeTrace.source,
|
|
8477
|
+
selectedTraceContext: gradeTrace.selected ? traceContextMetadata([gradeTrace.selected])[0] : void 0,
|
|
8478
|
+
traceContexts: traceContextMetadata(gradeTrace.contexts),
|
|
8657
8479
|
skipReason: result.skipReason,
|
|
8658
8480
|
error: result.error,
|
|
8659
|
-
toolCalls: (derived.toolCalls
|
|
8481
|
+
toolCalls: (Array.isArray(derived.toolCalls) ? derived.toolCalls : []).map((call) => call?.name).filter((name) => typeof name === "string"),
|
|
8660
8482
|
toolCallCount: derived.toolCallCount,
|
|
8661
8483
|
parked: derived.parked,
|
|
8662
8484
|
failureCode: derived.failureCode,
|
|
@@ -8665,44 +8487,30 @@ var LaminarReporter = class {
|
|
|
8665
8487
|
...failedAssertionsMetadata(assertions)
|
|
8666
8488
|
}
|
|
8667
8489
|
});
|
|
8668
|
-
const executorOutput = task.output ?? task.finalMessage ?? null;
|
|
8669
8490
|
await client.evals.updateDatapoint({
|
|
8670
8491
|
evalId,
|
|
8671
8492
|
datapointId,
|
|
8672
|
-
scores: resultToScores(result),
|
|
8673
|
-
executorOutput
|
|
8493
|
+
scores: resultToScores(result, assertions),
|
|
8494
|
+
executorOutput: task.output ?? task.finalMessage ?? null
|
|
8674
8495
|
});
|
|
8675
8496
|
} catch (error) {
|
|
8676
|
-
|
|
8497
|
+
gradeTrace.span.recordException(error);
|
|
8677
8498
|
logger$2.error(`Laminar eve reporter: failed to report eval "${result.id ?? index}": ${require_utils.errorMessage(error)}`);
|
|
8678
8499
|
} finally {
|
|
8679
|
-
|
|
8680
|
-
|
|
8681
|
-
evalId: result.id,
|
|
8682
|
-
description: evalDescription,
|
|
8683
|
-
verdict: result.verdict,
|
|
8684
|
-
status: task.status,
|
|
8500
|
+
this.finishGradeTrace(gradeTrace, {
|
|
8501
|
+
assertions,
|
|
8685
8502
|
output: task.output ?? task.finalMessage ?? null,
|
|
8686
|
-
|
|
8503
|
+
status: task.status
|
|
8687
8504
|
});
|
|
8688
8505
|
await this.flush();
|
|
8689
8506
|
}
|
|
8690
8507
|
}
|
|
8691
|
-
async onRunComplete() {
|
|
8692
|
-
this.finishOpenSessionTraces();
|
|
8693
|
-
if (activeSessionTraceFactory === this.sessionTraceFactory) activeSessionTraceFactory = null;
|
|
8508
|
+
async onRunComplete(_summary) {
|
|
8694
8509
|
await this.flush();
|
|
8695
8510
|
this.evalId = void 0;
|
|
8696
8511
|
this.evalsById.clear();
|
|
8512
|
+
this.evalIndexById.clear();
|
|
8697
8513
|
}
|
|
8698
|
-
/**
|
|
8699
|
-
* Point the process's Laminar tracing at this reporter's project, unless
|
|
8700
|
-
* something already initialized it. Mirrors `LaminarAiSdkTelemetry`, which
|
|
8701
|
-
* initializes the same way for judge spans — so eve eval spans and judge
|
|
8702
|
-
* spans share ONE tracer provider however the run was set up. Without this,
|
|
8703
|
-
* `getTracer()` returns a noop whenever nobody registered the AI SDK
|
|
8704
|
-
* integration, and every eve span would be dropped silently.
|
|
8705
|
-
*/
|
|
8706
8514
|
ensureTracing() {
|
|
8707
8515
|
if (Laminar.initialized()) {
|
|
8708
8516
|
const ourKey = this.client?.apiKey ?? this.options.projectApiKey;
|
|
@@ -8718,140 +8526,79 @@ var LaminarReporter = class {
|
|
|
8718
8526
|
disableBatch: true
|
|
8719
8527
|
});
|
|
8720
8528
|
}
|
|
8721
|
-
/**
|
|
8722
|
-
* Flush the process's Laminar pipeline — eve eval spans AND judge spans, which
|
|
8723
|
-
* now share one provider. eve ends `eve eval` with `process.exit()` (skipping
|
|
8724
|
-
* `beforeExit`), so these calls are the only points at which spans ship.
|
|
8725
|
-
* Never `shutdown()`: the pipeline may belong to the host, not to us.
|
|
8726
|
-
*/
|
|
8727
8529
|
async flush() {
|
|
8728
8530
|
try {
|
|
8729
8531
|
await Laminar.flush();
|
|
8730
8532
|
} catch (error) {
|
|
8731
|
-
logger$2.warn(`Laminar eve reporter: failed to flush spans:
|
|
8533
|
+
logger$2.warn(`Laminar eve reporter: failed to flush spans: ${require_utils.errorMessage(error)}`);
|
|
8732
8534
|
}
|
|
8733
8535
|
}
|
|
8734
|
-
|
|
8735
|
-
|
|
8736
|
-
* root plus the EXECUTOR child the agent's turn is parented to. Called from
|
|
8737
|
-
* the patched `send`, so it must never throw into the user's eval.
|
|
8738
|
-
*/
|
|
8739
|
-
mintSessionTrace() {
|
|
8536
|
+
openGradeTrace({ contexts, description, evalId, verdict }) {
|
|
8537
|
+
const selected = selectTraceContext(contexts);
|
|
8740
8538
|
const tracer = getTracer();
|
|
8741
|
-
const
|
|
8742
|
-
|
|
8743
|
-
|
|
8744
|
-
"lmnr.eve.reporter": EVE_REPORTER_TRACER_NAME
|
|
8745
|
-
} }, _opentelemetry_api.ROOT_CONTEXT);
|
|
8746
|
-
const executorSpan = tracer.startSpan(EXECUTOR_SPAN_NAME, { attributes: {
|
|
8747
|
-
[require_utils.SPAN_TYPE]: "EXECUTOR",
|
|
8748
|
-
[require_utils.TRACE_TYPE]: "EVALUATION"
|
|
8749
|
-
} }, _opentelemetry_api.trace.setSpan(_opentelemetry_api.ROOT_CONTEXT, rootSpan));
|
|
8750
|
-
const sessionTrace = {
|
|
8751
|
-
traceId: require_utils.otelTraceIdToUUID(rootSpan.spanContext().traceId),
|
|
8752
|
-
rootSpan,
|
|
8753
|
-
executorSpan,
|
|
8754
|
-
tracer,
|
|
8755
|
-
headers: buildTraceHeaders(rootSpan, executorSpan),
|
|
8756
|
-
sessionIds: /* @__PURE__ */ new Set(),
|
|
8757
|
-
ended: false
|
|
8758
|
-
};
|
|
8759
|
-
this.openSessionTraces.add(sessionTrace);
|
|
8760
|
-
return sessionTrace;
|
|
8761
|
-
}
|
|
8762
|
-
/** Grade, close and release one session trace. Idempotent. */
|
|
8763
|
-
finishSessionTrace(sessionTrace, grade) {
|
|
8764
|
-
if (sessionTrace.ended) return;
|
|
8765
|
-
sessionTrace.ended = true;
|
|
8766
|
-
this.openSessionTraces.delete(sessionTrace);
|
|
8767
|
-
for (const sessionId of sessionTrace.sessionIds) traceByEveSessionId.delete(sessionId);
|
|
8768
|
-
try {
|
|
8769
|
-
sessionTrace.rootSpan.setAttributes({
|
|
8770
|
-
"lmnr.eve.eval.id": grade.evalId ?? "",
|
|
8771
|
-
"lmnr.eve.eval.verdict": grade.verdict ?? "",
|
|
8772
|
-
"lmnr.eve.eval.status": grade.status ?? "",
|
|
8773
|
-
[require_utils.SPAN_INPUT]: JSON.stringify({
|
|
8774
|
-
evalId: grade.evalId,
|
|
8775
|
-
description: grade.description
|
|
8776
|
-
}),
|
|
8777
|
-
[require_utils.SPAN_OUTPUT]: JSON.stringify(grade.output ?? null),
|
|
8778
|
-
...require_utils.metadataToAttributes({
|
|
8779
|
-
source: "eve",
|
|
8780
|
-
eveEvalId: grade.evalId ?? "",
|
|
8781
|
-
eveEvalDescription: grade.description ?? "",
|
|
8782
|
-
eveEvalVerdict: grade.verdict ?? "",
|
|
8783
|
-
eveSessionId: [...sessionTrace.sessionIds].join(",")
|
|
8784
|
-
})
|
|
8785
|
-
});
|
|
8786
|
-
sessionTrace.executorSpan.setAttribute(require_utils.SPAN_OUTPUT, JSON.stringify(grade.output ?? null));
|
|
8787
|
-
this.recordEvaluatorSpans(sessionTrace, grade.assertions ?? []);
|
|
8788
|
-
sessionTrace.executorSpan.end();
|
|
8789
|
-
sessionTrace.rootSpan.end();
|
|
8790
|
-
} catch (error) {
|
|
8791
|
-
logger$2.warn(`Laminar eve reporter: failed to finish the session trace: ` + require_utils.errorMessage(error));
|
|
8792
|
-
}
|
|
8793
|
-
}
|
|
8794
|
-
/**
|
|
8795
|
-
* One EVALUATOR span per eve assertion, judge or not. They are created at
|
|
8796
|
-
* grade time (the assertions only exist then) as short children of the
|
|
8797
|
-
* still-open root. No explicit parent path is needed: the root keeps the name
|
|
8798
|
-
* it was minted with, so the processor's cached path for it is still correct.
|
|
8799
|
-
*/
|
|
8800
|
-
recordEvaluatorSpans(sessionTrace, assertions) {
|
|
8801
|
-
const rootContext = _opentelemetry_api.trace.setSpan(_opentelemetry_api.ROOT_CONTEXT, sessionTrace.rootSpan);
|
|
8802
|
-
for (const assertion of assertions) {
|
|
8803
|
-
const name = assertion.name ?? "assertion";
|
|
8804
|
-
const output = JSON.stringify({
|
|
8805
|
-
score: assertion.score ?? null,
|
|
8806
|
-
passed: didAssertionPass(assertion),
|
|
8807
|
-
severity: assertion.severity,
|
|
8808
|
-
threshold: assertion.threshold,
|
|
8809
|
-
message: assertion.message
|
|
8810
|
-
});
|
|
8811
|
-
sessionTrace.tracer.startSpan(name, { attributes: {
|
|
8812
|
-
[require_utils.SPAN_TYPE]: "EVALUATOR",
|
|
8813
|
-
[require_utils.TRACE_TYPE]: "EVALUATION",
|
|
8814
|
-
[require_utils.SPAN_OUTPUT]: output
|
|
8815
|
-
} }, rootContext).end();
|
|
8816
|
-
}
|
|
8817
|
-
}
|
|
8818
|
-
finishOpenSessionTraces() {
|
|
8819
|
-
for (const sessionTrace of [...this.openSessionTraces]) this.finishSessionTrace(sessionTrace, {});
|
|
8820
|
-
}
|
|
8821
|
-
/**
|
|
8822
|
-
* Resolve the trace this datapoint links to. There is no lookup: either
|
|
8823
|
-
* `patchEveClientSession` already minted the session's trace (and we know its
|
|
8824
|
-
* id), or we open a reporter-owned EVALUATION span to hold the grade.
|
|
8825
|
-
*/
|
|
8826
|
-
resolveDatapointTrace({ evalId, description, verdict, sessionId }) {
|
|
8827
|
-
const sessionTrace = sessionId ? traceByEveSessionId.get(sessionId) : void 0;
|
|
8828
|
-
if (sessionTrace) return {
|
|
8829
|
-
traceId: sessionTrace.traceId,
|
|
8830
|
-
source: "propagated",
|
|
8831
|
-
sessionTrace
|
|
8832
|
-
};
|
|
8833
|
-
const span = getTracer().startSpan(`eve eval ${evalId}`, { attributes: {
|
|
8539
|
+
const parentContext = selected ? _opentelemetry_api.trace.setSpanContext(_opentelemetry_api.ROOT_CONTEXT, this.toSpanContext(selected)) : _opentelemetry_api.ROOT_CONTEXT;
|
|
8540
|
+
const source = selected ? "eve-native" : "reporter-fallback";
|
|
8541
|
+
const span = tracer.startSpan(`eve eval ${evalId}`, { attributes: {
|
|
8834
8542
|
[require_utils.SPAN_TYPE]: "EVALUATION",
|
|
8835
8543
|
[require_utils.TRACE_TYPE]: "EVALUATION",
|
|
8836
8544
|
"lmnr.eve.reporter": EVE_REPORTER_TRACER_NAME,
|
|
8837
8545
|
"lmnr.eve.eval.id": evalId,
|
|
8838
8546
|
"lmnr.eve.eval.verdict": verdict ?? "",
|
|
8839
|
-
|
|
8840
|
-
"lmnr.eve.trace_resolution": "reporter-fallback",
|
|
8547
|
+
"lmnr.eve.trace_resolution": source,
|
|
8841
8548
|
...require_utils.metadataToAttributes({
|
|
8842
8549
|
source: "eve",
|
|
8843
8550
|
eveEvalId: evalId,
|
|
8844
8551
|
eveEvalDescription: description ?? "",
|
|
8845
8552
|
eveEvalVerdict: verdict ?? "",
|
|
8846
|
-
eveSessionId: sessionId ?? ""
|
|
8553
|
+
eveSessionId: selected?.sessionId ?? "",
|
|
8554
|
+
eveTraceCount: contexts.length
|
|
8555
|
+
}),
|
|
8556
|
+
[require_utils.SPAN_INPUT]: JSON.stringify({
|
|
8557
|
+
evalId,
|
|
8558
|
+
description
|
|
8847
8559
|
})
|
|
8848
|
-
} },
|
|
8560
|
+
} }, parentContext);
|
|
8561
|
+
return {
|
|
8562
|
+
traceId: require_utils.otelTraceIdToUUID(span.spanContext().traceId),
|
|
8563
|
+
source,
|
|
8564
|
+
span,
|
|
8565
|
+
tracer,
|
|
8566
|
+
selected,
|
|
8567
|
+
contexts
|
|
8568
|
+
};
|
|
8569
|
+
}
|
|
8570
|
+
toSpanContext(context) {
|
|
8849
8571
|
return {
|
|
8850
|
-
traceId:
|
|
8851
|
-
|
|
8852
|
-
|
|
8572
|
+
traceId: context.traceId.toLowerCase(),
|
|
8573
|
+
spanId: context.spanId.toLowerCase(),
|
|
8574
|
+
traceFlags: context.traceFlags,
|
|
8575
|
+
isRemote: true
|
|
8853
8576
|
};
|
|
8854
8577
|
}
|
|
8578
|
+
finishGradeTrace(gradeTrace, grade) {
|
|
8579
|
+
try {
|
|
8580
|
+
gradeTrace.span.setAttributes({
|
|
8581
|
+
"lmnr.eve.eval.status": grade.status ?? "",
|
|
8582
|
+
[require_utils.SPAN_OUTPUT]: JSON.stringify(grade.output ?? null)
|
|
8583
|
+
});
|
|
8584
|
+
const parentContext = _opentelemetry_api.trace.setSpan(_opentelemetry_api.ROOT_CONTEXT, gradeTrace.span);
|
|
8585
|
+
for (const assertion of grade.assertions) gradeTrace.tracer.startSpan(assertion.name ?? "assertion", { attributes: {
|
|
8586
|
+
[require_utils.SPAN_TYPE]: "EVALUATOR",
|
|
8587
|
+
[require_utils.TRACE_TYPE]: "EVALUATION",
|
|
8588
|
+
[require_utils.SPAN_OUTPUT]: JSON.stringify({
|
|
8589
|
+
score: assertion.score ?? null,
|
|
8590
|
+
passed: didAssertionPass(assertion),
|
|
8591
|
+
severity: assertion.severity,
|
|
8592
|
+
threshold: assertion.threshold,
|
|
8593
|
+
message: assertion.message
|
|
8594
|
+
})
|
|
8595
|
+
} }, parentContext).end();
|
|
8596
|
+
} catch (error) {
|
|
8597
|
+
logger$2.warn(`Laminar eve reporter: failed to finish the grade trace: ${require_utils.errorMessage(error)}`);
|
|
8598
|
+
} finally {
|
|
8599
|
+
gradeTrace.span.end();
|
|
8600
|
+
}
|
|
8601
|
+
}
|
|
8855
8602
|
};
|
|
8856
8603
|
//#endregion
|
|
8857
8604
|
//#region src/debug/hash.ts
|