@lmnr-ai/lmnr 0.8.50 → 0.8.51

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -1,6 +1,6 @@
1
1
  Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
2
2
  const require_rolldown_runtime = require("./rolldown-runtime-Ctb5zv51.cjs");
3
- const require_utils = require("./utils-CNR1tzh5.cjs");
3
+ const require_utils = require("./utils-Cq24YsD_.cjs");
4
4
  const require_consts = require("./consts-9rSS0oeR.cjs");
5
5
  let path = require("path");
6
6
  path = require_rolldown_runtime.__toESM(path);
@@ -183,7 +183,7 @@ var LaminarDataset = class extends EvaluationDataset {
183
183
  */
184
184
  async push(paths, recursive = false) {
185
185
  if (!this.client) throw new Error("Client not set");
186
- const { loadFromPaths } = await Promise.resolve().then(() => require("./file-utils-RXJzJyiI.cjs")).then((n) => n.file_utils_exports);
186
+ const { loadFromPaths } = await Promise.resolve().then(() => require("./file-utils-Cyx-2kqE.cjs")).then((n) => n.file_utils_exports);
187
187
  const data = await loadFromPaths(Array.isArray(paths) ? paths : [paths], recursive);
188
188
  if (data.length === 0) {
189
189
  console.warn("No data to push. Skipping");
@@ -8334,165 +8334,10 @@ async function evaluate({ data, executor, evaluators, groupName, name, metadata,
8334
8334
  //#region src/integrations/eve.ts
8335
8335
  const logger$2 = require_utils.initializeLogger();
8336
8336
  const EVE_REPORTER_TRACER_NAME = "@lmnr-ai/lmnr/eve-reporter";
8337
- const EVE_CLIENT_MODULE = "eve/client";
8338
- const TRACEPARENT_HEADER$1 = "traceparent";
8339
- /** Same wire name the Temporal instrumentation uses (`temporal/consts.ts`). */
8340
- const LAMINAR_SPAN_CONTEXT_HEADER$1 = "x-lmnr-span-context";
8341
- /**
8342
- * Root span name for every propagated eval, and it never changes. eve calls
8343
- * `ClientSession.send` before any hook that knows which eval is in flight, and
8344
- * eve exposes no `onEvalStart`, so the id simply is not knowable here. Renaming
8345
- * the span later would invalidate the `lmnr.span.path` the processor already
8346
- * stamped on it and on every descendant, and would make the path we propagate
8347
- * to the agent stale. The eval id rides on `lmnr.eve.eval.id`, on the trace
8348
- * metadata, and on the datapoint instead.
8349
- */
8350
- const ROOT_SPAN_NAME = "eve eval";
8351
- const EXECUTOR_SPAN_NAME = "executor";
8352
- /** Prototypes already patched — `send` must never be wrapped twice. */
8353
- const patchedSendPrototypes = /* @__PURE__ */ new WeakSet();
8354
- /**
8355
- * Per-eve-client-session state. Keyed off the `ClientSession` instance rather
8356
- * than an AsyncLocalStorage frame or a "current eval" global: eve runs up to
8357
- * `maxConcurrency` evals at once, and each eval builds its own `ClientSession`
8358
- * (`EvalSessionManager` calls `client.session()` per `EvalSessionDriver`), so
8359
- * the instance is the only correct correlation key available inside `send`.
8360
- */
8361
- const traceByClientSession = /* @__PURE__ */ new WeakMap();
8362
- /** eve session id -> trace, read by `onEvalComplete` via `result.sessionId`. */
8363
- const traceByEveSessionId = /* @__PURE__ */ new Map();
8364
- /** Set by the running reporter's `onRunStart`; cleared by `onRunComplete`. */
8365
- let activeSessionTraceFactory = null;
8366
- /** Mirrors eve's own `normalizeSendTurnInput`, then merges the trace headers. */
8367
- const withTraceHeaders = (input, headers) => {
8368
- const payload = typeof input === "string" ? { message: input } : { ...input };
8369
- payload.headers = {
8370
- ...payload.headers ?? {},
8371
- ...headers
8372
- };
8373
- return payload;
8374
- };
8375
- const openSessionTrace = (clientSession) => {
8376
- try {
8377
- const existing = traceByClientSession.get(clientSession);
8378
- if (existing) return existing;
8379
- const minted = activeSessionTraceFactory?.();
8380
- if (minted) traceByClientSession.set(clientSession, minted);
8381
- return minted;
8382
- } catch (error) {
8383
- logger$2.warn(`Laminar eve reporter: failed to open a session trace: ` + require_utils.errorMessage(error));
8384
- return;
8385
- }
8386
- };
8387
- /**
8388
- * eve assigns the session id on the POST that `send` awaits, so the returned
8389
- * `MessageResponse` is the earliest place the runner can bind the eve session
8390
- * id to the trace it minted.
8391
- */
8392
- const recordEveSessionId = (response, sessionTrace) => {
8393
- const sessionId = response?.sessionId;
8394
- if (typeof sessionId !== "string" || sessionId.length === 0) return;
8395
- sessionTrace.sessionIds.add(sessionId);
8396
- traceByEveSessionId.set(sessionId, sessionTrace);
8397
- };
8398
- /**
8399
- * Build the propagation headers for the EXECUTOR span. Plain HTTP strings — eve
8400
- * forwards `SendTurnPayload.headers` verbatim to `fetch`.
8401
- *
8402
- * The Laminar context carries the full path. Laminar nests spans by
8403
- * `lmnr.span.ids_path`, not by OTel `parentSpanId`, so a receiver that adopts
8404
- * only the ids lands its spans as a second root on our trace. The path is what
8405
- * makes the agent's subtree render UNDER the executor. Both paths are constants
8406
- * here — neither the root nor the executor span is ever renamed.
8407
- */
8408
- const buildTraceHeaders = (rootSpan, executorSpan) => {
8409
- const spanContext = executorSpan.spanContext();
8410
- const laminarContext = {
8411
- traceId: require_utils.otelTraceIdToUUID(spanContext.traceId),
8412
- spanId: require_utils.otelSpanIdToUUID(spanContext.spanId),
8413
- isRemote: true,
8414
- spanPath: [ROOT_SPAN_NAME, EXECUTOR_SPAN_NAME],
8415
- spanIdsPath: [require_utils.otelSpanIdToUUID(rootSpan.spanContext().spanId), require_utils.otelSpanIdToUUID(spanContext.spanId)]
8416
- };
8417
- return {
8418
- [TRACEPARENT_HEADER$1]: `00-${spanContext.traceId}-${spanContext.spanId}-01`,
8419
- [LAMINAR_SPAN_CONTEXT_HEADER$1]: JSON.stringify(laminarContext)
8420
- };
8421
- };
8422
- /**
8423
- * Bind this session's ROOT span into the calling eval's async context, so work
8424
- * the eval does in the RUNNER process after `send` resolves lands on this trace
8425
- * instead of opening its own. The concrete case is `t.judge.autoevals.*`: eve
8426
- * runs the judge's model call in the runner, and Laminar's AI SDK integration
8427
- * parents it to `LaminarContextManager.getContext()`.
8428
- *
8429
- * The root — not a pre-opened judge span — is the binding target on purpose.
8430
- * An eval may record any number of judge assertions, and each one runs its own
8431
- * model call. Binding the root gives every judge its own child of the root; a
8432
- * single pre-opened parent would collect all of them under whichever judge
8433
- * happened to run first.
8434
- *
8435
- * `pushContext` uses `AsyncLocalStorage.enterWith`, so this MUST run
8436
- * synchronously before `send`'s first `await` — at that point we are still
8437
- * executing in the caller's async context, so the store we set is the one
8438
- * `test(t)` continues with. Each eval's `test(t)` runs in its own async chain,
8439
- * so concurrent evals never see each other's root span; do NOT replace this
8440
- * with the process-global context stack, which would cross-attribute.
8441
- *
8442
- * Judge spans reach Laminar only when the runner opts in — `Laminar.initialize()`
8443
- * plus a globally registered AI SDK telemetry integration. Without that the
8444
- * binding is inert, never wrong.
8445
- *
8446
- * Never popped: the store dies with the eval's async chain.
8447
- */
8448
- const bindEvalContext = (sessionTrace) => {
8449
- try {
8450
- LaminarContextManager.pushContext(_opentelemetry_api.trace.setSpan(_opentelemetry_api.ROOT_CONTEXT, sessionTrace.rootSpan));
8451
- } catch (error) {
8452
- logger$2.warn(`Laminar eve reporter: failed to bind the eval context: ` + require_utils.errorMessage(error));
8453
- }
8454
- };
8455
- /**
8456
- * Wrap `ClientSession.prototype.send` so every eve turn carries the trace
8457
- * context of a runner-minted Laminar span. Idempotent per prototype.
8458
- *
8459
- * @internal Exported for tests, which drive it with a stub class so the suite
8460
- * keeps no dependency on eve.
8461
- */
8462
- const patchEveClientSession = (sessionClass) => {
8463
- const prototype = sessionClass?.prototype;
8464
- const original = prototype?.send;
8465
- if (!prototype || typeof original !== "function") return false;
8466
- if (patchedSendPrototypes.has(prototype)) return true;
8467
- prototype.send = async function(input) {
8468
- const sessionTrace = openSessionTrace(this);
8469
- if (!sessionTrace) return original.call(this, input);
8470
- bindEvalContext(sessionTrace);
8471
- const response = await original.call(this, withTraceHeaders(input, sessionTrace.headers));
8472
- try {
8473
- recordEveSessionId(response, sessionTrace);
8474
- } catch (error) {
8475
- logger$2.warn(`Laminar eve reporter: failed to record the eve session id: ` + require_utils.errorMessage(error));
8476
- }
8477
- return response;
8478
- };
8479
- patchedSendPrototypes.add(prototype);
8480
- return true;
8481
- };
8482
- /**
8483
- * Load eve's `ClientSession` without depending on eve. Mirrors the guarded
8484
- * dynamic import in eve's own Braintrust reporter (`loadBraintrustSdk`), except
8485
- * a missing eve degrades instead of throwing — the reporter must keep working
8486
- * when it is registered outside an eve run.
8487
- */
8488
- const loadEveClientSession = async () => {
8489
- try {
8490
- const sessionClass = (await import(EVE_CLIENT_MODULE)).ClientSession;
8491
- return typeof sessionClass === "function" ? sessionClass : void 0;
8492
- } catch {
8493
- return;
8494
- }
8495
- };
8337
+ const OTEL_TRACE_ID_RE = /^[0-9a-f]{32}$/iu;
8338
+ const OTEL_SPAN_ID_RE = /^[0-9a-f]{16}$/iu;
8339
+ const ZERO_TRACE_ID = "0".repeat(32);
8340
+ const ZERO_SPAN_ID = "0".repeat(16);
8496
8341
  const didAssertionPass = (assertion) => {
8497
8342
  if (typeof assertion.passed === "boolean") return assertion.passed;
8498
8343
  if (typeof assertion.score === "number") {
@@ -8501,14 +8346,8 @@ const didAssertionPass = (assertion) => {
8501
8346
  }
8502
8347
  return true;
8503
8348
  };
8504
- /**
8505
- * Keep Laminar's top-level score columns stable across heterogeneous eve evals.
8506
- * Detailed assertion outcomes live in metadata; sparse per-assertion score keys
8507
- * make Laminar's eval UI look incomplete when different eval files assert
8508
- * different things.
8509
- */
8510
- const resultToScores = (result) => {
8511
- const assertions = result.assertions ?? [];
8349
+ /** Keep Laminar's score columns stable across heterogeneous eve evals. */
8350
+ const resultToScores = (result, assertions) => {
8512
8351
  const gateAssertions = assertions.filter((assertion) => assertion.severity === "gate");
8513
8352
  const thresholdedSoftAssertions = assertions.filter((assertion) => assertion.severity === "soft" && typeof assertion.threshold === "number");
8514
8353
  return {
@@ -8526,7 +8365,6 @@ const assertionsMetadata = (assertions) => assertions.map((assertion) => ({
8526
8365
  message: assertion.message,
8527
8366
  metadata: assertion.metadata
8528
8367
  }));
8529
- /** Surface failed assertions in metadata so a failing run is debuggable. */
8530
8368
  const failedAssertionsMetadata = (assertions) => {
8531
8369
  const failed = assertions.filter((assertion) => !didAssertionPass(assertion)).map((assertion) => ({
8532
8370
  name: assertion.name,
@@ -8534,79 +8372,62 @@ const failedAssertionsMetadata = (assertions) => {
8534
8372
  }));
8535
8373
  return failed.length > 0 ? { failedAssertions: failed } : {};
8536
8374
  };
8537
- const traceIdFromSpan = (span) => require_utils.otelTraceIdToUUID(span.spanContext().traceId);
8375
+ const validTraceContext = (context) => context != null && typeof context.traceId === "string" && OTEL_TRACE_ID_RE.test(context.traceId) && context.traceId !== ZERO_TRACE_ID && typeof context.spanId === "string" && OTEL_SPAN_ID_RE.test(context.spanId) && context.spanId !== ZERO_SPAN_ID && typeof context.traceFlags === "number" && Number.isInteger(context.traceFlags) && context.traceFlags >= 0 && context.traceFlags <= 255;
8376
+ const normalizeTraceContexts = (contexts) => Array.isArray(contexts) ? contexts.filter(validTraceContext) : [];
8377
+ const isSampledTraceContext = (context) => (context.traceFlags & 1) === 1;
8378
+ const selectTraceContext = (contexts) => contexts.find((context) => context.primary === true && isSampledTraceContext(context)) ?? contexts.find(isSampledTraceContext);
8379
+ const traceContextMetadata = (contexts) => contexts.map((context) => ({
8380
+ traceId: context.traceId,
8381
+ spanId: context.spanId,
8382
+ traceFlags: context.traceFlags,
8383
+ primary: context.primary,
8384
+ sessionId: context.sessionId
8385
+ }));
8538
8386
  /**
8539
- * A Laminar reporter for eve evals. Register it globally in `evals.config.ts`
8540
- * or per-eval via the `reporters` field — eve runs and grades each eval, then
8541
- * hands the graded result here, and this reporter ships it to Laminar as an
8542
- * evaluation run with one datapoint per eval.
8543
- *
8544
- * The datapoint's trace id is never looked up — the reporter OWNS the trace.
8545
- * `patchEveClientSession` wraps eve's `ClientSession.send` and mints the
8546
- * EVALUATION root + EXECUTOR child on the first turn of each session, so the
8547
- * runner already knows the trace id and pushes it to the agent as a
8548
- * `traceparent` header. The eve session id is only the in-process key that ties
8549
- * the send we instrumented to the graded result eve hands back later.
8550
- *
8551
- * Two outcomes, recorded on `metadata.traceResolution`:
8552
- *
8553
- * 1. `"propagated"` — the normal path, above. The agent joins the trace only
8554
- * when `agent/instrumentation.ts` sets `traceChannelRequests: true` and the
8555
- * agent process runs with `WORKFLOW_TRACE_MODE=continuous`.
8556
- * 2. `"reporter-fallback"` — no session trace was minted (propagation disabled,
8557
- * or eve produced no session id). The reporter opens its own EVALUATION span
8558
- * so the grade still lands somewhere debuggable; it holds no agent work.
8559
- *
8560
- * `t.judge.autoevals.*` model calls run in the RUNNER process, and they reach
8561
- * Laminar only when the runner registers an AI SDK telemetry integration. eve
8562
- * imports `generateText` inside its own bundle, so `wrapAISDK` cannot reach it —
8563
- * but AI SDK v7 reads registered integrations off `globalThis`, which does. One
8564
- * line in `evals.config.ts`:
8387
+ * Reports eve eval results to Laminar.
8565
8388
  *
8566
- * ```ts
8567
- * registerTelemetry(new LaminarAiSdkTelemetry());
8568
- * ```
8389
+ * Eve 0.35+ gives reporters every native trace context observed during an eval.
8390
+ * The reporter links the datapoint to the first sampled primary trace (or the
8391
+ * first sampled trace when there is no sampled primary), records every valid
8392
+ * context in metadata, and adds
8393
+ * a small EVALUATION span to the selected native trace for the grade. When eve
8394
+ * reports no usable context, a reporter-owned fallback trace preserves the
8395
+ * result without pretending it contains agent work.
8569
8396
  *
8570
- * Judge spans and eve eval spans then share ONE tracer provider — whichever of
8571
- * the two initialized Laminar first (see {@link LaminarReporter.ensureTracing}).
8572
- * Each judge gets its own span under the eval's root — see
8573
- * {@link bindEvalContext}, and {@link LaminarReporter.flush} for how spans ship.
8574
- * Without that line the reporter still records every assertion's score as an
8575
- * EVALUATOR span; only the model call is missing.
8397
+ * The agent must export its native OpenTelemetry spans to the same Laminar
8398
+ * project. In the eve agent, declare a destination like:
8576
8399
  *
8577
- * @example
8578
8400
  * ```ts
8579
- * import { defineEvalConfig } from "eve/evals";
8580
- * import { LaminarReporter } from "@lmnr-ai/lmnr";
8401
+ * import { LaminarSpanProcessor } from "@lmnr-ai/lmnr";
8402
+ * import { otelIntegration } from "eve/instrumentation/otel";
8581
8403
  *
8582
- * export default defineEvalConfig({
8583
- * reporters: [new LaminarReporter({ name: "weather-agent" })],
8404
+ * export default otelIntegration({
8405
+ * spanProcessors: [new LaminarSpanProcessor()],
8584
8406
  * });
8585
8407
  * ```
8586
8408
  */
8587
8409
  var LaminarReporter = class {
8588
8410
  constructor(options = {}) {
8589
8411
  this.evalsById = /* @__PURE__ */ new Map();
8590
- this.index = 0;
8591
- this.openSessionTraces = /* @__PURE__ */ new Set();
8592
- this.sessionTraceFactory = () => this.mintSessionTrace();
8412
+ this.evalIndexById = /* @__PURE__ */ new Map();
8413
+ this.nextFallbackIndex = 0;
8593
8414
  this.options = options;
8594
8415
  }
8595
8416
  async onRunStart(evaluations, target) {
8596
8417
  this.evalId = void 0;
8597
- this.evalsById = new Map(evaluations.filter((evaluation) => typeof evaluation.id === "string").map((evaluation) => [evaluation.id, evaluation]));
8598
- this.index = 0;
8599
- this.finishOpenSessionTraces();
8418
+ const identifiedEvaluations = evaluations.map((evaluation, index) => ({
8419
+ evaluation,
8420
+ index
8421
+ })).filter((item) => typeof item.evaluation.id === "string");
8422
+ this.evalsById = new Map(identifiedEvaluations.map(({ evaluation }) => [evaluation.id, evaluation]));
8423
+ this.evalIndexById = new Map(identifiedEvaluations.map(({ evaluation, index }) => [evaluation.id, index]));
8424
+ this.nextFallbackIndex = evaluations.length;
8600
8425
  try {
8601
8426
  this.client = this.options.client ?? new require_utils.LaminarClient({
8602
8427
  baseUrl: this.options.baseUrl,
8603
8428
  projectApiKey: this.options.projectApiKey
8604
8429
  });
8605
8430
  this.ensureTracing();
8606
- if (this.options.propagateTraceContext !== false) {
8607
- activeSessionTraceFactory = this.sessionTraceFactory;
8608
- patchEveClientSession(await loadEveClientSession());
8609
- }
8610
8431
  const evaluation = await this.client.evals.init(this.options.name, this.options.groupName, {
8611
8432
  ...this.options.metadata ?? {},
8612
8433
  source: "eve",
@@ -8619,26 +8440,25 @@ var LaminarReporter = class {
8619
8440
  process.stdout.write(`\nLaminar: check eve eval results at ${url}\n`);
8620
8441
  } catch (error) {
8621
8442
  logger$2.error(`Laminar eve reporter: failed to start run: ${require_utils.errorMessage(error)}`);
8622
- if (!this.evalId) {
8623
- this.client = void 0;
8624
- if (activeSessionTraceFactory === this.sessionTraceFactory) activeSessionTraceFactory = null;
8625
- }
8443
+ if (!this.evalId) this.client = void 0;
8626
8444
  }
8627
8445
  }
8628
- async onEvalComplete(result) {
8446
+ async onEvalComplete(result, context) {
8629
8447
  const client = this.client;
8630
8448
  const evalId = this.evalId;
8631
8449
  if (!client || !evalId) return;
8632
- const index = this.index++;
8450
+ const index = typeof result.id === "string" ? this.evalIndexById.get(result.id) ?? this.nextFallbackIndex++ : this.nextFallbackIndex++;
8633
8451
  const task = result.result ?? {};
8634
8452
  const derived = task.derived ?? {};
8635
- const assertions = result.assertions ?? [];
8636
- const evalDescription = (typeof result.id === "string" ? this.evalsById.get(result.id) : void 0)?.description;
8637
- const traceResolution = this.resolveDatapointTrace({
8638
- evalId: result.id ?? String(index),
8453
+ const assertions = Array.isArray(result.assertions) ? result.assertions.filter((assertion) => assertion != null && typeof assertion === "object") : [];
8454
+ const evalDefinition = typeof result.id === "string" ? this.evalsById.get(result.id) : void 0;
8455
+ const evalDescription = context?.evaluation?.description ?? evalDefinition?.description;
8456
+ const contexts = normalizeTraceContexts(context?.traceContexts ?? task.traceContexts ?? []);
8457
+ const gradeTrace = this.openGradeTrace({
8458
+ contexts,
8639
8459
  description: evalDescription,
8640
- verdict: result.verdict,
8641
- sessionId: task.sessionId
8460
+ evalId: result.id ?? String(index),
8461
+ verdict: result.verdict
8642
8462
  });
8643
8463
  try {
8644
8464
  const datapointId = await client.evals.createDatapoint({
@@ -8646,17 +8466,19 @@ var LaminarReporter = class {
8646
8466
  data: result.id ?? null,
8647
8467
  target: task.finalMessage ?? null,
8648
8468
  index,
8649
- traceId: traceResolution.traceId,
8469
+ traceId: gradeTrace.traceId,
8650
8470
  metadata: {
8651
8471
  name: result.id,
8652
8472
  verdict: result.verdict,
8653
8473
  status: task.status,
8654
8474
  description: evalDescription,
8655
8475
  sessionId: task.sessionId,
8656
- traceResolution: traceResolution.source,
8476
+ traceResolution: gradeTrace.source,
8477
+ selectedTraceContext: gradeTrace.selected ? traceContextMetadata([gradeTrace.selected])[0] : void 0,
8478
+ traceContexts: traceContextMetadata(gradeTrace.contexts),
8657
8479
  skipReason: result.skipReason,
8658
8480
  error: result.error,
8659
- toolCalls: (derived.toolCalls ?? []).map((call) => call.name).filter((name) => typeof name === "string"),
8481
+ toolCalls: (Array.isArray(derived.toolCalls) ? derived.toolCalls : []).map((call) => call?.name).filter((name) => typeof name === "string"),
8660
8482
  toolCallCount: derived.toolCallCount,
8661
8483
  parked: derived.parked,
8662
8484
  failureCode: derived.failureCode,
@@ -8665,44 +8487,30 @@ var LaminarReporter = class {
8665
8487
  ...failedAssertionsMetadata(assertions)
8666
8488
  }
8667
8489
  });
8668
- const executorOutput = task.output ?? task.finalMessage ?? null;
8669
8490
  await client.evals.updateDatapoint({
8670
8491
  evalId,
8671
8492
  datapointId,
8672
- scores: resultToScores(result),
8673
- executorOutput
8493
+ scores: resultToScores(result, assertions),
8494
+ executorOutput: task.output ?? task.finalMessage ?? null
8674
8495
  });
8675
8496
  } catch (error) {
8676
- if (traceResolution.source === "reporter-fallback") traceResolution.span.recordException(error);
8497
+ gradeTrace.span.recordException(error);
8677
8498
  logger$2.error(`Laminar eve reporter: failed to report eval "${result.id ?? index}": ${require_utils.errorMessage(error)}`);
8678
8499
  } finally {
8679
- if (traceResolution.source === "reporter-fallback") traceResolution.span.end();
8680
- if (traceResolution.source === "propagated") this.finishSessionTrace(traceResolution.sessionTrace, {
8681
- evalId: result.id,
8682
- description: evalDescription,
8683
- verdict: result.verdict,
8684
- status: task.status,
8500
+ this.finishGradeTrace(gradeTrace, {
8501
+ assertions,
8685
8502
  output: task.output ?? task.finalMessage ?? null,
8686
- assertions
8503
+ status: task.status
8687
8504
  });
8688
8505
  await this.flush();
8689
8506
  }
8690
8507
  }
8691
- async onRunComplete() {
8692
- this.finishOpenSessionTraces();
8693
- if (activeSessionTraceFactory === this.sessionTraceFactory) activeSessionTraceFactory = null;
8508
+ async onRunComplete(_summary) {
8694
8509
  await this.flush();
8695
8510
  this.evalId = void 0;
8696
8511
  this.evalsById.clear();
8512
+ this.evalIndexById.clear();
8697
8513
  }
8698
- /**
8699
- * Point the process's Laminar tracing at this reporter's project, unless
8700
- * something already initialized it. Mirrors `LaminarAiSdkTelemetry`, which
8701
- * initializes the same way for judge spans — so eve eval spans and judge
8702
- * spans share ONE tracer provider however the run was set up. Without this,
8703
- * `getTracer()` returns a noop whenever nobody registered the AI SDK
8704
- * integration, and every eve span would be dropped silently.
8705
- */
8706
8514
  ensureTracing() {
8707
8515
  if (Laminar.initialized()) {
8708
8516
  const ourKey = this.client?.apiKey ?? this.options.projectApiKey;
@@ -8718,140 +8526,79 @@ var LaminarReporter = class {
8718
8526
  disableBatch: true
8719
8527
  });
8720
8528
  }
8721
- /**
8722
- * Flush the process's Laminar pipeline — eve eval spans AND judge spans, which
8723
- * now share one provider. eve ends `eve eval` with `process.exit()` (skipping
8724
- * `beforeExit`), so these calls are the only points at which spans ship.
8725
- * Never `shutdown()`: the pipeline may belong to the host, not to us.
8726
- */
8727
8529
  async flush() {
8728
8530
  try {
8729
8531
  await Laminar.flush();
8730
8532
  } catch (error) {
8731
- logger$2.warn(`Laminar eve reporter: failed to flush spans: ` + require_utils.errorMessage(error));
8533
+ logger$2.warn(`Laminar eve reporter: failed to flush spans: ${require_utils.errorMessage(error)}`);
8732
8534
  }
8733
8535
  }
8734
- /**
8735
- * Open the eval's Laminar spans for one eve client session: an EVALUATION
8736
- * root plus the EXECUTOR child the agent's turn is parented to. Called from
8737
- * the patched `send`, so it must never throw into the user's eval.
8738
- */
8739
- mintSessionTrace() {
8536
+ openGradeTrace({ contexts, description, evalId, verdict }) {
8537
+ const selected = selectTraceContext(contexts);
8740
8538
  const tracer = getTracer();
8741
- const rootSpan = tracer.startSpan(ROOT_SPAN_NAME, { attributes: {
8742
- [require_utils.SPAN_TYPE]: "EVALUATION",
8743
- [require_utils.TRACE_TYPE]: "EVALUATION",
8744
- "lmnr.eve.reporter": EVE_REPORTER_TRACER_NAME
8745
- } }, _opentelemetry_api.ROOT_CONTEXT);
8746
- const executorSpan = tracer.startSpan(EXECUTOR_SPAN_NAME, { attributes: {
8747
- [require_utils.SPAN_TYPE]: "EXECUTOR",
8748
- [require_utils.TRACE_TYPE]: "EVALUATION"
8749
- } }, _opentelemetry_api.trace.setSpan(_opentelemetry_api.ROOT_CONTEXT, rootSpan));
8750
- const sessionTrace = {
8751
- traceId: require_utils.otelTraceIdToUUID(rootSpan.spanContext().traceId),
8752
- rootSpan,
8753
- executorSpan,
8754
- tracer,
8755
- headers: buildTraceHeaders(rootSpan, executorSpan),
8756
- sessionIds: /* @__PURE__ */ new Set(),
8757
- ended: false
8758
- };
8759
- this.openSessionTraces.add(sessionTrace);
8760
- return sessionTrace;
8761
- }
8762
- /** Grade, close and release one session trace. Idempotent. */
8763
- finishSessionTrace(sessionTrace, grade) {
8764
- if (sessionTrace.ended) return;
8765
- sessionTrace.ended = true;
8766
- this.openSessionTraces.delete(sessionTrace);
8767
- for (const sessionId of sessionTrace.sessionIds) traceByEveSessionId.delete(sessionId);
8768
- try {
8769
- sessionTrace.rootSpan.setAttributes({
8770
- "lmnr.eve.eval.id": grade.evalId ?? "",
8771
- "lmnr.eve.eval.verdict": grade.verdict ?? "",
8772
- "lmnr.eve.eval.status": grade.status ?? "",
8773
- [require_utils.SPAN_INPUT]: JSON.stringify({
8774
- evalId: grade.evalId,
8775
- description: grade.description
8776
- }),
8777
- [require_utils.SPAN_OUTPUT]: JSON.stringify(grade.output ?? null),
8778
- ...require_utils.metadataToAttributes({
8779
- source: "eve",
8780
- eveEvalId: grade.evalId ?? "",
8781
- eveEvalDescription: grade.description ?? "",
8782
- eveEvalVerdict: grade.verdict ?? "",
8783
- eveSessionId: [...sessionTrace.sessionIds].join(",")
8784
- })
8785
- });
8786
- sessionTrace.executorSpan.setAttribute(require_utils.SPAN_OUTPUT, JSON.stringify(grade.output ?? null));
8787
- this.recordEvaluatorSpans(sessionTrace, grade.assertions ?? []);
8788
- sessionTrace.executorSpan.end();
8789
- sessionTrace.rootSpan.end();
8790
- } catch (error) {
8791
- logger$2.warn(`Laminar eve reporter: failed to finish the session trace: ` + require_utils.errorMessage(error));
8792
- }
8793
- }
8794
- /**
8795
- * One EVALUATOR span per eve assertion, judge or not. They are created at
8796
- * grade time (the assertions only exist then) as short children of the
8797
- * still-open root. No explicit parent path is needed: the root keeps the name
8798
- * it was minted with, so the processor's cached path for it is still correct.
8799
- */
8800
- recordEvaluatorSpans(sessionTrace, assertions) {
8801
- const rootContext = _opentelemetry_api.trace.setSpan(_opentelemetry_api.ROOT_CONTEXT, sessionTrace.rootSpan);
8802
- for (const assertion of assertions) {
8803
- const name = assertion.name ?? "assertion";
8804
- const output = JSON.stringify({
8805
- score: assertion.score ?? null,
8806
- passed: didAssertionPass(assertion),
8807
- severity: assertion.severity,
8808
- threshold: assertion.threshold,
8809
- message: assertion.message
8810
- });
8811
- sessionTrace.tracer.startSpan(name, { attributes: {
8812
- [require_utils.SPAN_TYPE]: "EVALUATOR",
8813
- [require_utils.TRACE_TYPE]: "EVALUATION",
8814
- [require_utils.SPAN_OUTPUT]: output
8815
- } }, rootContext).end();
8816
- }
8817
- }
8818
- finishOpenSessionTraces() {
8819
- for (const sessionTrace of [...this.openSessionTraces]) this.finishSessionTrace(sessionTrace, {});
8820
- }
8821
- /**
8822
- * Resolve the trace this datapoint links to. There is no lookup: either
8823
- * `patchEveClientSession` already minted the session's trace (and we know its
8824
- * id), or we open a reporter-owned EVALUATION span to hold the grade.
8825
- */
8826
- resolveDatapointTrace({ evalId, description, verdict, sessionId }) {
8827
- const sessionTrace = sessionId ? traceByEveSessionId.get(sessionId) : void 0;
8828
- if (sessionTrace) return {
8829
- traceId: sessionTrace.traceId,
8830
- source: "propagated",
8831
- sessionTrace
8832
- };
8833
- const span = getTracer().startSpan(`eve eval ${evalId}`, { attributes: {
8539
+ const parentContext = selected ? _opentelemetry_api.trace.setSpanContext(_opentelemetry_api.ROOT_CONTEXT, this.toSpanContext(selected)) : _opentelemetry_api.ROOT_CONTEXT;
8540
+ const source = selected ? "eve-native" : "reporter-fallback";
8541
+ const span = tracer.startSpan(`eve eval ${evalId}`, { attributes: {
8834
8542
  [require_utils.SPAN_TYPE]: "EVALUATION",
8835
8543
  [require_utils.TRACE_TYPE]: "EVALUATION",
8836
8544
  "lmnr.eve.reporter": EVE_REPORTER_TRACER_NAME,
8837
8545
  "lmnr.eve.eval.id": evalId,
8838
8546
  "lmnr.eve.eval.verdict": verdict ?? "",
8839
- ...sessionId ? { "lmnr.eve.session.id": sessionId } : {},
8840
- "lmnr.eve.trace_resolution": "reporter-fallback",
8547
+ "lmnr.eve.trace_resolution": source,
8841
8548
  ...require_utils.metadataToAttributes({
8842
8549
  source: "eve",
8843
8550
  eveEvalId: evalId,
8844
8551
  eveEvalDescription: description ?? "",
8845
8552
  eveEvalVerdict: verdict ?? "",
8846
- eveSessionId: sessionId ?? ""
8553
+ eveSessionId: selected?.sessionId ?? "",
8554
+ eveTraceCount: contexts.length
8555
+ }),
8556
+ [require_utils.SPAN_INPUT]: JSON.stringify({
8557
+ evalId,
8558
+ description
8847
8559
  })
8848
- } }, _opentelemetry_api.ROOT_CONTEXT);
8560
+ } }, parentContext);
8561
+ return {
8562
+ traceId: require_utils.otelTraceIdToUUID(span.spanContext().traceId),
8563
+ source,
8564
+ span,
8565
+ tracer,
8566
+ selected,
8567
+ contexts
8568
+ };
8569
+ }
8570
+ toSpanContext(context) {
8849
8571
  return {
8850
- traceId: traceIdFromSpan(span),
8851
- source: "reporter-fallback",
8852
- span
8572
+ traceId: context.traceId.toLowerCase(),
8573
+ spanId: context.spanId.toLowerCase(),
8574
+ traceFlags: context.traceFlags,
8575
+ isRemote: true
8853
8576
  };
8854
8577
  }
8578
+ finishGradeTrace(gradeTrace, grade) {
8579
+ try {
8580
+ gradeTrace.span.setAttributes({
8581
+ "lmnr.eve.eval.status": grade.status ?? "",
8582
+ [require_utils.SPAN_OUTPUT]: JSON.stringify(grade.output ?? null)
8583
+ });
8584
+ const parentContext = _opentelemetry_api.trace.setSpan(_opentelemetry_api.ROOT_CONTEXT, gradeTrace.span);
8585
+ for (const assertion of grade.assertions) gradeTrace.tracer.startSpan(assertion.name ?? "assertion", { attributes: {
8586
+ [require_utils.SPAN_TYPE]: "EVALUATOR",
8587
+ [require_utils.TRACE_TYPE]: "EVALUATION",
8588
+ [require_utils.SPAN_OUTPUT]: JSON.stringify({
8589
+ score: assertion.score ?? null,
8590
+ passed: didAssertionPass(assertion),
8591
+ severity: assertion.severity,
8592
+ threshold: assertion.threshold,
8593
+ message: assertion.message
8594
+ })
8595
+ } }, parentContext).end();
8596
+ } catch (error) {
8597
+ logger$2.warn(`Laminar eve reporter: failed to finish the grade trace: ${require_utils.errorMessage(error)}`);
8598
+ } finally {
8599
+ gradeTrace.span.end();
8600
+ }
8601
+ }
8855
8602
  };
8856
8603
  //#endregion
8857
8604
  //#region src/debug/hash.ts