@tangle-network/agent-eval 0.36.0 → 0.38.0

package/dist/index.js

@@ -195,6 +195,8 @@ import {
   buildTraceAnalystTools,
   buildTraceInsightContext,
   buildTraceInsightPrompt,
+  createOtelExporter,
+  createOtelTracingStore,
   createReplayFetch,
   defaultTraceInsightPanel,
   describeTraceInsightScope,
@@ -202,6 +204,7 @@ import {
   exportRunAsOtlp,
   inferDomainKeywords,
   iterateRawCalls,
+  otelRunCompleteHook,
   planTraceInsightQuestions,
   redactString,
   redactValue,
@@ -209,7 +212,7 @@ import {
   tokenizeDomainWords,
   traceAnalystFunctionGroup,
   traceAnalystOnRunComplete
-} from "./chunk-HIO4UIS5.js";
+} from "./chunk-L5UNCDAJ.js";
 import {
   aggregateLlm,
   argHash,
@@ -11031,6 +11034,197 @@ function aggregateTrialsByMode(trials, opts) {
     metrics: meanMetrics(counted.map((t) => t.metrics ?? {}))
   };
 }
+
+// src/otel-pipeline.ts
+function withOtelPipeline(opts) {
+  const config = {
+    ...opts?.otelConfig,
+    resourceAttributes: {
+      ...opts?.pipelineKind ? { "pipeline.kind": opts.pipelineKind } : {},
+      ...opts?.pipelineId ? { "pipeline.id": opts.pipelineId } : {},
+      ...opts?.otelConfig?.resourceAttributes
+    }
+  };
+  const exporter = createOtelExporter(config);
+  return {
+    exporter,
+    async shutdown() {
+      if (exporter) await exporter.shutdown();
+    }
+  };
+}
+function isOtelConfigured() {
+  return !!(typeof process !== "undefined" && process.env.OTEL_EXPORTER_OTLP_ENDPOINT);
+}
+
+// src/traced-judges.ts
+function traceJudge(judge, judgeName, opts) {
+  return async (tc, input) => {
+    const span = await opts.emitter.span({
+      kind: "llm",
+      name: `judge:${judgeName}`,
+      parentSpanId: opts.parentSpanId,
+      attributes: {
+        "judge.name": judgeName,
+        "eval.phase": "judge"
+      }
+    });
+    try {
+      const scores = await judge(tc, input);
+      const composite = scores.length > 0 ? scores.reduce((sum3, s) => sum3 + s.score, 0) / scores.length : 0;
+      await span.end({
+        attributes: {
+          "judge.name": judgeName,
+          "judge.composite_score": composite,
+          "judge.dimension_count": scores.length,
+          "eval.phase": "judge"
+        }
+      });
+      return scores;
+    } catch (err) {
+      await span.fail(err instanceof Error ? err : String(err));
+      throw err;
+    }
+  };
+}
+function traceJudgeEnsemble(judges, judgeNames, opts) {
+  return async (tc, input) => {
+    const ensembleSpan = await opts.emitter.span({
+      kind: "custom",
+      name: "judge:ensemble",
+      parentSpanId: opts.parentSpanId,
+      attributes: {
+        "judge.ensemble_size": judges.length,
+        "eval.phase": "judge"
+      }
+    });
+    try {
+      const allScores = [];
+      for (let i = 0; i < judges.length; i++) {
+        const judge = judges[i];
+        const name = judgeNames[i] ?? `judge_${i}`;
+        const tracedFn = traceJudge(judge, name, {
+          emitter: opts.emitter,
+          parentSpanId: ensembleSpan.span.spanId
+        });
+        const scores = await tracedFn(tc, input);
+        allScores.push(...scores);
+      }
+      const composite = allScores.length > 0 ? allScores.reduce((sum3, s) => sum3 + s.score, 0) / allScores.length : 0;
+      await ensembleSpan.end({
+        attributes: {
+          "judge.ensemble_size": judges.length,
+          "judge.composite_score": composite,
+          "judge.total_dimensions": allScores.length,
+          "eval.phase": "judge"
+        }
+      });
+      return allScores;
+    } catch (err) {
+      await ensembleSpan.fail(err instanceof Error ? err : String(err));
+      throw err;
+    }
+  };
+}
+
+// src/traced-analyst.ts
+async function tracedAnalyzeTraces(input, options, traceOpts) {
+  const parentSpan = await traceOpts.emitter.span({
+    kind: "custom",
+    name: "analyst:analyze-traces",
+    parentSpanId: traceOpts.parentSpanId,
+    attributes: {
+      "analyst.question_length": input.question.length,
+      "analyst.max_turns": options.maxTurns ?? 12,
+      "analyst.max_depth": options.maxDepth ?? 1,
+      "eval.phase": "analyst"
+    }
+  });
+  const originalOnTurn = options.onTurn;
+  const turnSpanIds = [];
+  const wrappedOptions = {
+    ...options,
+    onTurn: async (turn) => {
+      const turnSpan = await traceOpts.emitter.span({
+        kind: "custom",
+        name: `analyst:turn-${turn.turn}`,
+        parentSpanId: parentSpan.span.spanId,
+        attributes: {
+          "analyst.turn": turn.turn,
+          "analyst.is_error": turn.isError,
+          "analyst.code_length": turn.code.length,
+          "analyst.output_length": turn.output.length,
+          "eval.phase": "analyst"
+        }
+      });
+      turnSpanIds.push(turnSpan.span.spanId);
+      if (turn.isError) {
+        await turnSpan.fail("Turn produced an error");
+      } else {
+        await turnSpan.end();
+      }
+      if (originalOnTurn) await originalOnTurn(turn);
+    }
+  };
+  try {
+    const result = await analyzeTraces(input, wrappedOptions);
+    await parentSpan.end({
+      attributes: {
+        "analyst.question_length": input.question.length,
+        "analyst.turn_count": result.turnCount,
+        "analyst.finding_count": result.findings.length,
+        "analyst.answer_length": result.answer.length,
+        "eval.phase": "analyst"
+      }
+    });
+    return result;
+  } catch (err) {
+    await parentSpan.fail(err instanceof Error ? err : String(err));
+    throw err;
+  }
+}
+
+// src/traced-mutator.ts
+function traceMutator(adapter, opts) {
+  return {
+    async mutate(args) {
+      const span = await opts.emitter.span({
+        kind: "llm",
+        name: `mutator:gen-${args.generation}`,
+        parentSpanId: opts.parentSpanId,
+        attributes: {
+          "mutator.parent_id": args.parent.id,
+          "mutator.generation": args.generation,
+          "mutator.child_count": args.childCount,
+          "mutator.top_trials": args.topTrials.length,
+          "mutator.bottom_trials": args.bottomTrials.length,
+          "mutator.parent_score": args.parentAggregate.meanScore,
+          "eval.phase": "mutator"
+        }
+      });
+      try {
+        const children = await adapter.mutate(args);
+        await span.end({
+          attributes: {
+            "mutator.parent_id": args.parent.id,
+            "mutator.generation": args.generation,
+            "mutator.child_count": args.childCount,
+            "mutator.top_trials": args.topTrials.length,
+            "mutator.bottom_trials": args.bottomTrials.length,
+            "mutator.parent_score": args.parentAggregate.meanScore,
+            "mutator.produced_count": children.length,
+            "mutator.child_ids": children.map((c) => c.id).join(","),
+            "eval.phase": "mutator"
+          }
+        });
+        return children;
+      } catch (err) {
+        await span.fail(err instanceof Error ? err : String(err));
+        throw err;
+      }
+    }
+  };
+}
 export {
   AGENT_PROFILE_KINDS,
   ANALYST_SEVERITIES,
@@ -11229,6 +11423,8 @@ export {
   createJudgeAdapter,
   createLlmCorrectnessChecker,
   createLlmReviewer,
+  createOtelExporter,
+  createOtelTracingStore,
   createReplayFetch,
   createRunCriticAdapter,
   createSandboxCodeMutator,
@@ -11312,6 +11508,7 @@ export {
   iqr,
   isJudgeSpan,
   isLlmSpan,
+  isOtelConfigured,
   isRetrievalSpan,
   isRunRecord,
   isSandboxSpan,
@@ -11345,6 +11542,7 @@ export {
   normalizeScores,
   notBlocked,
   objectiveEval,
+  otelRunCompleteHook,
   pairedBootstrap,
   pairedEvalueSequence,
   pairedMde,
@@ -11467,6 +11665,10 @@ export {
   toolSpans,
   traceAnalystFunctionGroup,
   traceAnalystOnRunComplete,
+  traceJudge,
+  traceJudgeEnsemble,
+  traceMutator,
+  tracedAnalyzeTraces,
   trialTraceFromMultiShotTrial,
   typoMutator,
   urlContains,
@@ -11487,6 +11689,7 @@ export {
   wilcoxonSignedRank,
   withAssignedFeedbackSplit,
   withJudgeRetry,
+  withOtelPipeline,
   wranglerDeployRunner
 };
 //# sourceMappingURL=index.js.map