@tangle-network/agent-eval 0.117.1 → 0.118.0

package/dist/contract/index.js

@@ -6,13 +6,14 @@ import {
   fromPiSession,
   fromPigraphSession,
   parseCodeAgentJsonl
-} from "../chunk-E4BUPP7Z.js";
+} from "../chunk-HKUCJ437.js";
 import {
   createHostedClient
 } from "../chunk-ZZUXHH3R.js";
 import {
-  analyzeRuns
-} from "../chunk-FQNLDL4D.js";
+  analyzeRuns,
+  summarizeExecution
+} from "../chunk-SVH2ANFD.js";
 import {
   REFERENCE_EQUIVALENCE_INPUT_LIMITS,
   REFERENCE_EQUIVALENCE_JUDGE_VERSION,
@@ -33,7 +34,7 @@ import {
   runReferenceEquivalenceJudge,
   surfaceContentHash,
   surfaceHash
-} from "../chunk-HQPHZGL6.js";
+} from "../chunk-YLMUS4MM.js";
 import {
   createRunCostLedger,
   fsCampaignStorage,
@@ -44,33 +45,38 @@ import {
 import {
   buildDefaultAnalystRegistry,
   createChatClient
-} from "../chunk-VF3XSYTI.js";
+} from "../chunk-33JA4TFA.js";
 import "../chunk-HHWE3POT.js";
-import "../chunk-MGEHEHSN.js";
+import "../chunk-OIIMMLRB.js";
 import {
   FileSystemOutcomeStore,
   InMemoryOutcomeStore
 } from "../chunk-3RF76KTD.js";
 import "../chunk-ARU2PZFM.js";
+import "../chunk-NJC7U437.js";
 import "../chunk-DPZAEKA6.js";
 import "../chunk-PJQFMIOX.js";
-import "../chunk-LQUTGLOZ.js";
+import "../chunk-VCTY3W6J.js";
+import "../chunk-VI2UW6B6.js";
+import {
+  recordAggregateMeasurements,
+  summarizeExecutionMeasurements
+} from "../chunk-KSDQVPLR.js";
+import "../chunk-PXE2VKMX.js";
+import "../chunk-ZET2UAYW.js";
 import "../chunk-GGE4NNQT.js";
 import {
-  LLM_INPUT_TOKEN_ATTR_KEYS,
-  LLM_MODEL_ATTR_KEYS,
-  LLM_OUTPUT_TOKEN_ATTR_KEYS
-} from "../chunk-S2F4J57L.js";
+  LLM_MODEL_ATTR_KEYS
+} from "../chunk-QKEGNI5B.js";
+import "../chunk-PC4UYEBM.js";
 import {
   parseRunRecordSafe
-} from "../chunk-5UF54T55.js";
+} from "../chunk-S3UZOQ5Y.js";
+import "../chunk-MA6HLL3S.js";
 import "../chunk-XJYR7XFV.js";
 import "../chunk-VSMTAMNK.js";
-import "../chunk-NJC7U437.js";
-import "../chunk-VCTY3W6J.js";
-import "../chunk-VI2UW6B6.js";
-import "../chunk-PC4UYEBM.js";
 import "../chunk-ONWEPEDO.js";
+import "../chunk-WDHBCA3M.js";
 import "../chunk-PZ5AY32C.js";
 
 // src/contract/self-improve.ts
@@ -917,9 +923,6 @@ function fromFeedbackTable(opts) {
 // src/contract/intake/otel-spans.ts
 var SCORE_KEYS = ["tangle.score", "eval.score", "score"];
 var MODEL_KEYS = ["tangle.model", ...LLM_MODEL_ATTR_KEYS, "model"];
-var COST_KEYS = ["tangle.cost.usd", "gen_ai.usage.cost_usd", "cost.usd", "cost"];
-var INPUT_TOKEN_KEYS = [...LLM_INPUT_TOKEN_ATTR_KEYS, "tangle.tokens.in", "tokens.in"];
-var OUTPUT_TOKEN_KEYS = [...LLM_OUTPUT_TOKEN_ATTR_KEYS, "tangle.tokens.out", "tokens.out"];
 var PROMPT_HASH_KEYS = ["tangle.prompt_hash", "prompt.hash"];
 var CONFIG_HASH_KEYS = ["tangle.config_hash", "config.hash"];
 function fromOtelSpans(opts) {
@@ -929,19 +932,31 @@ function fromOtelSpans(opts) {
   for (const [groupKey, groupSpans2] of grouped) {
     const root = findRoot(groupSpans2);
     if (!root) continue;
+    const measurements = summarizeExecutionMeasurements(
+      groupSpans2.map((span) => ({
+        id: span.spanId,
+        ...span.parentSpanId ? { parentId: span.parentSpanId } : {},
+        attributes: span.attributes,
+        modelCall: isExplicitModelCall(span),
+        aggregate: isExplicitAggregate(span)
+      }))
+    );
+    const callSpanIds = new Set(measurements.callSpanIds);
+    const callSpans = groupSpans2.filter((span) => callSpanIds.has(span.spanId));
     const wallMs = Math.max(0, (root.endTimeUnixNano - root.startTimeUnixNano) / 1e6);
-    const model = readAttrString(groupSpans2, MODEL_KEYS) ?? "unknown@unknown";
-    const costUsd = readAttrNumber(groupSpans2, COST_KEYS) ?? 0;
-    const inputTokens = readAttrNumber(groupSpans2, INPUT_TOKEN_KEYS) ?? 0;
-    const outputTokens = readAttrNumber(groupSpans2, OUTPUT_TOKEN_KEYS) ?? 0;
+    const model = readAttrString(callSpans, MODEL_KEYS) ?? readAttrString(groupSpans2, MODEL_KEYS) ?? "unknown@unknown";
+    const capturedCost = (measurements.cost.complete ? measurements.cost.value : void 0) ?? measurements.aggregate?.costUsd;
+    const costUsd = capturedCost ?? 0;
     const promptHash = readAttrString(groupSpans2, PROMPT_HASH_KEYS) ?? "sha256:unknown";
     const configHash = readAttrString(groupSpans2, CONFIG_HASH_KEYS) ?? "sha256:unknown";
     const score = readAttrNumber(groupSpans2, SCORE_KEYS);
     const rawNumeric = collectNumericAttrs(groupSpans2);
-    const tokenUsage = {
-      input: inputTokens,
-      output: outputTokens
-    };
+    rawNumeric.error_span_count = groupSpans2.filter((span) => span.status?.code === "ERROR").length;
+    rawNumeric.llm_span_count = measurements.modelCallCount;
+    if (measurements.cost.value !== void 0 && !measurements.cost.complete) {
+      rawNumeric.partial_observed_cost_usd = measurements.cost.value;
+    }
+    recordAggregateMeasurements(rawNumeric, measurements.aggregate);
     const judgeScores = score !== void 0 ? {
       perJudge: { "otel-derived": { score } },
       perDimMean: { score },
@@ -964,7 +979,8 @@ function fromOtelSpans(opts) {
       commitSha: root.attributes["tangle.commit_sha"] ?? "unknown",
       wallMs,
       costUsd,
-      tokenUsage,
+      costProvenance: capturedCost === void 0 ? { kind: "uncaptured", usd: null } : { kind: "observed", usd: capturedCost },
+      tokenUsage: measurements.tokenUsage,
       outcome,
       splitTag: defaultSplit,
       ...errorSpan ? { failureMode: errorSpan.name } : {}
@@ -972,6 +988,16 @@ function fromOtelSpans(opts) {
   }
   return runs;
 }
+function isExplicitModelCall(span) {
+  const kind = span.attributes["openinference.span.kind"];
+  if (typeof kind === "string") return kind.toUpperCase() === "LLM";
+  const spanType = span.attributes["span.type"];
+  return typeof spanType === "string" && spanType.toLowerCase() === "llm_request" || span.name.toLowerCase().includes("llm") || readAttrString([span], MODEL_KEYS) !== void 0 || typeof span.attributes["gen_ai.operation.name"] === "string";
+}
+function isExplicitAggregate(span) {
+  const kind = span.attributes["openinference.span.kind"];
+  return typeof kind === "string" && kind.toUpperCase() !== "LLM";
+}
 function groupSpans(spans) {
   const m = /* @__PURE__ */ new Map();
   for (const span of spans) {
@@ -1057,6 +1083,7 @@ export {
   runEval,
   runImprovementLoop,
   runReferenceEquivalenceJudge,
-  selfImprove
+  selfImprove,
+  summarizeExecution
 };
 //# sourceMappingURL=index.js.map