@tangle-network/agent-eval 0.23.1 → 0.25.0

package/dist/chunk-JLZQWFV3.js

@@ -0,0 +1,618 @@
+import {
+  cohensD
+} from "./chunk-I4MBDTY5.js";
+import {
+  argHash,
+  groupBy,
+  toolSpans
+} from "./chunk-47X6LRCE.js";
+
+// src/failure-taxonomy.ts
+var DEFAULT_RULES = [
+  // Outcome already named? Respect it.
+  {
+    id: "explicit-outcome",
+    match: ({ run }) => {
+      const fc = run.outcome?.failureClass;
+      if (fc && fc !== "unknown")
+        return { failureClass: fc, reason: "outcome.failureClass set explicitly" };
+      return null;
+    }
+  },
+  {
+    id: "knowledge-readiness-blocked",
+    match: ({ events }) => {
+      const event = events.find(
+        (e) => e.kind === "custom" && e.payload.kind === "readiness_scored" && e.payload.passed === false
+      );
+      return event ? {
+        failureClass: "knowledge_readiness_blocked",
+        reason: "knowledge readiness report blocked execution",
+        triggerEventId: event.eventId
+      } : null;
+    }
+  },
+  {
+    id: "bad-integration-manifest",
+    match: ({ events }) => {
+      const event = events.find(
+        (e) => e.kind === "custom" && (e.payload.kind === "integration_manifest_validated" && e.payload.valid === false || e.payload.kind === "integration_invoke_failed" && e.payload.code === "manifest_invalid")
+      );
+      return event ? {
+        failureClass: "bad_integration_manifest",
+        reason: "integration manifest validation failed before launch",
+        triggerEventId: event.eventId
+      } : null;
+    }
+  },
+  {
+    id: "missing-integration-connection",
+    match: ({ events }) => {
+      const event = events.find(
+        (e) => e.kind === "custom" && e.payload.kind === "integration_manifest_resolved" && hasResolutionStatus(e.payload, "missing_connection")
+      );
+      return event ? {
+        failureClass: "missing_integration_connection",
+        reason: "required integration connection was missing",
+        triggerEventId: event.eventId
+      } : null;
+    }
+  },
+  {
+    id: "missing-integration-scope",
+    match: ({ events }) => {
+      const event = events.find(
+        (e) => e.kind === "custom" && (e.payload.kind === "integration_manifest_resolved" && hasMissingScopes(e.payload) || e.payload.kind === "integration_invoke_failed" && e.payload.code === "scope_denied")
+      );
+      return event ? {
+        failureClass: "missing_integration_scope",
+        reason: "integration grant or connection lacks required scopes",
+        triggerEventId: event.eventId
+      } : null;
+    }
+  },
+  {
+    id: "integration-approval-required",
+    match: ({ events }) => {
+      const event = events.find(
+        (e) => e.kind === "custom" && (e.payload.kind === "integration_invoke" && e.payload.status === "approval_required" || e.payload.kind === "integration_invoke_failed" && e.payload.code === "approval_required" || e.payload.kind === "integration_approval_required")
+      );
+      return event ? {
+        failureClass: "integration_approval_required",
+        reason: "integration write paused for user approval",
+        triggerEventId: event.eventId
+      } : null;
+    }
+  },
+  {
+    id: "integration-auth-expired",
+    match: ({ events }) => {
+      const event = events.find(
+        (e) => e.kind === "custom" && e.payload.kind === "integration_invoke_failed" && (e.payload.code === "auth_expired" || e.payload.code === "connection_not_active" || e.payload.code === "capability_expired" || e.payload.status === "expired")
+      );
+      return event ? {
+        failureClass: "integration_auth_expired",
+        reason: "integration connection or capability expired",
+        triggerEventId: event.eventId
+      } : null;
+    }
+  },
+  {
+    id: "unsafe-integration-write-denied",
+    match: ({ events }) => {
+      const event = events.find(
+        (e) => e.kind === "custom" && e.payload.kind === "integration_invoke_failed" && (e.payload.code === "unsafe_write_denied" || e.payload.code === "policy_denied" || e.payload.code === "action_denied")
+      );
+      return event ? {
+        failureClass: "unsafe_integration_write_denied",
+        reason: "integration write was denied by policy or capability scope",
+        triggerEventId: event.eventId
+      } : null;
+    }
+  },
+  {
+    id: "integration-provider-failure",
+    match: ({ events }) => {
+      const event = events.find(
+        (e) => e.kind === "custom" && e.payload.kind === "integration_invoke_failed" && ![
+          "scope_denied",
+          "approval_required",
+          "auth_expired",
+          "connection_not_active",
+          "capability_expired",
+          "unsafe_write_denied",
+          "policy_denied",
+          "action_denied",
+          "manifest_invalid"
+        ].includes(String(e.payload.code))
+      );
+      return event ? {
+        failureClass: "integration_provider_failure",
+        reason: "integration provider invocation failed",
+        triggerEventId: event.eventId
+      } : null;
+    }
+  },
+  {
+    id: "missing-credentials",
+    match: ({ events }) => {
+      const event = events.find(
+        (e) => e.kind === "custom" && e.payload.kind === "knowledge_gap" && e.payload.category === "credential_or_secret"
+      );
+      return event ? {
+        failureClass: "missing_credentials",
+        reason: "required credential or secret was missing",
+        triggerEventId: event.eventId
+      } : null;
+    }
+  },
+  {
+    id: "bad-retrieval",
+    match: ({ run, spans }) => {
+      if (run.outcome?.pass !== false) return null;
+      const retrieval = spans.find(
+        (s) => s.kind === "retrieval" && (s.hits.length === 0 || s.hits.every((hit) => hit.score <= 0))
+      );
+      return retrieval ? {
+        failureClass: "bad_retrieval",
+        reason: "retrieval returned no useful hits for a failed run",
+        triggerSpanId: retrieval.spanId
+      } : null;
+    }
+  },
+  {
+    id: "insufficient-evidence",
+    match: ({ events }) => {
+      const event = events.find(
+        (e) => e.kind === "custom" && e.payload.kind === "knowledge_gap" && e.payload.reason === "insufficient_evidence"
+      );
+      return event ? {
+        failureClass: "insufficient_evidence",
+        reason: "task proceeded with insufficient supporting evidence",
+        triggerEventId: event.eventId
+      } : null;
+    }
+  },
+  {
+    id: "contradictory-evidence",
+    match: ({ events }) => {
+      const event = events.find(
+        (e) => e.kind === "custom" && e.payload.kind === "knowledge_gap" && e.payload.reason === "contradictory_evidence"
+      );
+      return event ? {
+        failureClass: "contradictory_evidence",
+        reason: "supporting evidence contradicted itself",
+        triggerEventId: event.eventId
+      } : null;
+    }
+  },
+  // Budget breach events
+  {
+    id: "budget-breach",
+    match: ({ events }) => {
+      const breach = events.find((e) => e.kind === "budget_breach");
+      return breach ? {
+        failureClass: "budget_exceeded",
+        reason: `budget breached on ${breach.payload.dimension ?? "unknown dimension"}`,
+        triggerEventId: breach.eventId
+      } : null;
+    }
+  },
+  // Policy violations
+  {
+    id: "policy-violation",
+    match: ({ events }) => {
+      const e = events.find((x) => x.kind === "policy_violation");
+      return e ? {
+        failureClass: "policy_violation",
+        reason: "policy_violation event emitted",
+        triggerEventId: e.eventId
+      } : null;
+    }
+  },
+  // Sandbox non-zero exit code
+  {
+    id: "sandbox-failure",
+    match: ({ spans }) => {
+      const s = spans.find(
+        (x) => x.kind === "sandbox" && typeof x.exitCode === "number" && x.exitCode !== 0
+      );
+      if (!s) return null;
+      return {
+        failureClass: "sandbox_failure",
+        reason: `sandbox exited ${s.exitCode}`,
+        triggerSpanId: s.spanId
+      };
+    }
+  },
+  // Timeout: run aborted by external signal
+  {
+    id: "timeout",
+    match: ({ run, events }) => {
+      if (run.status !== "aborted") return null;
+      const hasTimeout = events.some(
+        (e) => e.kind === "error" && String(e.payload.reason ?? "").toLowerCase().includes("timeout")
+      );
+      const note = (run.outcome?.notes ?? "").toLowerCase();
+      if (hasTimeout || note.includes("timeout") || note.includes("deadline")) {
+        return { failureClass: "timeout", reason: "timeout signal observed" };
+      }
+      return null;
+    }
+  },
+  // Tool recovery failure: many consecutive tool errors on the same tool
+  {
+    id: "tool-recovery-failure",
+    match: ({ spans }) => {
+      const tools = spans.filter((s) => s.kind === "tool");
+      const byTool = /* @__PURE__ */ new Map();
+      for (const t of tools) {
+        const name = t.toolName;
+        const arr = byTool.get(name) ?? [];
+        arr.push(t);
+        byTool.set(name, arr);
+      }
+      for (const [name, arr] of byTool) {
+        const errs = arr.filter((s) => s.status === "error");
+        if (errs.length >= 3 && errs.length === arr.length) {
+          return {
+            failureClass: "tool_recovery_failure",
+            reason: `${errs.length} consecutive errors on tool "${name}"`,
+            triggerSpanId: errs[errs.length - 1].spanId
+          };
+        }
+      }
+      return null;
+    }
+  },
+  // Tool selection error: the run failed and agent called zero tools despite having them
+  {
+    id: "tool-selection-error",
+    match: ({ run, spans }) => {
+      if (run.outcome?.pass !== false) return null;
+      const hasToolsAvailable = spans.some(
+        (s) => s.kind === "agent" && s.attributes?.toolsAvailable !== void 0 && s.attributes?.toolsAvailable > 0
+      );
+      const tools = spans.filter((s) => s.kind === "tool");
+      if (hasToolsAvailable && tools.length === 0) {
+        return {
+          failureClass: "tool_selection_error",
+          reason: "tools were available but none were called"
+        };
+      }
+      return null;
+    }
+  },
+  // Format drift: scored by a judge with dimension='format' below threshold
+  {
+    id: "format-drift",
+    match: ({ spans }) => {
+      const judge = spans.find(
+        (s) => s.kind === "judge" && s.dimension === "format" && s.score < 0.5
+      );
+      return judge ? {
+        failureClass: "format_drift",
+        reason: "format judge scored below 0.5",
+        triggerSpanId: judge.spanId
+      } : null;
+    }
+  }
+];
+function hasResolutionStatus(payload, status) {
+  if (status === "missing_connection" && stringArray(payload.missingConnections).length > 0)
+    return true;
+  return resolutionItems(payload).some((item) => item.status === status);
+}
+function hasMissingScopes(payload) {
+  if (stringArray(payload.missingScopes).length > 0) return true;
+  return resolutionItems(payload).some(
+    (item) => Array.isArray(item.missingScopes) && item.missingScopes.length > 0
+  );
+}
+function resolutionItems(payload) {
+  return [
+    ...records(payload.missing),
+    ...records(payload.optionalMissing),
+    ...records(payload.ready)
+  ];
+}
+function records(value) {
+  if (!Array.isArray(value)) return [];
+  return value.filter(
+    (item) => Boolean(item) && typeof item === "object" && !Array.isArray(item)
+  );
+}
+function stringArray(value) {
+  return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
+}
+function classifyFailure(ctx, rules = DEFAULT_RULES) {
+  if (ctx.run.outcome?.pass !== false && ctx.run.status === "completed") {
+    return { failureClass: "success", reason: "run completed with pass=true (or no explicit fail)" };
+  }
+  for (const rule of rules) {
+    const hit = rule.match(ctx);
+    if (hit) return hit;
+  }
+  return { failureClass: "unknown", reason: "no rule matched; run failed for unclassified reason" };
+}
+
+// src/pipelines/failure-cluster.ts
+async function failureClusterView(store, options = {}) {
+  const rules = options.rules ?? DEFAULT_RULES;
+  const minSize = options.minClusterSize ?? 1;
+  const runs = await store.listRuns();
+  const clusters = /* @__PURE__ */ new Map();
+  let totalFailures = 0;
+  for (const run of runs) {
+    if (run.status === "completed" && run.outcome?.pass !== false) continue;
+    totalFailures++;
+    const spans = await store.spans({ runId: run.runId });
+    const events = await store.events({ runId: run.runId });
+    const cls = classifyFailure({ run, spans, events }, rules);
+    let toolName;
+    let argPrefix;
+    let dimension;
+    if (cls.triggerSpanId) {
+      const trig = spans.find((s) => s.spanId === cls.triggerSpanId);
+      if (trig?.kind === "tool") {
+        toolName = trig.toolName;
+        argPrefix = argHash(trig.args).slice(0, 16);
+      } else if (trig?.kind === "judge") {
+        dimension = trig.dimension;
+      }
+    }
+    if (!toolName) {
+      const ts = await toolSpans(store, run.runId);
+      const errored = ts.filter((t) => t.status === "error").pop();
+      if (errored) {
+        toolName = errored.toolName;
+        argPrefix = argHash(errored.args).slice(0, 16);
+      }
+    }
+    if (!dimension) {
+      const judge = spans.find((s) => s.kind === "judge" && typeof s.dimension === "string");
+      if (judge?.kind === "judge") dimension = judge.dimension;
+    }
+    const key = `${cls.failureClass}|${toolName ?? ""}|${argPrefix ?? ""}|${dimension ?? ""}`;
+    let cluster = clusters.get(key);
+    if (!cluster) {
+      cluster = {
+        failureClass: cls.failureClass,
+        toolName,
+        argPrefix,
+        dimension,
+        runCount: 0,
+        scenarioIds: [],
+        exampleRunId: run.runId,
+        exampleError: firstErrorMessage(spans) ?? cls.reason
+      };
+      clusters.set(key, cluster);
+    }
+    cluster.runCount++;
+    if (!cluster.scenarioIds.includes(run.scenarioId)) cluster.scenarioIds.push(run.scenarioId);
+  }
+  const arr = [...clusters.values()].filter((c) => c.runCount >= minSize).sort((a, b) => b.runCount - a.runCount);
+  return { clusters: arr, totalFailures, totalRuns: runs.length };
+}
+function firstErrorMessage(spans) {
+  const errored = spans.find((s) => s.status === "error");
+  return errored?.error;
+}
+
+// src/tool-use-metrics.ts
+async function computeToolUseMetrics(store, runId, options = {}) {
+  const tools = await toolSpans(store, runId);
+  if (tools.length === 0) {
+    return { runId, totalCalls: 0, byTool: {}, errorRate: 0, duplicateRate: 0, retryRate: 0 };
+  }
+  const byTool = {};
+  let totalErrors = 0;
+  let totalDuplicates = 0;
+  const sortedTools = [...tools].sort((a, b) => a.startedAt - b.startedAt);
+  const seenSignatures = /* @__PURE__ */ new Set();
+  for (const t of sortedTools) {
+    const stat = byTool[t.toolName] ??= { calls: 0, errors: 0, avgLatencyMs: 0, duplicates: 0 };
+    stat.calls += 1;
+    if (t.status === "error") {
+      stat.errors += 1;
+      totalErrors += 1;
+    }
+    if (typeof t.latencyMs === "number") stat.avgLatencyMs += t.latencyMs;
+    const sig = `${t.toolName}|${argHash(t.args)}`;
+    if (seenSignatures.has(sig)) {
+      stat.duplicates += 1;
+      totalDuplicates += 1;
+    }
+    seenSignatures.add(sig);
+  }
+  for (const stat of Object.values(byTool)) {
+    stat.avgLatencyMs = stat.calls > 0 ? stat.avgLatencyMs / stat.calls : 0;
+  }
+  let retryOpportunities = 0;
+  let retriesFollowed = 0;
+  for (const [, arr] of groupBy(sortedTools, (t) => t.toolName)) {
+    for (let i = 0; i < arr.length; i++) {
+      if (arr[i].status !== "error") continue;
+      retryOpportunities += 1;
+      if (arr[i + 1]) retriesFollowed += 1;
+    }
+  }
+  const retryRate = retryOpportunities > 0 ? retriesFollowed / retryOpportunities : 0;
+  let selectionAccuracy;
+  if (options.selectionLabels) {
+    const labeled = sortedTools.filter((t) => t.spanId in options.selectionLabels);
+    if (labeled.length > 0) {
+      selectionAccuracy = labeled.filter((t) => options.selectionLabels[t.spanId]).length / labeled.length;
+    }
+  }
+  return {
+    runId,
+    totalCalls: sortedTools.length,
+    byTool,
+    errorRate: totalErrors / sortedTools.length,
+    duplicateRate: totalDuplicates / sortedTools.length,
+    retryRate,
+    selectionAccuracy
+  };
+}
+
+// src/baseline.ts
+function compareToBaseline(samples, options = {}) {
+  const effectThreshold = options.effectThreshold ?? 0.5;
+  const alpha = options.alpha ?? 0.05;
+  const cvThreshold = options.unstableCvThreshold ?? 0.3;
+  const metrics = samples.map((s) => {
+    if (s.baseline.length < 2 || s.candidate.length < 2) {
+      throw new Error(`compareToBaseline: need \u22652 samples per side for "${s.metric}"`);
+    }
+    const bMean = mean(s.baseline);
+    const cMean = mean(s.candidate);
+    const delta = cMean - bMean;
+    const d = cohensD(s.baseline, s.candidate);
+    const { t, df, p } = welchsTTest(s.baseline, s.candidate);
+    const baselineIqr = iqr(s.baseline);
+    const candidateIqr = iqr(s.candidate);
+    const baselineStable = baselineIqr / Math.max(Math.abs(bMean), 1e-9) <= cvThreshold;
+    const candidateStable = candidateIqr / Math.max(Math.abs(cMean), 1e-9) <= cvThreshold;
+    const stable = baselineStable && candidateStable;
+    const reportedIqr = Math.max(baselineIqr, candidateIqr);
+    let verdict;
+    if (!stable) {
+      verdict = "unstable";
+    } else if (p < alpha && Math.abs(d) >= effectThreshold) {
+      const candidateIsBetter = s.higherIsBetter ? delta > 0 : delta < 0;
+      verdict = candidateIsBetter ? "improved" : "regressed";
+    } else {
+      verdict = "stable";
+    }
+    return {
+      metric: s.metric,
+      baselineMean: bMean,
+      candidateMean: cMean,
+      delta,
+      cohensD: d,
+      welchT: t,
+      welchDf: df,
+      welchP: p,
+      stable,
+      iqr: reportedIqr,
+      verdict
+    };
+  });
+  return {
+    metrics,
+    hasRegression: metrics.some((m) => m.verdict === "regressed"),
+    hasUnstable: metrics.some((m) => m.verdict === "unstable")
+  };
+}
+function mean(xs) {
+  return xs.reduce((a, b) => a + b, 0) / xs.length;
+}
+function iqr(xs) {
+  if (xs.length === 0) return 0;
+  const sorted = [...xs].sort((a, b) => a - b);
+  const q = (p) => {
+    const idx = p * (sorted.length - 1);
+    const lo = Math.floor(idx);
+    const hi = Math.ceil(idx);
+    return sorted[lo] + (sorted[hi] - sorted[lo]) * (idx - lo);
+  };
+  return q(0.75) - q(0.25);
+}
+function welchsTTest(a, b) {
+  if (a.length < 2 || b.length < 2) return { t: 0, df: 0, p: 1 };
+  const mA = mean(a);
+  const mB = mean(b);
+  const vA = variance(a, mA);
+  const vB = variance(b, mB);
+  const seSquared = vA / a.length + vB / b.length;
+  if (seSquared === 0) return { t: mA === mB ? 0 : Infinity, df: 0, p: mA === mB ? 1 : 0 };
+  const t = (mB - mA) / Math.sqrt(seSquared);
+  const df = seSquared * seSquared / ((vA / a.length) ** 2 / (a.length - 1) + (vB / b.length) ** 2 / (b.length - 1));
+  const p = 2 * (1 - studentTCdf(Math.abs(t), df));
+  return { t, df, p };
+}
+function variance(xs, m) {
+  return xs.reduce((acc, x) => acc + (x - m) ** 2, 0) / (xs.length - 1);
+}
+function studentTCdf(t, df) {
+  if (df <= 0) return 0.5;
+  if (df > 100) return normalCdf(t);
+  const x = df / (df + t * t);
+  const ib = incompleteBeta(x, df / 2, 0.5);
+  return t >= 0 ? 1 - 0.5 * ib : 0.5 * ib;
+}
+function incompleteBeta(x, a, b) {
+  if (x <= 0) return 0;
+  if (x >= 1) return 1;
+  const lnBeta = lnGamma(a) + lnGamma(b) - lnGamma(a + b);
+  const front = Math.exp(Math.log(x) * a + Math.log(1 - x) * b - lnBeta) / a;
+  let c = 1;
+  let d = 1 - (a + b) * x / (a + 1);
+  if (Math.abs(d) < 1e-30) d = 1e-30;
+  d = 1 / d;
+  let f = d;
+  for (let m = 1; m <= 200; m++) {
+    const m2 = 2 * m;
+    let num = m * (b - m) * x / ((a + m2 - 1) * (a + m2));
+    d = 1 + num * d;
+    if (Math.abs(d) < 1e-30) d = 1e-30;
+    c = 1 + num / c;
+    if (Math.abs(c) < 1e-30) c = 1e-30;
+    d = 1 / d;
+    f *= d * c;
+    num = -((a + m) * (a + b + m) * x) / ((a + m2) * (a + m2 + 1));
+    d = 1 + num * d;
+    if (Math.abs(d) < 1e-30) d = 1e-30;
+    c = 1 + num / c;
+    if (Math.abs(c) < 1e-30) c = 1e-30;
+    d = 1 / d;
+    const delta = d * c;
+    f *= delta;
+    if (Math.abs(delta - 1) < 3e-7) break;
+  }
+  return front * f;
+}
+function lnGamma(z) {
+  const coefs = [
+    0.9999999999998099,
+    676.5203681218851,
+    -1259.1392167224028,
+    771.3234287776531,
+    -176.6150291621406,
+    12.507343278686905,
+    -0.13857109526572012,
+    9984369578019572e-21,
+    15056327351493116e-23
+  ];
+  if (z < 0.5) return Math.log(Math.PI / Math.sin(Math.PI * z)) - lnGamma(1 - z);
+  z -= 1;
+  let x = coefs[0];
+  for (let i = 1; i < 9; i++) x += coefs[i] / (z + i);
+  const t = z + 7.5;
+  return 0.5 * Math.log(2 * Math.PI) + (z + 0.5) * Math.log(t) - t + Math.log(x);
+}
+function normalCdf(x) {
+  const a1 = 0.254829592;
+  const a2 = -0.284496736;
+  const a3 = 1.421413741;
+  const a4 = -1.453152027;
+  const a5 = 1.061405429;
+  const p = 0.3275911;
+  const sign = x < 0 ? -1 : 1;
+  const absX = Math.abs(x);
+  const t = 1 / (1 + p * absX);
+  const y = 1 - ((((a5 * t + a4) * t + a3) * t + a2) * t + a1) * t * Math.exp(-absX * absX / 2);
+  return 0.5 * (1 + sign * y);
+}
+
+export {
+  DEFAULT_RULES,
+  classifyFailure,
+  failureClusterView,
+  computeToolUseMetrics,
+  compareToBaseline,
+  iqr,
+  welchsTTest
+};
+//# sourceMappingURL=chunk-JLZQWFV3.js.map