@tangle-network/agent-eval 0.120.0 → 0.120.2

package/dist/chunk-QBRSJK47.js

@@ -1,622 +0,0 @@
-import {
-  cohensD
-} from "./chunk-PJQFMIOX.js";
-import {
-  argHash,
-  groupBy,
-  hasCapturedToolArgs,
-  toolSpans
-} from "./chunk-ZET2UAYW.js";
-
-// src/failure-taxonomy.ts
-var DEFAULT_RULES = [
-  // Outcome already named? Respect it.
-  {
-    id: "explicit-outcome",
-    match: ({ run }) => {
-      const fc = run.outcome?.failureClass;
-      if (fc && fc !== "unknown")
-        return { failureClass: fc, reason: "outcome.failureClass set explicitly" };
-      return null;
-    }
-  },
-  {
-    id: "knowledge-readiness-blocked",
-    match: ({ events }) => {
-      const event = events.find(
-        (e) => e.kind === "custom" && e.payload.kind === "readiness_scored" && e.payload.passed === false
-      );
-      return event ? {
-        failureClass: "knowledge_readiness_blocked",
-        reason: "knowledge readiness report blocked execution",
-        triggerEventId: event.eventId
-      } : null;
-    }
-  },
-  {
-    id: "bad-integration-manifest",
-    match: ({ events }) => {
-      const event = events.find(
-        (e) => e.kind === "custom" && (e.payload.kind === "integration_manifest_validated" && e.payload.valid === false || e.payload.kind === "integration_invoke_failed" && e.payload.code === "manifest_invalid")
-      );
-      return event ? {
-        failureClass: "bad_integration_manifest",
-        reason: "integration manifest validation failed before launch",
-        triggerEventId: event.eventId
-      } : null;
-    }
-  },
-  {
-    id: "missing-integration-connection",
-    match: ({ events }) => {
-      const event = events.find(
-        (e) => e.kind === "custom" && e.payload.kind === "integration_manifest_resolved" && hasResolutionStatus(e.payload, "missing_connection")
-      );
-      return event ? {
-        failureClass: "missing_integration_connection",
-        reason: "required integration connection was missing",
-        triggerEventId: event.eventId
-      } : null;
-    }
-  },
-  {
-    id: "missing-integration-scope",
-    match: ({ events }) => {
-      const event = events.find(
-        (e) => e.kind === "custom" && (e.payload.kind === "integration_manifest_resolved" && hasMissingScopes(e.payload) || e.payload.kind === "integration_invoke_failed" && e.payload.code === "scope_denied")
-      );
-      return event ? {
-        failureClass: "missing_integration_scope",
-        reason: "integration grant or connection lacks required scopes",
-        triggerEventId: event.eventId
-      } : null;
-    }
-  },
-  {
-    id: "integration-approval-required",
-    match: ({ events }) => {
-      const event = events.find(
-        (e) => e.kind === "custom" && (e.payload.kind === "integration_invoke" && e.payload.status === "approval_required" || e.payload.kind === "integration_invoke_failed" && e.payload.code === "approval_required" || e.payload.kind === "integration_approval_required")
-      );
-      return event ? {
-        failureClass: "integration_approval_required",
-        reason: "integration write paused for user approval",
-        triggerEventId: event.eventId
-      } : null;
-    }
-  },
-  {
-    id: "integration-auth-expired",
-    match: ({ events }) => {
-      const event = events.find(
-        (e) => e.kind === "custom" && e.payload.kind === "integration_invoke_failed" && (e.payload.code === "auth_expired" || e.payload.code === "connection_not_active" || e.payload.code === "capability_expired" || e.payload.status === "expired")
-      );
-      return event ? {
-        failureClass: "integration_auth_expired",
-        reason: "integration connection or capability expired",
-        triggerEventId: event.eventId
-      } : null;
-    }
-  },
-  {
-    id: "unsafe-integration-write-denied",
-    match: ({ events }) => {
-      const event = events.find(
-        (e) => e.kind === "custom" && e.payload.kind === "integration_invoke_failed" && (e.payload.code === "unsafe_write_denied" || e.payload.code === "policy_denied" || e.payload.code === "action_denied")
-      );
-      return event ? {
-        failureClass: "unsafe_integration_write_denied",
-        reason: "integration write was denied by policy or capability scope",
-        triggerEventId: event.eventId
-      } : null;
-    }
-  },
-  {
-    id: "integration-provider-failure",
-    match: ({ events }) => {
-      const event = events.find(
-        (e) => e.kind === "custom" && e.payload.kind === "integration_invoke_failed" && ![
-          "scope_denied",
-          "approval_required",
-          "auth_expired",
-          "connection_not_active",
-          "capability_expired",
-          "unsafe_write_denied",
-          "policy_denied",
-          "action_denied",
-          "manifest_invalid"
-        ].includes(String(e.payload.code))
-      );
-      return event ? {
-        failureClass: "integration_provider_failure",
-        reason: "integration provider invocation failed",
-        triggerEventId: event.eventId
-      } : null;
-    }
-  },
-  {
-    id: "missing-credentials",
-    match: ({ events }) => {
-      const event = events.find(
-        (e) => e.kind === "custom" && e.payload.kind === "knowledge_gap" && e.payload.category === "credential_or_secret"
-      );
-      return event ? {
-        failureClass: "missing_credentials",
-        reason: "required credential or secret was missing",
-        triggerEventId: event.eventId
-      } : null;
-    }
-  },
-  {
-    id: "bad-retrieval",
-    match: ({ run, spans }) => {
-      if (run.outcome?.pass !== false) return null;
-      const retrieval = spans.find(
-        (s) => s.kind === "retrieval" && (s.hits.length === 0 || s.hits.every((hit) => hit.score <= 0))
-      );
-      return retrieval ? {
-        failureClass: "bad_retrieval",
-        reason: "retrieval returned no useful hits for a failed run",
-        triggerSpanId: retrieval.spanId
-      } : null;
-    }
-  },
-  {
-    id: "insufficient-evidence",
-    match: ({ events }) => {
-      const event = events.find(
-        (e) => e.kind === "custom" && e.payload.kind === "knowledge_gap" && e.payload.reason === "insufficient_evidence"
-      );
-      return event ? {
-        failureClass: "insufficient_evidence",
-        reason: "task proceeded with insufficient supporting evidence",
-        triggerEventId: event.eventId
-      } : null;
-    }
-  },
-  {
-    id: "contradictory-evidence",
-    match: ({ events }) => {
-      const event = events.find(
-        (e) => e.kind === "custom" && e.payload.kind === "knowledge_gap" && e.payload.reason === "contradictory_evidence"
-      );
-      return event ? {
-        failureClass: "contradictory_evidence",
-        reason: "supporting evidence contradicted itself",
-        triggerEventId: event.eventId
-      } : null;
-    }
-  },
-  // Budget breach events
-  {
-    id: "budget-breach",
-    match: ({ events }) => {
-      const breach = events.find((e) => e.kind === "budget_breach");
-      return breach ? {
-        failureClass: "budget_exceeded",
-        reason: `budget breached on ${breach.payload.dimension ?? "unknown dimension"}`,
-        triggerEventId: breach.eventId
-      } : null;
-    }
-  },
-  // Policy violations
-  {
-    id: "policy-violation",
-    match: ({ events }) => {
-      const e = events.find((x) => x.kind === "policy_violation");
-      return e ? {
-        failureClass: "policy_violation",
-        reason: "policy_violation event emitted",
-        triggerEventId: e.eventId
-      } : null;
-    }
-  },
-  // Sandbox non-zero exit code
-  {
-    id: "sandbox-failure",
-    match: ({ spans }) => {
-      const s = spans.find(
-        (x) => x.kind === "sandbox" && typeof x.exitCode === "number" && x.exitCode !== 0
-      );
-      if (!s) return null;
-      return {
-        failureClass: "sandbox_failure",
-        reason: `sandbox exited ${s.exitCode}`,
-        triggerSpanId: s.spanId
-      };
-    }
-  },
-  // Timeout: run aborted by external signal
-  {
-    id: "timeout",
-    match: ({ run, events }) => {
-      if (run.status !== "aborted") return null;
-      const hasTimeout = events.some(
-        (e) => e.kind === "error" && String(e.payload.reason ?? "").toLowerCase().includes("timeout")
-      );
-      const note = (run.outcome?.notes ?? "").toLowerCase();
-      if (hasTimeout || note.includes("timeout") || note.includes("deadline")) {
-        return { failureClass: "timeout", reason: "timeout signal observed" };
-      }
-      return null;
-    }
-  },
-  // Tool recovery failure: many consecutive tool errors on the same tool
-  {
-    id: "tool-recovery-failure",
-    match: ({ spans }) => {
-      const tools = spans.filter((s) => s.kind === "tool");
-      const byTool = /* @__PURE__ */ new Map();
-      for (const t of tools) {
-        const name = t.toolName;
-        const arr = byTool.get(name) ?? [];
-        arr.push(t);
-        byTool.set(name, arr);
-      }
-      for (const [name, arr] of byTool) {
-        const errs = arr.filter((s) => s.status === "error");
-        if (errs.length >= 3 && errs.length === arr.length) {
-          return {
-            failureClass: "tool_recovery_failure",
-            reason: `${errs.length} consecutive errors on tool "${name}"`,
-            triggerSpanId: errs[errs.length - 1].spanId
-          };
-        }
-      }
-      return null;
-    }
-  },
-  // Tool selection error: the run failed and agent called zero tools despite having them
-  {
-    id: "tool-selection-error",
-    match: ({ run, spans }) => {
-      if (run.outcome?.pass !== false) return null;
-      const hasToolsAvailable = spans.some(
-        (s) => s.kind === "agent" && s.attributes?.toolsAvailable !== void 0 && s.attributes?.toolsAvailable > 0
-      );
-      const tools = spans.filter((s) => s.kind === "tool");
-      if (hasToolsAvailable && tools.length === 0) {
-        return {
-          failureClass: "tool_selection_error",
-          reason: "tools were available but none were called"
-        };
-      }
-      return null;
-    }
-  },
-  // Format drift: scored by a judge with dimension='format' below threshold
-  {
-    id: "format-drift",
-    match: ({ spans }) => {
-      const judge = spans.find(
-        (s) => s.kind === "judge" && s.dimension === "format" && s.score < 0.5
-      );
-      return judge ? {
-        failureClass: "format_drift",
-        reason: "format judge scored below 0.5",
-        triggerSpanId: judge.spanId
-      } : null;
-    }
-  }
-];
-function hasResolutionStatus(payload, status) {
-  if (status === "missing_connection" && stringArray(payload.missingConnections).length > 0)
-    return true;
-  return resolutionItems(payload).some((item) => item.status === status);
-}
-function hasMissingScopes(payload) {
-  if (stringArray(payload.missingScopes).length > 0) return true;
-  return resolutionItems(payload).some(
-    (item) => Array.isArray(item.missingScopes) && item.missingScopes.length > 0
-  );
-}
-function resolutionItems(payload) {
-  return [
-    ...records(payload.missing),
-    ...records(payload.optionalMissing),
-    ...records(payload.ready)
-  ];
-}
-function records(value) {
-  if (!Array.isArray(value)) return [];
-  return value.filter(
-    (item) => Boolean(item) && typeof item === "object" && !Array.isArray(item)
-  );
-}
-function stringArray(value) {
-  return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
-}
-function classifyFailure(ctx, rules = DEFAULT_RULES) {
-  if (ctx.run.outcome?.pass !== false && ctx.run.status === "completed") {
-    return { failureClass: "success", reason: "run completed with pass=true (or no explicit fail)" };
-  }
-  for (const rule of rules) {
-    const hit = rule.match(ctx);
-    if (hit) return hit;
-  }
-  return { failureClass: "unknown", reason: "no rule matched; run failed for unclassified reason" };
-}
-
-// src/tool-use-metrics.ts
-async function computeToolUseMetrics(store, runId, options = {}) {
-  const tools = await toolSpans(store, runId);
-  if (tools.length === 0) {
-    return {
-      runId,
-      totalCalls: 0,
-      callsWithCapturedArgs: 0,
-      byTool: {},
-      errorRate: 0,
-      duplicateRate: 0,
-      retryRate: 0
-    };
-  }
-  const byTool = {};
-  let totalErrors = 0;
-  let totalDuplicates = 0;
-  let callsWithCapturedArgs = 0;
-  const sortedTools = [...tools].sort((a, b) => a.startedAt - b.startedAt);
-  const seenSignatures = /* @__PURE__ */ new Set();
-  for (const t of sortedTools) {
-    byTool[t.toolName] ??= {
-      calls: 0,
-      callsWithCapturedArgs: 0,
-      errors: 0,
-      avgLatencyMs: 0,
-      duplicates: 0
-    };
-    const stat = byTool[t.toolName];
-    stat.calls += 1;
-    if (t.status === "error") {
-      stat.errors += 1;
-      totalErrors += 1;
-    }
-    if (typeof t.latencyMs === "number") stat.avgLatencyMs += t.latencyMs;
-    if (hasCapturedToolArgs(t)) {
-      callsWithCapturedArgs += 1;
-      stat.callsWithCapturedArgs += 1;
-      const sig = `${t.toolName}|${argHash(t.args)}`;
-      if (seenSignatures.has(sig)) {
-        stat.duplicates += 1;
-        totalDuplicates += 1;
-      }
-      seenSignatures.add(sig);
-    }
-  }
-  for (const stat of Object.values(byTool)) {
-    stat.avgLatencyMs = stat.calls > 0 ? stat.avgLatencyMs / stat.calls : 0;
-  }
-  let retryOpportunities = 0;
-  let retriesFollowed = 0;
-  for (const [, arr] of groupBy(sortedTools, (t) => t.toolName)) {
-    for (let i = 0; i < arr.length; i++) {
-      if (arr[i].status !== "error") continue;
-      retryOpportunities += 1;
-      if (arr[i + 1]) retriesFollowed += 1;
-    }
-  }
-  const retryRate = retryOpportunities > 0 ? retriesFollowed / retryOpportunities : 0;
-  let selectionAccuracy;
-  if (options.selectionLabels) {
-    const labeled = sortedTools.filter((t) => t.spanId in options.selectionLabels);
-    if (labeled.length > 0) {
-      selectionAccuracy = labeled.filter((t) => options.selectionLabels[t.spanId]).length / labeled.length;
-    }
-  }
-  return {
-    runId,
-    totalCalls: sortedTools.length,
-    callsWithCapturedArgs,
-    byTool,
-    errorRate: totalErrors / sortedTools.length,
-    duplicateRate: callsWithCapturedArgs > 0 ? totalDuplicates / callsWithCapturedArgs : 0,
-    retryRate,
-    selectionAccuracy
-  };
-}
-
-// src/trajectory.ts
-async function buildTrajectory(store, runId) {
-  const spans = await store.spans({ runId });
-  const events = await store.events({ runId });
-  const childrenOf = /* @__PURE__ */ new Map();
-  for (const s of spans) {
-    const arr = childrenOf.get(s.parentSpanId) ?? [];
-    arr.push(s);
-    childrenOf.set(s.parentSpanId, arr);
-  }
-  for (const arr of childrenOf.values()) arr.sort((a, b) => a.startedAt - b.startedAt);
-  const eventsBySpan = /* @__PURE__ */ new Map();
-  for (const e of events) {
-    if (!e.spanId) continue;
-    const arr = eventsBySpan.get(e.spanId) ?? [];
-    arr.push(e);
-    eventsBySpan.set(e.spanId, arr);
-  }
-  const steps = [];
-  const walk = (spanId, depth) => {
-    const kids = childrenOf.get(spanId) ?? [];
-    for (const child of kids) {
-      steps.push({
-        index: steps.length,
-        span: child,
-        depth,
-        events: eventsBySpan.get(child.spanId) ?? []
-      });
-      walk(child.spanId, depth + 1);
-    }
-  };
-  walk(void 0, 0);
-  const llmTurns = steps.filter((s) => s.span.kind === "llm").length;
-  const toolCalls = steps.filter((s) => s.span.kind === "tool").length;
-  const judgeVerdicts = steps.filter((s) => s.span.kind === "judge").length;
-  const retrievals = steps.filter((s) => s.span.kind === "retrieval").length;
-  let totalDurationMs = 0;
-  if (steps.length > 0) {
-    const starts = spans.map((s) => s.startedAt);
-    const ends = spans.map((s) => s.endedAt ?? s.startedAt);
-    totalDurationMs = Math.max(...ends) - Math.min(...starts);
-  }
-  return { runId, steps, llmTurns, toolCalls, judgeVerdicts, retrievals, totalDurationMs };
-}
-
-// src/baseline.ts
-function compareToBaseline(samples, options = {}) {
-  const effectThreshold = options.effectThreshold ?? 0.5;
-  const alpha = options.alpha ?? 0.05;
-  const cvThreshold = options.unstableCvThreshold ?? 0.3;
-  const metrics = samples.map((s) => {
-    if (s.baseline.length < 2 || s.candidate.length < 2) {
-      throw new Error(`compareToBaseline: need \u22652 samples per side for "${s.metric}"`);
-    }
-    const bMean = mean(s.baseline);
-    const cMean = mean(s.candidate);
-    const delta = cMean - bMean;
-    const d = cohensD(s.baseline, s.candidate);
-    const { t, df, p } = welchsTTest(s.baseline, s.candidate);
-    const baselineIqr = iqr(s.baseline);
-    const candidateIqr = iqr(s.candidate);
-    const baselineStable = baselineIqr / Math.max(Math.abs(bMean), 1e-9) <= cvThreshold;
-    const candidateStable = candidateIqr / Math.max(Math.abs(cMean), 1e-9) <= cvThreshold;
-    const stable = baselineStable && candidateStable;
-    const reportedIqr = Math.max(baselineIqr, candidateIqr);
-    let verdict;
-    if (!stable) {
-      verdict = "unstable";
-    } else if (p < alpha && Math.abs(d) >= effectThreshold) {
-      const candidateIsBetter = s.higherIsBetter ? delta > 0 : delta < 0;
-      verdict = candidateIsBetter ? "improved" : "regressed";
-    } else {
-      verdict = "stable";
-    }
-    return {
-      metric: s.metric,
-      baselineMean: bMean,
-      candidateMean: cMean,
-      delta,
-      cohensD: d,
-      welchT: t,
-      welchDf: df,
-      welchP: p,
-      stable,
-      iqr: reportedIqr,
-      verdict
-    };
-  });
-  return {
-    metrics,
-    hasRegression: metrics.some((m) => m.verdict === "regressed"),
-    hasUnstable: metrics.some((m) => m.verdict === "unstable")
-  };
-}
-function mean(xs) {
-  return xs.reduce((a, b) => a + b, 0) / xs.length;
-}
-function iqr(xs) {
-  if (xs.length === 0) return 0;
-  const sorted = [...xs].sort((a, b) => a - b);
-  const q = (p) => {
-    const idx = p * (sorted.length - 1);
-    const lo = Math.floor(idx);
-    const hi = Math.ceil(idx);
-    return sorted[lo] + (sorted[hi] - sorted[lo]) * (idx - lo);
-  };
-  return q(0.75) - q(0.25);
-}
-function welchsTTest(a, b) {
-  if (a.length < 2 || b.length < 2) return { t: 0, df: 0, p: 1 };
-  const mA = mean(a);
-  const mB = mean(b);
-  const vA = variance(a, mA);
-  const vB = variance(b, mB);
-  const seSquared = vA / a.length + vB / b.length;
-  if (seSquared === 0) return { t: mA === mB ? 0 : Infinity, df: 0, p: mA === mB ? 1 : 0 };
-  const t = (mB - mA) / Math.sqrt(seSquared);
-  const df = seSquared * seSquared / ((vA / a.length) ** 2 / (a.length - 1) + (vB / b.length) ** 2 / (b.length - 1));
-  const p = 2 * (1 - studentTCdf(Math.abs(t), df));
-  return { t, df, p };
-}
-function variance(xs, m) {
-  return xs.reduce((acc, x) => acc + (x - m) ** 2, 0) / (xs.length - 1);
-}
-function studentTCdf(t, df) {
-  if (df <= 0) return 0.5;
-  if (df > 100) return normalCdf(t);
-  const x = df / (df + t * t);
-  const ib = incompleteBeta(x, df / 2, 0.5);
-  return t >= 0 ? 1 - 0.5 * ib : 0.5 * ib;
-}
-function incompleteBeta(x, a, b) {
-  if (x <= 0) return 0;
-  if (x >= 1) return 1;
-  const lnBeta = lnGamma(a) + lnGamma(b) - lnGamma(a + b);
-  const front = Math.exp(Math.log(x) * a + Math.log(1 - x) * b - lnBeta) / a;
-  let c = 1;
-  let d = 1 - (a + b) * x / (a + 1);
-  if (Math.abs(d) < 1e-30) d = 1e-30;
-  d = 1 / d;
-  let f = d;
-  for (let m = 1; m <= 200; m++) {
-    const m2 = 2 * m;
-    let num = m * (b - m) * x / ((a + m2 - 1) * (a + m2));
-    d = 1 + num * d;
-    if (Math.abs(d) < 1e-30) d = 1e-30;
-    c = 1 + num / c;
-    if (Math.abs(c) < 1e-30) c = 1e-30;
-    d = 1 / d;
-    f *= d * c;
-    num = -((a + m) * (a + b + m) * x) / ((a + m2) * (a + m2 + 1));
-    d = 1 + num * d;
-    if (Math.abs(d) < 1e-30) d = 1e-30;
-    c = 1 + num / c;
-    if (Math.abs(c) < 1e-30) c = 1e-30;
-    d = 1 / d;
-    const delta = d * c;
-    f *= delta;
-    if (Math.abs(delta - 1) < 3e-7) break;
-  }
-  return front * f;
-}
-function lnGamma(z) {
-  const coefs = [
-    0.9999999999998099,
-    676.5203681218851,
-    -1259.1392167224028,
-    771.3234287776531,
-    -176.6150291621406,
-    12.507343278686905,
-    -0.13857109526572012,
-    9984369578019572e-21,
-    15056327351493116e-23
-  ];
-  if (z < 0.5) return Math.log(Math.PI / Math.sin(Math.PI * z)) - lnGamma(1 - z);
-  z -= 1;
-  let x = coefs[0];
-  for (let i = 1; i < 9; i++) x += coefs[i] / (z + i);
-  const t = z + 7.5;
-  return 0.5 * Math.log(2 * Math.PI) + (z + 0.5) * Math.log(t) - t + Math.log(x);
-}
-function normalCdf(x) {
-  const a1 = 0.254829592;
-  const a2 = -0.284496736;
-  const a3 = 1.421413741;
-  const a4 = -1.453152027;
-  const a5 = 1.061405429;
-  const p = 0.3275911;
-  const sign = x < 0 ? -1 : 1;
-  const absX = Math.abs(x);
-  const t = 1 / (1 + p * absX);
-  const y = 1 - ((((a5 * t + a4) * t + a3) * t + a2) * t + a1) * t * Math.exp(-absX * absX / 2);
-  return 0.5 * (1 + sign * y);
-}
-
-export {
-  DEFAULT_RULES,
-  classifyFailure,
-  computeToolUseMetrics,
-  buildTrajectory,
-  compareToBaseline,
-  iqr,
-  welchsTTest
-};
-//# sourceMappingURL=chunk-QBRSJK47.js.map