@tangle-network/agent-eval 0.100.0 → 0.100.2

package/dist/index.js

@@ -51,7 +51,7 @@ import {
   llmJudge,
   parseCorrectnessResponse,
   verifyCompletion
-} from "./chunk-OYU4D7FY.js";
+} from "./chunk-VWQ6PO5O.js";
 import {
   DEFAULT_MUTATION_PRIMITIVES,
   DEFAULT_RED_TEAM_CORPUS,
@@ -75,7 +75,7 @@ import {
   scoreRedTeamOutput,
   surfaceContentHash,
   toolNamesForRun
-} from "./chunk-WMBLMTUE.js";
+} from "./chunk-2KTBHICD.js";
 import {
   BackendIntegrityError,
   assertRealBackend,
@@ -133,7 +133,7 @@ import {
   defaultIsMaterial,
   diffFindings,
   runSemanticConceptJudge
-} from "./chunk-SJISCGWD.js";
+} from "./chunk-JU6ZX3CX.js";
 import {
   LockedJsonlAppender,
   Mutex
@@ -141,12 +141,24 @@ import {
 import {
   buildDefaultAnalystRegistry,
   computeTraceMetrics
-} from "./chunk-3NHEO6ZC.js";
+} from "./chunk-L5TVEZFT.js";
 import {
   DEFAULT_RUN_SCORE_WEIGHTS,
+  POLICY_EDIT_AXES,
+  POLICY_EDIT_TARGET_SURFACES,
+  PolicyEditValidationError,
+  admitPolicyEdit,
   aggregateRunScore,
-  clamp01
-} from "./chunk-BUTW4RGG.js";
+  applyPolicyEditToSurface,
+  clamp01,
+  computePolicyEditId,
+  isPolicyEdit,
+  makePolicyEdit,
+  policyEditFromFinding,
+  policyEditsFromFindings,
+  scorePolicyEditReadiness,
+  validatePolicyEdit
+} from "./chunk-IN3SHQML.js";
 import {
   AnalystRegistry,
   DEFAULT_TRACE_ANALYST_KINDS,
@@ -156,7 +168,7 @@ import {
   KNOWLEDGE_POISONING_KIND_SPEC,
   createTraceAnalystKind,
   renderPriorFindings
-} from "./chunk-S4SYLDFX.js";
+} from "./chunk-FRI6RG3P.js";
 import {
   controlFailureClassFromVerification,
   controlRunToRunRecord,
@@ -167,7 +179,7 @@ import {
   runProposeReview,
   runProposeReviewAsControlLoop,
   scoreFromEvals
-} from "./chunk-NZEQVRH5.js";
+} from "./chunk-BOETF6BU.js";
 import {
   allCriticalPassed,
   errorStreakDetector,
@@ -189,7 +201,7 @@ import {
 } from "./chunk-VDGPPGE3.js";
 import {
   runEvalCampaign
-} from "./chunk-OKQ2LAT7.js";
+} from "./chunk-4LWD6GC7.js";
 import {
   LlmCallError,
   LlmClient,
@@ -292,13 +304,13 @@ import {
   scoreTraceInsightReadiness,
   tokenizeDomainWords,
   traceAnalystOnRunComplete
-} from "./chunk-77T4STFI.js";
+} from "./chunk-PMF5WIBX.js";
 import {
   TRACE_ANALYST_ACTOR_DESCRIPTION,
   TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
   TRACE_ANALYST_SUBAGENT_DESCRIPTION,
   analyzeTraces
-} from "./chunk-IZCEK2HR.js";
+} from "./chunk-2MLIEQSN.js";
 import {
   DEFAULT_REDACTION_RULES,
   REDACTION_VERSION,
@@ -370,29 +382,31 @@ import {
   defaultProviderRedactor,
   providerFromBaseUrl
 } from "./chunk-PC4UYEBM.js";
+import {
+  RunRecordValidationError,
+  isRunRecord,
+  parseRunRecordSafe,
+  roundTripRunRecord,
+  validateRunRecord
+} from "./chunk-G6S73VA7.js";
+import {
+  TraceEmitter,
+  llmSpanFromProvider
+} from "./chunk-TVVP3ZZQ.js";
 import {
   AGENT_PROFILE_KINDS,
   AgentProfileCellValidationError,
-  RunRecordValidationError,
   agentProfileCellHashMaterial,
   agentProfileCellKey,
   assertRunAgentProfileCell,
   buildAgentInterfaceProfileCell,
   buildAgentProfileCell,
   groupRunsByAgentProfileCell,
-  isRunRecord,
-  parseRunRecordSafe,
   requireAgentProfileCell,
-  roundTripRunRecord,
   toAgentProfileJson,
   validateAgentProfileCell,
-  validateRunRecord,
   verifyAgentProfileCell
-} from "./chunk-LO6IOIJ2.js";
-import {
-  TraceEmitter,
-  llmSpanFromProvider
-} from "./chunk-TVVP3ZZQ.js";
+} from "./chunk-ABOIVNXL.js";
 import {
   canonicalize,
   evaluateHypothesis,
@@ -4477,9 +4491,9 @@ function median(sorted) {
   const mid = Math.floor(n / 2);
   return n % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
 }
-function stddev(values, mean5) {
+function stddev(values, mean6) {
   if (values.length < 2) return 0;
-  const variance = values.reduce((acc, v) => acc + (v - mean5) ** 2, 0) / values.length;
+  const variance = values.reduce((acc, v) => acc + (v - mean6) ** 2, 0) / values.length;
   return Math.sqrt(variance);
 }
 function computeExperimentStats(reps, thresholds) {
@@ -4505,15 +4519,15 @@ function computeExperimentStats(reps, thresholds) {
     return r.score;
   });
   const sorted = [...scores2].sort((a, b) => a - b);
-  const mean5 = scores2.reduce((s, v) => s + v, 0) / n;
-  const sd = stddev(scores2, mean5);
+  const mean6 = scores2.reduce((s, v) => s + v, 0) / n;
+  const sd = stddev(scores2, mean6);
   const spread = iqr(scores2);
   const rated = reps.filter((r) => typeof r.passed === "boolean");
   const passRate = rated.length === 0 ? null : rated.filter((r) => r.passed).length / rated.length;
   const stable = spread < t.iqrUnstableAbove && sd < t.stddevUnstableAbove;
   return {
     median: median(sorted),
-    mean: mean5,
+    mean: mean6,
     min: sorted[0],
     max: sorted[n - 1],
     iqr: spread,
@@ -4683,6 +4697,49 @@ var ExperimentTracker = class {
   }
 };
 
+// src/leaderboard.ts
+function mean3(xs) {
+  return xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : null;
+}
+function leaderboard(records, opts) {
+  const keyOf = opts.groupBy ?? ((r) => r.agentProfile?.cellId ?? "ungrouped");
+  const groups = /* @__PURE__ */ new Map();
+  for (const r of records) {
+    const k = keyOf(r);
+    const g = groups.get(k);
+    if (g) g.push(r);
+    else groups.set(k, [r]);
+  }
+  const rows = [...groups.entries()].map(([key, rs]) => {
+    const passes = rs.filter((r) => opts.passed(r)).length;
+    const ci = wilson(passes, rs.length);
+    const profile = rs[0].agentProfile;
+    const harness = profile?.harness?.id;
+    const model = profile?.model;
+    return {
+      key,
+      label: harness && model ? `${harness} \xB7 ${model}` : key,
+      ...harness ? { harness } : {},
+      ...model ? { model } : {},
+      rank: 0,
+      n: rs.length,
+      passRate: ci.estimate,
+      passRateCi95: [ci.lower, ci.upper],
+      meanCostUsd: mean3(rs.map((r) => r.costUsd)),
+      meanTokensIn: mean3(rs.map((r) => r.tokenUsage.input)),
+      meanTokensOut: mean3(rs.map((r) => r.tokenUsage.output)),
+      meanWallMs: mean3(rs.map((r) => r.wallMs))
+    };
+  });
+  rows.sort(
+    (a, b) => b.passRate - a.passRate || (a.meanCostUsd ?? Infinity) - (b.meanCostUsd ?? Infinity)
+  );
+  rows.forEach((row, i) => {
+    row.rank = i + 1;
+  });
+  return rows;
+}
+
 // src/muffled-gate-scanner.ts
 import { existsSync, readdirSync, readFileSync, statSync } from "fs";
 import { join } from "path";
@@ -6228,12 +6285,12 @@ async function paraphraseRobustness(prompt, mutators, scoreFn, options = {}) {
     variantScores.push({ mutator: id, score, mutated });
     all.push(score);
   }
-  const mean5 = all.reduce((a, b) => a + b, 0) / all.length;
-  const variance = all.reduce((a, v) => a + (v - mean5) ** 2, 0) / all.length;
+  const mean6 = all.reduce((a, b) => a + b, 0) / all.length;
+  const variance = all.reduce((a, v) => a + (v - mean6) ** 2, 0) / all.length;
   const stdDev = Math.sqrt(variance);
-  const ref = Math.abs(mean5) > 1e-9 ? Math.abs(mean5) : 1;
+  const ref = Math.abs(mean6) > 1e-9 ? Math.abs(mean6) : 1;
   const robustness = Math.max(0, 1 - stdDev / ref);
-  return { originalScore, variantScores, meanScore: mean5, stdDev, robustness };
+  return { originalScore, variantScores, meanScore: mean6, stdDev, robustness };
 }
 var lowercaseMutator = (p) => p.toLowerCase();
 var sentenceReorderMutator = (p, seed) => {
@@ -6299,8 +6356,8 @@ async function paraphraseRobustnessScenarios(args) {
         });
         scores2.push(out.score);
       }
-      const mean5 = scores2.reduce((a, b) => a + b, 0) / scores2.length;
-      deltas[m.name] = mean5 - originalScore;
+      const mean6 = scores2.reduce((a, b) => a + b, 0) / scores2.length;
+      deltas[m.name] = mean6 - originalScore;
       paraphrasedAll.push(...scores2);
     }
     const paraphrasedMean = paraphrasedAll.length === 0 ? originalScore : paraphrasedAll.reduce((a, b) => a + b, 0) / paraphrasedAll.length;
@@ -6634,8 +6691,8 @@ async function proposeSynthesisTargets(dataset, traceStore, options = {}) {
     const sRuns = runs.filter((r) => r.scenarioId === s.id);
     const scores2 = sRuns.map((r) => r.outcome?.score).filter((x) => typeof x === "number");
     if (scores2.length < 3) continue;
-    const mean5 = scores2.reduce((a, b) => a + b, 0) / scores2.length;
-    const variance = scores2.reduce((a, b) => a + (b - mean5) ** 2, 0) / scores2.length;
+    const mean6 = scores2.reduce((a, b) => a + b, 0) / scores2.length;
+    const variance = scores2.reduce((a, b) => a + (b - mean6) ** 2, 0) / scores2.length;
     if (variance > varianceThreshold) {
       targets.push({
         reason: "high-variance",
@@ -8090,7 +8147,7 @@ function decideReferenceReplayPromotion(baseline, candidate, policy = {}) {
       regressions
     };
   }
-  const requiredMeanDelta = mean3(compared.map((item) => item.f1Delta));
+  const requiredMeanDelta = mean4(compared.map((item) => item.f1Delta));
   if (requiredMeanDelta < minF1Delta) {
     return {
       promote: false,
@@ -8326,7 +8383,7 @@ function clamp012(value) {
 function sum2(values) {
   return values.reduce((acc, value) => acc + value, 0);
 }
-function mean3(values) {
+function mean4(values) {
   return values.length ? sum2(values) / values.length : 0;
 }
 function formatPct(value) {
@@ -8780,10 +8837,10 @@ var HeldOutGate = class {
       afterHoldout.push(run.outcome.holdoutScore);
     }
     const productiveRuns = beforeHoldout.length;
-    const candidateSearchMean = mean4(scores(candidate, "searchScore", "search"));
-    const candidateHoldoutMean = mean4(scores(candidate, "holdoutScore", "holdout"));
-    const baselineSearchMean = mean4(scores(baseline, "searchScore", "search"));
-    const baselineHoldoutMean = mean4(scores(baseline, "holdoutScore", "holdout"));
+    const candidateSearchMean = mean5(scores(candidate, "searchScore", "search"));
+    const candidateHoldoutMean = mean5(scores(candidate, "holdoutScore", "holdout"));
+    const baselineSearchMean = mean5(scores(baseline, "searchScore", "search"));
+    const baselineHoldoutMean = mean5(scores(baseline, "holdoutScore", "holdout"));
     const overfitGap = safeDiff(candidateSearchMean, candidateHoldoutMean);
     const baselineOverfitGap = safeDiff(baselineSearchMean, baselineHoldoutMean);
     const medianCandidateCost = medianFinite(candidate.map((r) => r.costUsd));
@@ -8895,7 +8952,7 @@ function scores(runs, field, splitFilter) {
   }
   return out;
 }
-function mean4(xs) {
+function mean5(xs) {
   if (xs.length === 0) return Number.NaN;
   return xs.reduce((s, x) => s + x, 0) / xs.length;
 }
@@ -8935,9 +8992,9 @@ function passOrthogonality(input) {
       sims.push(cosineSimilarity(vectors[i], vectors[j]));
     }
   }
-  const mean5 = sims.length === 0 ? 0 : sims.reduce((a, b) => a + b, 0) / sims.length;
+  const mean6 = sims.length === 0 ? 0 : sims.reduce((a, b) => a + b, 0) / sims.length;
   return {
-    orthogonality: Math.max(0, Math.min(1, 1 - mean5)),
+    orthogonality: Math.max(0, Math.min(1, 1 - mean6)),
     passCount: passes.length,
     similarities: sims
   };
@@ -10126,7 +10183,10 @@ export {
   OPENINFERENCE_SPAN_KIND,
   OTEL_AGENT_EVAL_SCOPE,
   OtlpFileTraceStore,
+  POLICY_EDIT_AXES,
+  POLICY_EDIT_TARGET_SURFACES,
   PairwiseSteeringOptimizer,
+  PolicyEditValidationError,
   ProductClient,
   PromptRegistry,
   REDACTION_VERSION,
@@ -10165,6 +10225,7 @@ export {
   ValidationError,
   VerificationError,
   acquisitionPlansForKnowledgeGaps,
+  admitPolicyEdit,
   adversarialJudge,
   agentProfileCellHashMaterial,
   agentProfileCellKey,
@@ -10181,6 +10242,7 @@ export {
   analyzeSeries,
   analyzeTraces,
   appendScorecard,
+  applyPolicyEditToSurface,
   argHash,
   asNumber,
   asString,
@@ -10255,6 +10317,7 @@ export {
   composeValidators,
   computeExperimentStats,
   computeFindingId,
+  computePolicyEditId,
   computeToolUseMetrics,
   computeTraceMetrics,
   confidenceInterval,
@@ -10393,6 +10456,7 @@ export {
   isLlmSpan,
   isModelPriced,
   isOtelConfigured,
+  isPolicyEdit,
   isRetrievalSpan,
   isRunRecord,
   isSandboxSpan,
@@ -10410,6 +10474,7 @@ export {
   judgeSpans,
   keyPreserved,
   knowledgeReadinessTracePayload,
+  leaderboard,
   linterJudge,
   llmJudge,
   llmSpanFromProvider,
@@ -10421,6 +10486,7 @@ export {
   lowercaseMutator,
   makeEvalTools,
   makeFinding,
+  makePolicyEdit,
   mannWhitneyU,
   matchGoldens,
   matchSpan,
@@ -10465,6 +10531,8 @@ export {
   pearsonR,
   pixelDeltaRatio,
   planTraceInsightQuestions,
+  policyEditFromFinding,
+  policyEditsFromFindings,
   politenessPrefixMutator,
   positionalBias,
   preflightModels,
@@ -10540,6 +10608,7 @@ export {
   scoreContinuity,
   scoreFromEvals,
   scoreKnowledgeReadiness,
+  scorePolicyEditReadiness,
   scorePrReviewComments,
   scorePrReviewSource,
   scoreRedTeamOutput,
@@ -10589,6 +10658,7 @@ export {
   urlContains,
   userQuestionsForKnowledgeGaps,
   validateAgentProfileCell,
+  validatePolicyEdit,
   validateRunRecord,
   verbosityBias,
   verifyAgentProfileCell,