@tangle-network/agent-eval 0.99.0 → 0.100.0

package/dist/index.js

@@ -134,14 +134,14 @@ import {
   diffFindings,
   runSemanticConceptJudge
 } from "./chunk-SJISCGWD.js";
-import {
-  buildDefaultAnalystRegistry,
-  computeTraceMetrics
-} from "./chunk-3NHEO6ZC.js";
 import {
   LockedJsonlAppender,
   Mutex
 } from "./chunk-DJWX3GVS.js";
+import {
+  buildDefaultAnalystRegistry,
+  computeTraceMetrics
+} from "./chunk-3NHEO6ZC.js";
 import {
   DEFAULT_RUN_SCORE_WEIGHTS,
   aggregateRunScore,
@@ -2593,6 +2593,59 @@ function printDriverSummary(results) {
   console.log(`${completedCount}/${results.length} personas completed`);
 }
 
+// src/treatment-gate.ts
+function gateTreatmentApplied(input, opts = {}) {
+  const failOpen = opts.failOpenWhenNoTelemetry ?? true;
+  let observedTools = 0;
+  let matchedCalls = 0;
+  for (const [tool, count] of Object.entries(input.toolHistogram)) {
+    const n = Number.isFinite(count) && count > 0 ? count : 0;
+    observedTools += n;
+    if (n > 0 && input.matches(tool)) matchedCalls += n;
+  }
+  if (observedTools === 0) {
+    return failOpen ? {
+      applied: true,
+      gated: false,
+      reason: "no tool telemetry captured \u2014 fail-open (not quarantined)",
+      matchedCalls: 0,
+      observedTools: 0
+    } : {
+      applied: false,
+      gated: true,
+      reason: "no tool telemetry captured \u2014 fail-closed",
+      matchedCalls: 0,
+      observedTools: 0
+    };
+  }
+  if (matchedCalls === 0) {
+    return {
+      applied: false,
+      gated: true,
+      reason: `treatment tool never fired (${observedTools} tool calls, 0 matched)`,
+      matchedCalls: 0,
+      observedTools
+    };
+  }
+  return { applied: true, gated: false, matchedCalls, observedTools };
+}
+function gateTreatmentFromMetrics(metrics, matches2, opts) {
+  return gateTreatmentApplied({ toolHistogram: metrics.toolHistogram, matches: matches2 }, opts);
+}
+function gateTreatmentFromSpans(spans, matches2, opts) {
+  return gateTreatmentFromMetrics(computeTraceMetrics(spans), matches2, opts);
+}
+function gateTreatmentFromToolSpans(toolSpans2, matches2, opts) {
+  const toolHistogram = {};
+  for (const s of toolSpans2) {
+    if (s.toolName) toolHistogram[s.toolName] = (toolHistogram[s.toolName] ?? 0) + 1;
+  }
+  return gateTreatmentApplied({ toolHistogram, matches: matches2 }, opts);
+}
+function classifyTreatment(_record, gate) {
+  return gate.gated ? "treatment-not-applied" : "measurable";
+}
+
 // src/anti-slop.ts
 var DEFAULT_HEDGES = [
   /\bi\s+could\s+be\s+wrong\b/i,
@@ -3922,6 +3975,97 @@ var BudgetGuard = class {
   }
 };
 
+// src/hidden-criteria-grading.ts
+var hiddenDestinations = /* @__PURE__ */ new Set([
+  "grading-only",
+  "judge-only"
+]);
+function isHiddenDestination(destination) {
+  return hiddenDestinations.has(destination);
+}
+function routeFields(routing, values) {
+  const out = [];
+  for (const name of Object.keys(routing)) {
+    const value = values[name];
+    if (value === void 0) {
+      throw new ValidationError(
+        `routed field "${name}" has a destination but no value \u2014 every routed field must carry its value`
+      );
+    }
+    out.push({ name, value, destination: routing[name] });
+  }
+  return out;
+}
+function assertNoHiddenLeak(fields, agentContext, opts = {}) {
+  const minLen = opts.minMatchLength ?? 12;
+  const leaks = [];
+  for (const field of fields) {
+    if (!isHiddenDestination(field.destination)) continue;
+    const needle = field.value.trim();
+    if (needle.length < minLen) continue;
+    if (agentContext.includes(needle)) {
+      leaks.push({ field: field.name, destination: field.destination });
+    }
+  }
+  if (leaks.length > 0) {
+    const detail = leaks.map((l) => `"${l.field}" (${l.destination})`).join(", ");
+    throw new ValidationError(
+      `hidden-criteria firewall breached: ${leaks.length} hidden field(s) reached the agent context: ${detail}`
+    );
+  }
+  return fields;
+}
+function agentVisibleFields(fields) {
+  return fields.filter((f) => !isHiddenDestination(f.destination));
+}
+function hiddenGrade(passed, total, notes) {
+  const p = Number.isFinite(passed) && passed > 0 ? Math.floor(passed) : 0;
+  const t = Number.isFinite(total) && total > 0 ? Math.floor(total) : 0;
+  const passRate = t > 0 ? Math.min(1, p / t) : 0;
+  return { passed: Math.min(p, t > 0 ? t : p), total: t, passRate, notes };
+}
+async function gradeOnHidden(args) {
+  assertNoHiddenLeak(args.firewall.fields, args.firewall.agentContext, args.firewall.options);
+  const result = await args.grader(args.artifact, args.hiddenCriteria, args.signal);
+  return hiddenGrade(result.passed, result.total, result.notes);
+}
+var defaultBlendWeights = { heldout: 0.7, judge: 0.3 };
+function normalizeWeights(weights) {
+  const h = Number.isFinite(weights.heldout) && weights.heldout >= 0 ? weights.heldout : 0;
+  const j = Number.isFinite(weights.judge) && weights.judge >= 0 ? weights.judge : 0;
+  const sum3 = h + j;
+  if (sum3 <= 0) {
+    throw new ValidationError(
+      "blend weights must have a positive sum (got heldout+judge <= 0) \u2014 cannot weight a composite by zero"
+    );
+  }
+  return { heldout: h / sum3, judge: j / sum3 };
+}
+function blendHeldout(heldoutPassRate, judgeScore, weights = defaultBlendWeights) {
+  const w = normalizeWeights(weights);
+  const heldout = clampUnit(heldoutPassRate);
+  const judge = clampUnit(judgeScore);
+  return w.heldout * heldout + w.judge * judge;
+}
+function withHeldoutBlend(score, heldoutPassRate, weights = defaultBlendWeights) {
+  return async (input) => {
+    const base = await score(input);
+    if (base.failed) return base;
+    const rate = clampUnit(heldoutPassRate(input.artifact));
+    const composite = blendHeldout(rate, base.composite, weights);
+    const w = normalizeWeights(weights);
+    return {
+      ...base,
+      composite,
+      notes: `composite=${composite.toFixed(3)} (held-out ${(rate * 100).toFixed(0)}% \xD7 ${w.heldout.toFixed(2)} + quality ${base.composite.toFixed(3)} \xD7 ${w.judge.toFixed(2)})` + (base.notes ? ` \u2014 ${base.notes}` : "")
+    };
+  };
+}
+function clampUnit(value) {
+  if (!Number.isFinite(value)) return 0;
+  return Math.max(0, Math.min(1, value));
+}
+
 // src/cost-ledger.ts
 function modelPriceKey(model) {
   return isModelPriced(model) ? model : null;
@@ -10027,6 +10171,7 @@ export {
   agentProfileHash,
   agentProfileId,
   agentProfileModelId,
+  agentVisibleFields,
   aggregateJudgeVerdicts,
   aggregateLlm,
   aggregatePrReviewScore,
@@ -10042,6 +10187,7 @@ export {
   assertCrossFamily,
   assertLlmRoute,
   assertModelsServed,
+  assertNoHiddenLeak,
   assertRealBackend,
   assertRecordIntegrity,
   assertReleaseConfidence,
@@ -10058,6 +10204,7 @@ export {
   benchmarks_exports as benchmarks,
   benjaminiHochberg,
   bisect,
+  blendHeldout,
   blockingKnowledgeEval,
   bonferroni,
   bootstrapCi,
@@ -10092,6 +10239,7 @@ export {
   clamp01,
   classifyEuAiRisk,
   classifyFailure,
+  classifyTreatment,
   cliffsDelta,
   codeExecutionJudge,
   cohensD,
@@ -10145,6 +10293,7 @@ export {
   decideNextUserTurn,
   decideReferenceReplayPromotion,
   decideReferenceReplayRunPromotion,
+  defaultBlendWeights,
   defaultIsMaterial,
   defaultJudges,
   defaultParseStudentLabel,
@@ -10210,9 +10359,14 @@ export {
   formatScorecardDiff,
   gainHistogram,
   gatePerf,
+  gateTreatmentApplied,
+  gateTreatmentFromMetrics,
+  gateTreatmentFromSpans,
+  gateTreatmentFromToolSpans,
   ghCliClient,
   gitProvenanceReader,
   precision as goldenPrecision,
+  gradeOnHidden,
   gradeSemanticStatus,
   groupBy,
   groupRunsByAgentProfileCell,
@@ -10220,6 +10374,7 @@ export {
   hashJson,
   hashScenarios,
   hashToUnit,
+  hiddenGrade,
   htmlContainsElement,
   httpGithubClient,
   improvementVerdict,
@@ -10233,6 +10388,7 @@ export {
   interRaterReliability,
   interpretCliffs,
   iqr,
+  isHiddenDestination,
   isJudgeSpan,
   isLlmSpan,
   isModelPriced,
@@ -10351,6 +10507,7 @@ export {
   resolveModelPricing,
   resolveSeat,
   roundTripRunRecord,
+  routeFields,
   rowCount,
   rowWhere,
   runAgentControlLoop,
@@ -10449,6 +10606,7 @@ export {
   wilcoxonSignedRank,
   wilson,
   withAssignedFeedbackSplit,
+  withHeldoutBlend,
   withJudgeRetry,
   withOtelPipeline,
   wranglerDeployRunner