@tangle-network/agent-eval 0.99.0 → 0.100.1

package/dist/index.js

@@ -51,7 +51,7 @@ import {
   llmJudge,
   parseCorrectnessResponse,
   verifyCompletion
-} from "./chunk-OYU4D7FY.js";
+} from "./chunk-VWQ6PO5O.js";
 import {
   DEFAULT_MUTATION_PRIMITIVES,
   DEFAULT_RED_TEAM_CORPUS,
@@ -75,7 +75,7 @@ import {
   scoreRedTeamOutput,
   surfaceContentHash,
   toolNamesForRun
-} from "./chunk-WMBLMTUE.js";
+} from "./chunk-2KTBHICD.js";
 import {
   BackendIntegrityError,
   assertRealBackend,
@@ -133,20 +133,32 @@ import {
   defaultIsMaterial,
   diffFindings,
   runSemanticConceptJudge
-} from "./chunk-SJISCGWD.js";
-import {
-  buildDefaultAnalystRegistry,
-  computeTraceMetrics
-} from "./chunk-3NHEO6ZC.js";
+} from "./chunk-JU6ZX3CX.js";
 import {
   LockedJsonlAppender,
   Mutex
 } from "./chunk-DJWX3GVS.js";
+import {
+  buildDefaultAnalystRegistry,
+  computeTraceMetrics
+} from "./chunk-L5TVEZFT.js";
 import {
   DEFAULT_RUN_SCORE_WEIGHTS,
+  POLICY_EDIT_AXES,
+  POLICY_EDIT_TARGET_SURFACES,
+  PolicyEditValidationError,
+  admitPolicyEdit,
   aggregateRunScore,
-  clamp01
-} from "./chunk-BUTW4RGG.js";
+  applyPolicyEditToSurface,
+  clamp01,
+  computePolicyEditId,
+  isPolicyEdit,
+  makePolicyEdit,
+  policyEditFromFinding,
+  policyEditsFromFindings,
+  scorePolicyEditReadiness,
+  validatePolicyEdit
+} from "./chunk-IN3SHQML.js";
 import {
   AnalystRegistry,
   DEFAULT_TRACE_ANALYST_KINDS,
@@ -156,7 +168,7 @@ import {
   KNOWLEDGE_POISONING_KIND_SPEC,
   createTraceAnalystKind,
   renderPriorFindings
-} from "./chunk-S4SYLDFX.js";
+} from "./chunk-FRI6RG3P.js";
 import {
   controlFailureClassFromVerification,
   controlRunToRunRecord,
@@ -167,7 +179,7 @@ import {
   runProposeReview,
   runProposeReviewAsControlLoop,
   scoreFromEvals
-} from "./chunk-NZEQVRH5.js";
+} from "./chunk-BOETF6BU.js";
 import {
   allCriticalPassed,
   errorStreakDetector,
@@ -189,7 +201,7 @@ import {
 } from "./chunk-VDGPPGE3.js";
 import {
   runEvalCampaign
-} from "./chunk-OKQ2LAT7.js";
+} from "./chunk-4LWD6GC7.js";
 import {
   LlmCallError,
   LlmClient,
@@ -292,13 +304,13 @@ import {
   scoreTraceInsightReadiness,
   tokenizeDomainWords,
   traceAnalystOnRunComplete
-} from "./chunk-77T4STFI.js";
+} from "./chunk-PMF5WIBX.js";
 import {
   TRACE_ANALYST_ACTOR_DESCRIPTION,
   TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
   TRACE_ANALYST_SUBAGENT_DESCRIPTION,
   analyzeTraces
-} from "./chunk-IZCEK2HR.js";
+} from "./chunk-2MLIEQSN.js";
 import {
   DEFAULT_REDACTION_RULES,
   REDACTION_VERSION,
@@ -370,29 +382,31 @@ import {
   defaultProviderRedactor,
   providerFromBaseUrl
 } from "./chunk-PC4UYEBM.js";
+import {
+  RunRecordValidationError,
+  isRunRecord,
+  parseRunRecordSafe,
+  roundTripRunRecord,
+  validateRunRecord
+} from "./chunk-G6S73VA7.js";
+import {
+  TraceEmitter,
+  llmSpanFromProvider
+} from "./chunk-TVVP3ZZQ.js";
 import {
   AGENT_PROFILE_KINDS,
   AgentProfileCellValidationError,
-  RunRecordValidationError,
   agentProfileCellHashMaterial,
   agentProfileCellKey,
   assertRunAgentProfileCell,
   buildAgentInterfaceProfileCell,
   buildAgentProfileCell,
   groupRunsByAgentProfileCell,
-  isRunRecord,
-  parseRunRecordSafe,
   requireAgentProfileCell,
-  roundTripRunRecord,
   toAgentProfileJson,
   validateAgentProfileCell,
-  validateRunRecord,
   verifyAgentProfileCell
-} from "./chunk-LO6IOIJ2.js";
-import {
-  TraceEmitter,
-  llmSpanFromProvider
-} from "./chunk-TVVP3ZZQ.js";
+} from "./chunk-ABOIVNXL.js";
 import {
   canonicalize,
   evaluateHypothesis,
@@ -2593,6 +2607,59 @@ function printDriverSummary(results) {
   console.log(`${completedCount}/${results.length} personas completed`);
 }
 
+// src/treatment-gate.ts
+function gateTreatmentApplied(input, opts = {}) {
+  const failOpen = opts.failOpenWhenNoTelemetry ?? true;
+  let observedTools = 0;
+  let matchedCalls = 0;
+  for (const [tool, count] of Object.entries(input.toolHistogram)) {
+    const n = Number.isFinite(count) && count > 0 ? count : 0;
+    observedTools += n;
+    if (n > 0 && input.matches(tool)) matchedCalls += n;
+  }
+  if (observedTools === 0) {
+    return failOpen ? {
+      applied: true,
+      gated: false,
+      reason: "no tool telemetry captured \u2014 fail-open (not quarantined)",
+      matchedCalls: 0,
+      observedTools: 0
+    } : {
+      applied: false,
+      gated: true,
+      reason: "no tool telemetry captured \u2014 fail-closed",
+      matchedCalls: 0,
+      observedTools: 0
+    };
+  }
+  if (matchedCalls === 0) {
+    return {
+      applied: false,
+      gated: true,
+      reason: `treatment tool never fired (${observedTools} tool calls, 0 matched)`,
+      matchedCalls: 0,
+      observedTools
+    };
+  }
+  return { applied: true, gated: false, matchedCalls, observedTools };
+}
+function gateTreatmentFromMetrics(metrics, matches2, opts) {
+  return gateTreatmentApplied({ toolHistogram: metrics.toolHistogram, matches: matches2 }, opts);
+}
+function gateTreatmentFromSpans(spans, matches2, opts) {
+  return gateTreatmentFromMetrics(computeTraceMetrics(spans), matches2, opts);
+}
+function gateTreatmentFromToolSpans(toolSpans2, matches2, opts) {
+  const toolHistogram = {};
+  for (const s of toolSpans2) {
+    if (s.toolName) toolHistogram[s.toolName] = (toolHistogram[s.toolName] ?? 0) + 1;
+  }
+  return gateTreatmentApplied({ toolHistogram, matches: matches2 }, opts);
+}
+function classifyTreatment(_record, gate) {
+  return gate.gated ? "treatment-not-applied" : "measurable";
+}
+
 // src/anti-slop.ts
 var DEFAULT_HEDGES = [
   /\bi\s+could\s+be\s+wrong\b/i,
@@ -3922,6 +3989,97 @@ var BudgetGuard = class {
   }
 };
 
+// src/hidden-criteria-grading.ts
+var hiddenDestinations = /* @__PURE__ */ new Set([
+  "grading-only",
+  "judge-only"
+]);
+function isHiddenDestination(destination) {
+  return hiddenDestinations.has(destination);
+}
+function routeFields(routing, values) {
+  const out = [];
+  for (const name of Object.keys(routing)) {
+    const value = values[name];
+    if (value === void 0) {
+      throw new ValidationError(
+        `routed field "${name}" has a destination but no value \u2014 every routed field must carry its value`
+      );
+    }
+    out.push({ name, value, destination: routing[name] });
+  }
+  return out;
+}
+function assertNoHiddenLeak(fields, agentContext, opts = {}) {
+  const minLen = opts.minMatchLength ?? 12;
+  const leaks = [];
+  for (const field of fields) {
+    if (!isHiddenDestination(field.destination)) continue;
+    const needle = field.value.trim();
+    if (needle.length < minLen) continue;
+    if (agentContext.includes(needle)) {
+      leaks.push({ field: field.name, destination: field.destination });
+    }
+  }
+  if (leaks.length > 0) {
+    const detail = leaks.map((l) => `"${l.field}" (${l.destination})`).join(", ");
+    throw new ValidationError(
+      `hidden-criteria firewall breached: ${leaks.length} hidden field(s) reached the agent context: ${detail}`
+    );
+  }
+  return fields;
+}
+function agentVisibleFields(fields) {
+  return fields.filter((f) => !isHiddenDestination(f.destination));
+}
+function hiddenGrade(passed, total, notes) {
+  const p = Number.isFinite(passed) && passed > 0 ? Math.floor(passed) : 0;
+  const t = Number.isFinite(total) && total > 0 ? Math.floor(total) : 0;
+  const passRate = t > 0 ? Math.min(1, p / t) : 0;
+  return { passed: Math.min(p, t > 0 ? t : p), total: t, passRate, notes };
+}
+async function gradeOnHidden(args) {
+  assertNoHiddenLeak(args.firewall.fields, args.firewall.agentContext, args.firewall.options);
+  const result = await args.grader(args.artifact, args.hiddenCriteria, args.signal);
+  return hiddenGrade(result.passed, result.total, result.notes);
+}
+var defaultBlendWeights = { heldout: 0.7, judge: 0.3 };
+function normalizeWeights(weights) {
+  const h = Number.isFinite(weights.heldout) && weights.heldout >= 0 ? weights.heldout : 0;
+  const j = Number.isFinite(weights.judge) && weights.judge >= 0 ? weights.judge : 0;
+  const sum3 = h + j;
+  if (sum3 <= 0) {
+    throw new ValidationError(
+      "blend weights must have a positive sum (got heldout+judge <= 0) \u2014 cannot weight a composite by zero"
+    );
+  }
+  return { heldout: h / sum3, judge: j / sum3 };
+}
+function blendHeldout(heldoutPassRate, judgeScore, weights = defaultBlendWeights) {
+  const w = normalizeWeights(weights);
+  const heldout = clampUnit(heldoutPassRate);
+  const judge = clampUnit(judgeScore);
+  return w.heldout * heldout + w.judge * judge;
+}
+function withHeldoutBlend(score, heldoutPassRate, weights = defaultBlendWeights) {
+  return async (input) => {
+    const base = await score(input);
+    if (base.failed) return base;
+    const rate = clampUnit(heldoutPassRate(input.artifact));
+    const composite = blendHeldout(rate, base.composite, weights);
+    const w = normalizeWeights(weights);
+    return {
+      ...base,
+      composite,
+      notes: `composite=${composite.toFixed(3)} (held-out ${(rate * 100).toFixed(0)}% \xD7 ${w.heldout.toFixed(2)} + quality ${base.composite.toFixed(3)} \xD7 ${w.judge.toFixed(2)})` + (base.notes ? ` \u2014 ${base.notes}` : "")
+    };
+  };
+}
+function clampUnit(value) {
+  if (!Number.isFinite(value)) return 0;
+  return Math.max(0, Math.min(1, value));
+}
+
 // src/cost-ledger.ts
 function modelPriceKey(model) {
   return isModelPriced(model) ? model : null;
@@ -9982,7 +10140,10 @@ export {
   OPENINFERENCE_SPAN_KIND,
   OTEL_AGENT_EVAL_SCOPE,
   OtlpFileTraceStore,
+  POLICY_EDIT_AXES,
+  POLICY_EDIT_TARGET_SURFACES,
   PairwiseSteeringOptimizer,
+  PolicyEditValidationError,
   ProductClient,
   PromptRegistry,
   REDACTION_VERSION,
@@ -10021,12 +10182,14 @@ export {
   ValidationError,
   VerificationError,
   acquisitionPlansForKnowledgeGaps,
+  admitPolicyEdit,
   adversarialJudge,
   agentProfileCellHashMaterial,
   agentProfileCellKey,
   agentProfileHash,
   agentProfileId,
   agentProfileModelId,
+  agentVisibleFields,
   aggregateJudgeVerdicts,
   aggregateLlm,
   aggregatePrReviewScore,
@@ -10036,12 +10199,14 @@ export {
   analyzeSeries,
   analyzeTraces,
   appendScorecard,
+  applyPolicyEditToSurface,
   argHash,
   asNumber,
   asString,
   assertCrossFamily,
   assertLlmRoute,
   assertModelsServed,
+  assertNoHiddenLeak,
   assertRealBackend,
   assertRecordIntegrity,
   assertReleaseConfidence,
@@ -10058,6 +10223,7 @@ export {
   benchmarks_exports as benchmarks,
   benjaminiHochberg,
   bisect,
+  blendHeldout,
   blockingKnowledgeEval,
   bonferroni,
   bootstrapCi,
@@ -10092,6 +10258,7 @@ export {
   clamp01,
   classifyEuAiRisk,
   classifyFailure,
+  classifyTreatment,
   cliffsDelta,
   codeExecutionJudge,
   cohensD,
@@ -10107,6 +10274,7 @@ export {
   composeValidators,
   computeExperimentStats,
   computeFindingId,
+  computePolicyEditId,
   computeToolUseMetrics,
   computeTraceMetrics,
   confidenceInterval,
@@ -10145,6 +10313,7 @@ export {
   decideNextUserTurn,
   decideReferenceReplayPromotion,
   decideReferenceReplayRunPromotion,
+  defaultBlendWeights,
   defaultIsMaterial,
   defaultJudges,
   defaultParseStudentLabel,
@@ -10210,9 +10379,14 @@ export {
   formatScorecardDiff,
   gainHistogram,
   gatePerf,
+  gateTreatmentApplied,
+  gateTreatmentFromMetrics,
+  gateTreatmentFromSpans,
+  gateTreatmentFromToolSpans,
   ghCliClient,
   gitProvenanceReader,
   precision as goldenPrecision,
+  gradeOnHidden,
   gradeSemanticStatus,
   groupBy,
   groupRunsByAgentProfileCell,
@@ -10220,6 +10394,7 @@ export {
   hashJson,
   hashScenarios,
   hashToUnit,
+  hiddenGrade,
   htmlContainsElement,
   httpGithubClient,
   improvementVerdict,
@@ -10233,10 +10408,12 @@ export {
   interRaterReliability,
   interpretCliffs,
   iqr,
+  isHiddenDestination,
   isJudgeSpan,
   isLlmSpan,
   isModelPriced,
   isOtelConfigured,
+  isPolicyEdit,
   isRetrievalSpan,
   isRunRecord,
   isSandboxSpan,
@@ -10265,6 +10442,7 @@ export {
   lowercaseMutator,
   makeEvalTools,
   makeFinding,
+  makePolicyEdit,
   mannWhitneyU,
   matchGoldens,
   matchSpan,
@@ -10309,6 +10487,8 @@ export {
   pearsonR,
   pixelDeltaRatio,
   planTraceInsightQuestions,
+  policyEditFromFinding,
+  policyEditsFromFindings,
   politenessPrefixMutator,
   positionalBias,
   preflightModels,
@@ -10351,6 +10531,7 @@ export {
   resolveModelPricing,
   resolveSeat,
   roundTripRunRecord,
+  routeFields,
   rowCount,
   rowWhere,
   runAgentControlLoop,
@@ -10383,6 +10564,7 @@ export {
   scoreContinuity,
   scoreFromEvals,
   scoreKnowledgeReadiness,
+  scorePolicyEditReadiness,
   scorePrReviewComments,
   scorePrReviewSource,
   scoreRedTeamOutput,
@@ -10432,6 +10614,7 @@ export {
   urlContains,
   userQuestionsForKnowledgeGaps,
   validateAgentProfileCell,
+  validatePolicyEdit,
   validateRunRecord,
   verbosityBias,
   verifyAgentProfileCell,
@@ -10449,6 +10632,7 @@ export {
   wilcoxonSignedRank,
   wilson,
   withAssignedFeedbackSplit,
+  withHeldoutBlend,
   withJudgeRetry,
   withOtelPipeline,
   wranglerDeployRunner