@tangle-network/agent-eval 0.82.0 → 0.84.0

package/dist/index.js

@@ -6,6 +6,10 @@ import {
   parseCorrectnessResponse,
   verifyCompletion
 } from "./chunk-YGYXHNAQ.js";
+import {
+  parseRuntimeTrajectoryHookEvent,
+  projectRuntimeTrajectoryEvidence
+} from "./chunk-T4SQEITX.js";
 import {
   HoldoutAuditor,
   canaryLeakView,
@@ -2911,14 +2915,14 @@ async function runHarnessExperiment(config) {
   const score = config.score ?? ((trace) => critic.scoreTrace(trace));
   const results = await mapLimit(jobs, config.parallelism ?? 1, async (request) => {
     const trace = await config.adapter.run(request);
-    const runScore2 = await score(trace, request);
+    const runScore3 = await score(trace, request);
     const result = {
       variant: request.variant,
       scenario: request.scenario,
       trialIndex: request.trialIndex,
       trace,
-      score: runScore2,
-      aggregate: aggregateRunScore(runScore2, config.weights)
+      score: runScore3,
+      aggregate: aggregateRunScore(runScore3, config.weights)
     };
     await config.onResult?.(result);
     return result;
@@ -7089,6 +7093,121 @@ function createDefaultReviewer(options) {
   };
 }
 
+// src/description-length-gate.ts
+import { gzipSync } from "zlib";
+function runScore2(run) {
+  const o = run.outcome;
+  const s = o.holdoutScore ?? o.searchScore ?? o.raw?.score;
+  return typeof s === "number" && Number.isFinite(s) ? s : void 0;
+}
+function taskKey(run) {
+  return run.scenarioId ?? run.experimentId;
+}
+function perTaskMeanScore(runs) {
+  const acc = /* @__PURE__ */ new Map();
+  for (const run of runs) {
+    const s = runScore2(run);
+    if (s === void 0) continue;
+    const key = taskKey(run);
+    const cur = acc.get(key) ?? { sum: 0, n: 0 };
+    cur.sum += s;
+    cur.n += 1;
+    acc.set(key, cur);
+  }
+  return new Map([...acc].map(([k, v]) => [k, v.sum / v.n]));
+}
+function modelDescriptionBits(content) {
+  return gzipSync(Buffer.from(content, "utf8")).byteLength * 8;
+}
+function dataDescriptionBits(scoreByTask, keys, scoreFloor) {
+  let bits = 0;
+  for (const key of keys) {
+    const s = scoreByTask.get(key);
+    if (s === void 0) continue;
+    bits += -Math.log2(Math.max(s, scoreFloor));
+  }
+  return bits;
+}
+var DescriptionLengthGate = class {
+  baselineKey;
+  lambda;
+  marginBits;
+  scoreFloor;
+  minTasks;
+  constructor(config) {
+    if (!config.baselineKey) throw new Error("DescriptionLengthGate: baselineKey is required");
+    this.baselineKey = config.baselineKey;
+    this.lambda = config.lambda ?? 1;
+    this.marginBits = config.marginBits ?? 0;
+    this.scoreFloor = config.scoreFloor ?? 2 ** -10;
+    this.minTasks = config.minTasks ?? 3;
+    if (!(this.lambda >= 0)) throw new Error("DescriptionLengthGate: lambda must be \u2265 0");
+    if (!(this.scoreFloor > 0 && this.scoreFloor < 1))
+      throw new Error("DescriptionLengthGate: scoreFloor must be in (0,1)");
+  }
+  /** Decide whether `candidate` should replace `baseline`. Both are scored on
+   *  the shared task set (the enlarged evidence); the candidate promotes only
+   *  if λ·L_model + L_data is strictly lower by at least `marginBits`. */
+  evaluate(candidate, baseline) {
+    const candidateId = inferCandidateId(candidate.runs, this.baselineKey);
+    const candScores = perTaskMeanScore(candidate.runs);
+    const baseScores = perTaskMeanScore(baseline.runs);
+    const shared = [...candScores.keys()].filter((k) => baseScores.has(k));
+    const modelBits = {
+      candidate: this.lambda * modelDescriptionBits(candidate.content),
+      baseline: this.lambda * modelDescriptionBits(baseline.content)
+    };
+    const dataBits = {
+      candidate: dataDescriptionBits(candScores, shared, this.scoreFloor),
+      baseline: dataDescriptionBits(baseScores, shared, this.scoreFloor)
+    };
+    const totalBits = {
+      candidate: modelBits.candidate + dataBits.candidate,
+      baseline: modelBits.baseline + dataBits.baseline
+    };
+    const evidence = {
+      tasks: shared.length,
+      modelBits,
+      dataBits,
+      totalBits,
+      deltaBits: totalBits.candidate - totalBits.baseline,
+      modelBitsDelta: modelBits.candidate - modelBits.baseline,
+      dataBitsDelta: dataBits.candidate - dataBits.baseline
+    };
+    const base = { candidateId, baselineId: this.baselineKey, evidence };
+    const fmt2 = (n) => n.toFixed(1);
+    if (shared.length < this.minTasks) {
+      return {
+        ...base,
+        promote: false,
+        reason: `few_tasks: ${shared.length} shared task(s) < min ${this.minTasks}`,
+        rejectionCode: "few_tasks"
+      };
+    }
+    if (!(evidence.deltaBits < -this.marginBits)) {
+      const code = evidence.dataBitsDelta < 0 ? "model_bloat" : "no_total_gain";
+      const why = code === "model_bloat" ? `model grew ${fmt2(evidence.modelBitsDelta)} bits, outpacing a ${fmt2(-evidence.dataBitsDelta)}-bit data gain` : `outcomes did not improve (data \u0394=${fmt2(evidence.dataBitsDelta)} bits)`;
+      return {
+        ...base,
+        promote: false,
+        reason: `${code}: total \u0394=${fmt2(evidence.deltaBits)} bits does not clear the ${fmt2(this.marginBits)}-bit margin \u2014 ${why}`,
+        rejectionCode: code
+      };
+    }
+    return {
+      ...base,
+      promote: true,
+      reason: `promote: total \u0394=${fmt2(evidence.deltaBits)} bits (model \u0394=${fmt2(evidence.modelBitsDelta)}, data \u0394=${fmt2(evidence.dataBitsDelta)}) over ${shared.length} tasks`,
+      rejectionCode: null
+    };
+  }
+};
+function inferCandidateId(runs, baselineKey) {
+  for (const run of runs)
+    if (run.candidateId && run.candidateId !== baselineKey) return run.candidateId;
+  return runs[0]?.candidateId ?? "(unknown candidate)";
+}
+
 // src/discover-personas.ts
 import { promises as fs } from "fs";
 import { basename, extname, join as join3 } from "path";
@@ -7240,7 +7359,7 @@ var HeldOutGate = class {
    *  the candidate run with the matching baseline run. Pairs without
    *  a holdout score on both sides are dropped. */
   evaluate(candidate, baseline) {
-    const candidateId = inferCandidateId(candidate, this.baselineKey);
+    const candidateId = inferCandidateId2(candidate, this.baselineKey);
     const baselineId = this.baselineKey;
     const baselineHoldoutByKey = indexHoldoutByKey(baseline);
     const beforeHoldout = [];
@@ -7343,7 +7462,7 @@ var HeldOutGate = class {
     };
   }
 };
-function inferCandidateId(candidate, baselineKey) {
+function inferCandidateId2(candidate, baselineKey) {
   for (const run of candidate) {
     if (run.candidateId && run.candidateId !== baselineKey) return run.candidateId;
   }
@@ -8314,6 +8433,7 @@ export {
   DEFAULT_TRACE_ANALYST_BUDGETS,
   DEFAULT_TRACE_ANALYST_KINDS,
   Dataset,
+  DescriptionLengthGate,
   DockerSandboxDriver,
   DualAgentBench,
   ERROR_COUNT_PATTERNS,
@@ -8485,6 +8605,7 @@ export {
   createTraceAnalystKind,
   crossTraceDiff,
   crowdingDistance,
+  dataDescriptionBits,
   decideNextUserTurn,
   decideReferenceReplayPromotion,
   decideReferenceReplayRunPromotion,
@@ -8591,6 +8712,7 @@ export {
   matchGoldens,
   mergeLayerResults,
   mergeSteeringBundle,
+  modelDescriptionBits,
   multiToolchainLayer,
   nistAiRmfReport,
   normalizeScores,
@@ -8613,6 +8735,7 @@ export {
   parseGoldJsonl,
   parseReflectionResponse,
   parseRunRecordSafe,
+  parseRuntimeTrajectoryHookEvent,
   partialCredit,
   passOrthogonality,
   pixelDeltaRatio,
@@ -8623,6 +8746,7 @@ export {
   probeLlm,
   profile_exports as profile,
   projectOtlpFlatLine,
+  projectRuntimeTrajectoryEvidence,
   promptBisect,
   proposeAutomatedPullRequest,
   proposeSynthesisTargets,