@tangle-network/agent-eval 0.83.0 → 0.85.0

package/dist/index.js

@@ -6,6 +6,10 @@ import {
   parseCorrectnessResponse,
   verifyCompletion
 } from "./chunk-YGYXHNAQ.js";
+import {
+  parseRuntimeTrajectoryHookEvent,
+  projectRuntimeTrajectoryEvidence
+} from "./chunk-T4SQEITX.js";
 import {
   HoldoutAuditor,
   canaryLeakView,
@@ -1747,6 +1751,107 @@ function canonicalize2(value) {
   return out;
 }
 
+// src/integrity/preflight.ts
+function stripSlash(url) {
+  return url.replace(/\/+$/, "");
+}
+function errorMessage(body) {
+  if (body == null || typeof body !== "object") return null;
+  const b = body;
+  if (b.error && typeof b.error.message === "string") return b.error.message;
+  if (typeof b.message === "string") return b.message;
+  return null;
+}
+async function preflightModels(opts) {
+  const fetchImpl = opts.fetchImpl ?? fetch;
+  const baseUrl = stripSlash(opts.baseUrl);
+  const authHeaders = { authorization: `Bearer ${opts.apiKey}` };
+  let served;
+  try {
+    const res = await fetchImpl(`${baseUrl}/models`, { method: "GET", headers: authHeaders });
+    if (!res.ok) {
+      const text = await res.text().catch(() => "");
+      return {
+        succeeded: false,
+        value: null,
+        error: `preflightModels: GET ${baseUrl}/models \u2192 ${res.status} ${text.slice(0, 400)}`
+      };
+    }
+    const body = await res.json();
+    const ids = Array.isArray(body.data) ? body.data : [];
+    served = new Set(ids.map((m) => m.id).filter((id) => typeof id === "string"));
+  } catch (err) {
+    return {
+      succeeded: false,
+      value: null,
+      error: `preflightModels: GET ${baseUrl}/models failed \u2014 ${err instanceof Error ? err.message : String(err)}`
+    };
+  }
+  const results = [];
+  for (const model of opts.models) {
+    const listed = served.has(model);
+    if (!opts.probe) {
+      results.push({ model, listed, served: null, status: null, detail: null });
+      continue;
+    }
+    try {
+      const res = await fetchImpl(`${baseUrl}/chat/completions`, {
+        method: "POST",
+        headers: { ...authHeaders, "content-type": "application/json" },
+        body: JSON.stringify({
+          model,
+          messages: [{ role: "user", content: "ping" }],
+          max_tokens: 5
+        })
+      });
+      let detail = null;
+      if (!res.ok) {
+        const body = await res.json().catch(() => null);
+        detail = errorMessage(body);
+      }
+      results.push({ model, listed, served: res.ok, status: res.status, detail });
+    } catch (err) {
+      return {
+        succeeded: false,
+        value: null,
+        error: `preflightModels: probe POST ${baseUrl}/chat/completions (model ${model}) failed \u2014 ${err instanceof Error ? err.message : String(err)}`
+      };
+    }
+  }
+  return { succeeded: true, value: results, error: null };
+}
+var ModelsUnreachableError = class extends AgentEvalError {
+  constructor(message, results) {
+    super("config", message);
+    this.results = results;
+    this.name = "ModelsUnreachableError";
+  }
+  results;
+};
+function describeFailure(r) {
+  if (!r.listed) {
+    const probeNote = r.served === false ? ` (probe ${r.status}${r.detail ? `: ${r.detail}` : ""})` : "";
+    return `${r.model}: not in /models${probeNote}`;
+  }
+  return `${r.model}: listed but probe ${r.status}${r.detail ? ` \u2014 ${r.detail}` : ""}`;
+}
+async function assertModelsServed(opts) {
+  const outcome = await preflightModels(opts);
+  if (!outcome.succeeded || outcome.value === null) {
+    throw new ConfigError(
+      outcome.error ?? "assertModelsServed: preflight failed without an error message"
+    );
+  }
+  const dead = outcome.value.filter((r) => !r.listed || r.served === false);
+  if (dead.length > 0) {
+    throw new ModelsUnreachableError(
+      `assertModelsServed: ${dead.length}/${outcome.value.length} model(s) unreachable on the router \u2014 ${dead.map(describeFailure).join("; ")}`,
+      outcome.value
+    );
+  }
+  return outcome.value;
+}
+
 // src/integrity/single-backend.ts
 var SingleBackendError = class extends AgentEvalError {
   constructor(message, report) {
@@ -1756,7 +1861,7 @@ var SingleBackendError = class extends AgentEvalError {
   }
   report;
 };
-function stripSlash(url) {
+function stripSlash2(url) {
   return url.replace(/\/+$/, "");
 }
 function assertSingleBackend(agent, judge, opts = {}) {
@@ -1764,7 +1869,7 @@ function assertSingleBackend(agent, judge, opts = {}) {
   if (agent.kind !== judge.kind) {
     divergences.push({ field: "kind", agent: agent.kind, judge: judge.kind });
   }
-  if (stripSlash(agent.baseUrl) !== stripSlash(judge.baseUrl)) {
+  if (stripSlash2(agent.baseUrl) !== stripSlash2(judge.baseUrl)) {
     divergences.push({ field: "baseUrl", agent: agent.baseUrl, judge: judge.baseUrl });
   }
   if (agent.model !== judge.model) {
@@ -2911,14 +3016,14 @@ async function runHarnessExperiment(config) {
   const score = config.score ?? ((trace) => critic.scoreTrace(trace));
   const results = await mapLimit(jobs, config.parallelism ?? 1, async (request) => {
     const trace = await config.adapter.run(request);
-    const runScore2 = await score(trace, request);
+    const runScore3 = await score(trace, request);
     const result = {
       variant: request.variant,
       scenario: request.scenario,
       trialIndex: request.trialIndex,
       trace,
-      score: runScore2,
-      aggregate: aggregateRunScore(runScore2, config.weights)
+      score: runScore3,
+      aggregate: aggregateRunScore(runScore3, config.weights)
     };
     await config.onResult?.(result);
     return result;
@@ -7089,6 +7194,121 @@ function createDefaultReviewer(options) {
   };
 }
 
+// src/description-length-gate.ts
+import { gzipSync } from "zlib";
+function runScore2(run) {
+  const o = run.outcome;
+  const s = o.holdoutScore ?? o.searchScore ?? o.raw?.score;
+  return typeof s === "number" && Number.isFinite(s) ? s : void 0;
+}
+function taskKey(run) {
+  return run.scenarioId ?? run.experimentId;
+}
+function perTaskMeanScore(runs) {
+  const acc = /* @__PURE__ */ new Map();
+  for (const run of runs) {
+    const s = runScore2(run);
+    if (s === void 0) continue;
+    const key = taskKey(run);
+    const cur = acc.get(key) ?? { sum: 0, n: 0 };
+    cur.sum += s;
+    cur.n += 1;
+    acc.set(key, cur);
+  }
+  return new Map([...acc].map(([k, v]) => [k, v.sum / v.n]));
+}
+function modelDescriptionBits(content) {
+  return gzipSync(Buffer.from(content, "utf8")).byteLength * 8;
+}
+function dataDescriptionBits(scoreByTask, keys, scoreFloor) {
+  let bits = 0;
+  for (const key of keys) {
+    const s = scoreByTask.get(key);
+    if (s === void 0) continue;
+    bits += -Math.log2(Math.max(s, scoreFloor));
+  }
+  return bits;
+}
+var DescriptionLengthGate = class {
+  baselineKey;
+  lambda;
+  marginBits;
+  scoreFloor;
+  minTasks;
+  constructor(config) {
+    if (!config.baselineKey) throw new Error("DescriptionLengthGate: baselineKey is required");
+    this.baselineKey = config.baselineKey;
+    this.lambda = config.lambda ?? 1;
+    this.marginBits = config.marginBits ?? 0;
+    this.scoreFloor = config.scoreFloor ?? 2 ** -10;
+    this.minTasks = config.minTasks ?? 3;
+    if (!(this.lambda >= 0)) throw new Error("DescriptionLengthGate: lambda must be \u2265 0");
+    if (!(this.scoreFloor > 0 && this.scoreFloor < 1))
+      throw new Error("DescriptionLengthGate: scoreFloor must be in (0,1)");
+  }
+  /** Decide whether `candidate` should replace `baseline`. Both are scored on
+   *  the shared task set (the enlarged evidence); the candidate promotes only
+   *  if λ·L_model + L_data is strictly lower by at least `marginBits`. */
+  evaluate(candidate, baseline) {
+    const candidateId = inferCandidateId(candidate.runs, this.baselineKey);
+    const candScores = perTaskMeanScore(candidate.runs);
+    const baseScores = perTaskMeanScore(baseline.runs);
+    const shared = [...candScores.keys()].filter((k) => baseScores.has(k));
+    const modelBits = {
+      candidate: this.lambda * modelDescriptionBits(candidate.content),
+      baseline: this.lambda * modelDescriptionBits(baseline.content)
+    };
+    const dataBits = {
+      candidate: dataDescriptionBits(candScores, shared, this.scoreFloor),
+      baseline: dataDescriptionBits(baseScores, shared, this.scoreFloor)
+    };
+    const totalBits = {
+      candidate: modelBits.candidate + dataBits.candidate,
+      baseline: modelBits.baseline + dataBits.baseline
+    };
+    const evidence = {
+      tasks: shared.length,
+      modelBits,
+      dataBits,
+      totalBits,
+      deltaBits: totalBits.candidate - totalBits.baseline,
+      modelBitsDelta: modelBits.candidate - modelBits.baseline,
+      dataBitsDelta: dataBits.candidate - dataBits.baseline
+    };
+    const base = { candidateId, baselineId: this.baselineKey, evidence };
+    const fmt2 = (n) => n.toFixed(1);
+    if (shared.length < this.minTasks) {
+      return {
+        ...base,
+        promote: false,
+        reason: `few_tasks: ${shared.length} shared task(s) < min ${this.minTasks}`,
+        rejectionCode: "few_tasks"
+      };
+    }
+    if (!(evidence.deltaBits < -this.marginBits)) {
+      const code = evidence.dataBitsDelta < 0 ? "model_bloat" : "no_total_gain";
+      const why = code === "model_bloat" ? `model grew ${fmt2(evidence.modelBitsDelta)} bits, outpacing a ${fmt2(-evidence.dataBitsDelta)}-bit data gain` : `outcomes did not improve (data \u0394=${fmt2(evidence.dataBitsDelta)} bits)`;
+      return {
+        ...base,
+        promote: false,
+        reason: `${code}: total \u0394=${fmt2(evidence.deltaBits)} bits does not clear the ${fmt2(this.marginBits)}-bit margin \u2014 ${why}`,
+        rejectionCode: code
+      };
+    }
+    return {
+      ...base,
+      promote: true,
+      reason: `promote: total \u0394=${fmt2(evidence.deltaBits)} bits (model \u0394=${fmt2(evidence.modelBitsDelta)}, data \u0394=${fmt2(evidence.dataBitsDelta)}) over ${shared.length} tasks`,
+      rejectionCode: null
+    };
+  }
+};
+function inferCandidateId(runs, baselineKey) {
+  for (const run of runs)
+    if (run.candidateId && run.candidateId !== baselineKey) return run.candidateId;
+  return runs[0]?.candidateId ?? "(unknown candidate)";
+}
+
 // src/discover-personas.ts
 import { promises as fs } from "fs";
 import { basename, extname, join as join3 } from "path";
@@ -7240,7 +7460,7 @@ var HeldOutGate = class {
    *  the candidate run with the matching baseline run. Pairs without
    *  a holdout score on both sides are dropped. */
   evaluate(candidate, baseline) {
-    const candidateId = inferCandidateId(candidate, this.baselineKey);
+    const candidateId = inferCandidateId2(candidate, this.baselineKey);
     const baselineId = this.baselineKey;
     const baselineHoldoutByKey = indexHoldoutByKey(baseline);
     const beforeHoldout = [];
@@ -7343,7 +7563,7 @@ var HeldOutGate = class {
     };
   }
 };
-function inferCandidateId(candidate, baselineKey) {
+function inferCandidateId2(candidate, baselineKey) {
   for (const run of candidate) {
     if (run.candidateId && run.candidateId !== baselineKey) return run.candidateId;
   }
@@ -8314,6 +8534,7 @@ export {
   DEFAULT_TRACE_ANALYST_BUDGETS,
   DEFAULT_TRACE_ANALYST_KINDS,
   Dataset,
+  DescriptionLengthGate,
   DockerSandboxDriver,
   DualAgentBench,
   ERROR_COUNT_PATTERNS,
@@ -8342,6 +8563,7 @@ export {
   LockedJsonlAppender,
   MODEL_PRICING,
   MetricsCollector,
+  ModelsUnreachableError,
   MultiLayerVerifier,
   Mutex,
   NoopRawProviderSink,
@@ -8401,6 +8623,7 @@ export {
   asString,
   assertCrossFamily,
   assertLlmRoute,
+  assertModelsServed,
   assertRealBackend,
   assertReleaseConfidence,
   assertRunAgentProfileCell,
@@ -8485,6 +8708,7 @@ export {
   createTraceAnalystKind,
   crossTraceDiff,
   crowdingDistance,
+  dataDescriptionBits,
   decideNextUserTurn,
   decideReferenceReplayPromotion,
   decideReferenceReplayRunPromotion,
@@ -8591,6 +8815,7 @@ export {
   matchGoldens,
   mergeLayerResults,
   mergeSteeringBundle,
+  modelDescriptionBits,
   multiToolchainLayer,
   nistAiRmfReport,
   normalizeScores,
@@ -8613,16 +8838,19 @@ export {
   parseGoldJsonl,
   parseReflectionResponse,
   parseRunRecordSafe,
+  parseRuntimeTrajectoryHookEvent,
   partialCredit,
   passOrthogonality,
   pixelDeltaRatio,
   planTraceInsightQuestions,
   politenessPrefixMutator,
   positionalBias,
+  preflightModels,
   printDriverSummary,
   probeLlm,
   profile_exports as profile,
   projectOtlpFlatLine,
+  projectRuntimeTrajectoryEvidence,
   promptBisect,
   proposeAutomatedPullRequest,
   proposeSynthesisTargets,