@tangle-network/agent-eval 0.58.1 → 0.58.2

package/dist/index.js

@@ -1,10 +1,13 @@
 import {
+  BackendIntegrityError,
   HoldoutAuditor,
+  assertRealBackend,
   canaryLeakView,
   checkBehavioralCanary,
   checkCanaries,
-  runBehavioralCanaries
-} from "./chunk-SHTXZ4O2.js";
+  runBehavioralCanaries,
+  summarizeBackendIntegrity
+} from "./chunk-ZWEQJIM6.js";
 import {
   DEFAULT_MUTATION_PRIMITIVES,
   DEFAULT_RED_TEAM_CORPUS,
@@ -3532,107 +3535,6 @@ function canonicalize2(value) {
   return out;
 }
 
-// src/integrity/backend-integrity.ts
-var BackendIntegrityError = class extends AgentEvalError {
-  constructor(message, report) {
-    super("backend_integrity", message);
-    this.report = report;
-  }
-  report;
-};
-function isStubRecord(rec) {
-  return rec.tokenUsage.input === 0 && rec.tokenUsage.output === 0;
-}
-function isUncostedRecord(rec) {
-  return rec.tokenUsage.output > 0 && rec.costUsd === 0;
-}
-function summarizeBackendIntegrity(records) {
-  const totalRecords = records.length;
-  let stubRecords = 0;
-  let realRecords = 0;
-  let uncostedRecords = 0;
-  let totalInputTokens = 0;
-  let totalOutputTokens = 0;
-  let totalCostUsd = 0;
-  for (const rec of records) {
-    totalInputTokens += rec.tokenUsage.input;
-    totalOutputTokens += rec.tokenUsage.output;
-    totalCostUsd += rec.costUsd;
-    if (isStubRecord(rec)) stubRecords++;
-    else realRecords++;
-    if (isUncostedRecord(rec)) uncostedRecords++;
-  }
-  const verdict = totalRecords === 0 ? "stub" : stubRecords === totalRecords ? "stub" : stubRecords === 0 ? "real" : "mixed";
-  const diagnosis = buildDiagnosis({
-    totalRecords,
-    stubRecords,
-    realRecords,
-    uncostedRecords,
-    totalInputTokens,
-    totalOutputTokens,
-    totalCostUsd,
-    verdict
-  });
-  return {
-    totalRecords,
-    stubRecords,
-    realRecords,
-    uncostedRecords,
-    totalInputTokens,
-    totalOutputTokens,
-    totalCostUsd,
-    verdict,
-    diagnosis
-  };
-}
-function buildDiagnosis(r) {
-  if (r.totalRecords === 0) {
-    return "no records \u2014 eval produced zero runs; backend likely failed before first turn";
-  }
-  if (r.verdict === "stub") {
-    return [
-      `all ${r.totalRecords} records have zero token usage \u2014 the LLM backend was never called.`,
-      "common causes: --backend sandbox without a sandbox bridge running; stub model returning hard-coded strings;",
-      "auth misconfigured so requests were silently dropped before the LLM. Re-run with --backend tcloud and TANGLE_API_KEY set,",
-      "or boot the cli-bridge / sandbox before invoking the eval."
-    ].join(" ");
-  }
-  if (r.verdict === "mixed") {
-    const pct = (r.stubRecords / r.totalRecords * 100).toFixed(0);
-    return [
-      `${r.stubRecords}/${r.totalRecords} records (${pct}%) have zero token usage \u2014 the backend partially failed.`,
-      "common causes: rate-limit cascade (429s after the first N personas);",
-      "transient auth expiry mid-run; provider outage. Treat the affected records as missing data, not agent failures."
-    ].join(" ");
-  }
-  if (r.uncostedRecords > 0) {
-    const pct = (r.uncostedRecords / r.totalRecords * 100).toFixed(0);
-    return [
-      `${r.totalRecords} records with real LLM activity (in=${r.totalInputTokens}, out=${r.totalOutputTokens} tokens).`,
-      `${r.uncostedRecords} (${pct}%) have output tokens but costUsd=0 \u2014 cost ledger is mis-wired (no input-token`,
-      "propagation from the runtime stream into RunRecord)."
-    ].join(" ");
-  }
-  return `${r.totalRecords} records with real LLM activity (in=${r.totalInputTokens}, out=${r.totalOutputTokens} tokens, $${r.totalCostUsd.toFixed(4)}).`;
-}
-function assertRealBackend(records, opts = {}) {
-  const report = summarizeBackendIntegrity(records);
-  const allowMixed = opts.allowMixed ?? true;
-  if (report.verdict === "stub") {
-    throw new BackendIntegrityError(
-      `backend-integrity: ran against a stub or unconfigured backend \u2014 ${report.diagnosis}`,
-      report
-    );
-  }
-  if (!allowMixed && report.verdict === "mixed") {
-    throw new BackendIntegrityError(
-      `backend-integrity: partial backend failure rejected \u2014 ${report.diagnosis}`,
-      report
-    );
-  }
-  return report;
-}
-
 // src/integrity/single-backend.ts
 var SingleBackendError = class extends AgentEvalError {
   constructor(message, report) {