llm-output-guard 0.3.0 → 0.4.0

package/README.md

@@ -215,6 +215,50 @@ They are starting points calibrated against the fixture corpus in this repo —
 
 ---
 
+## Calibrating against your own traffic
+
+The shipped presets are tuned on the fixture corpus, which is not your traffic.
+Log your scores for a week, then let the CLI read them back:
+
+```bash
+npx llm-output-guard calibrate scores.jsonl
+# or:  cat scores.jsonl | npx llm-output-guard calibrate --fpr 0.001
+```
+
+```
+8,000 verdicts — flagging budget 0.10% of traffic
+! sample is too small for a 0.10% rate: it rests on the top ~8 scores, and
+  ~10,000 verdicts are needed before that tail means anything
+
+REPETITION   n=7,993
+  p50 0.000   p90 0.000   p99 0.100   p99.9 0.944   max 0.991
+  gap 0.114 -> 0.705  (15 above, 0.19% of traffic)
+  suggest maxRepetition: 0.409
+
+TAIL_LOOP   n=7,993
+  p50 0.000   p90 0.000   p99 0.000   p99.9 0.000   max 0.789
+  gap 0.000 -> 0.789  (1 above, 0.01% of traffic)
+  suggest maxTailLoop: 0.394
+    ! the separation rests on 1 sample; treat it as a lead to confirm, not a
+      calibrated threshold
+```
+
+Input is JSONL and the parsing is deliberately forgiving — a bare scores
+object, a whole `Verdict`, or either of those buried in a wider log record all
+work, because a calibration step you have to reshape your logs for is one you
+will not run. `--json` emits the same analysis as data.
+
+**What it can and cannot tell you.** The corpus can compute a real margin
+because every fixture is labelled. Your logs are not, and no arithmetic
+recovers a label that was never written down. So these numbers bound *false
+positives* — how much of your own traffic a threshold would flag — on the
+assumption that degeneration is rare in it. They say nothing about what a
+threshold catches; a detector that never fires has a perfect false-positive
+rate. The `gap` line is the exception worth trusting, because a hole between
+the bulk and a cluster of outliers is real separation observed in your data
+rather than an assumption about rarity — and when that hole rests on one or
+two samples, the report says so.
+
 ## On thresholds
 
 A miss is annoying. **A false positive is worse**: a healthy response gets discarded and retried against a slower provider for nothing.

package/dist/ai-sdk.d.cts

@@ -1,4 +1,4 @@
-import { S as StreamGuardOptions, V as Verdict } from './stream-4a1hXc1m.cjs';
+import { S as StreamGuardOptions, V as Verdict } from './stream-mT0r7Yhq.cjs';
 
 /**
  * Middleware adapter for the Vercel AI SDK.

package/dist/ai-sdk.d.ts

@@ -1,4 +1,4 @@
-import { S as StreamGuardOptions, V as Verdict } from './stream-4a1hXc1m.js';
+import { S as StreamGuardOptions, V as Verdict } from './stream-mT0r7Yhq.js';
 
 /**
  * Middleware adapter for the Vercel AI SDK.

package/dist/cli.cjs

@@ -0,0 +1,285 @@
+#!/usr/bin/env node
+'use strict';
+
+var fs = require('fs');
+
+// src/calibrate.ts
+function percentile(sorted, p) {
+  if (sorted.length === 0) return NaN;
+  if (sorted.length === 1) return sorted[0];
+  const rank = (sorted.length - 1) * p;
+  const low = Math.floor(rank);
+  const high = Math.ceil(rank);
+  if (low === high) return sorted[low];
+  return sorted[low] + (sorted[high] - sorted[low]) * (rank - low);
+}
+function findGap(sorted, minWidth = 0.15) {
+  if (sorted.length < 20) return null;
+  const start = Math.floor(sorted.length * 0.5);
+  let best = null;
+  for (let i = start; i < sorted.length - 1; i += 1) {
+    const width = sorted[i + 1] - sorted[i];
+    if (width < minWidth || best && width <= best.above - best.below) continue;
+    const count = sorted.length - (i + 1);
+    best = {
+      below: sorted[i],
+      above: sorted[i + 1],
+      count,
+      share: count / sorted.length
+    };
+  }
+  return best;
+}
+function summarise(code, scores, options = {}) {
+  const { falsePositiveRate = 1e-3 } = options;
+  const sorted = [...scores].sort((a, b) => a - b);
+  const n = sorted.length;
+  const distribution = {
+    n,
+    nonZero: sorted.filter((s) => s > 0).length,
+    min: sorted[0] ?? NaN,
+    max: sorted[n - 1] ?? NaN,
+    p50: percentile(sorted, 0.5),
+    p90: percentile(sorted, 0.9),
+    p99: percentile(sorted, 0.99),
+    p999: percentile(sorted, 0.999)
+  };
+  const caveats = [];
+  const gap = findGap(sorted);
+  const tailSamples = n * falsePositiveRate;
+  if (tailSamples < 10) {
+    caveats.push(
+      `sample is too small for a ${(falsePositiveRate * 100).toFixed(2)}% rate: it rests on the top ~${tailSamples.toFixed(0)} scores, and ~${Math.ceil(10 / falsePositiveRate).toLocaleString()} verdicts are needed before that tail means anything`
+    );
+  }
+  if (distribution.max === 0) {
+    caveats.push("every score is 0 -- this detector never moved on your traffic, so there is nothing to calibrate");
+  } else if (distribution.p50 === distribution.max) {
+    caveats.push("the distribution is a single value; a threshold from it describes nothing");
+  }
+  let suggested;
+  if (gap) {
+    suggested = (gap.below + gap.above) / 2;
+    if (gap.count < 5) {
+      caveats.push(
+        `the separation rests on ${gap.count} sample${gap.count === 1 ? "" : "s"}; treat it as a lead to confirm, not a calibrated threshold`
+      );
+    }
+  } else {
+    suggested = percentile(sorted, 1 - falsePositiveRate);
+    caveats.push(
+      "no clean separation in the tail, so this is a false-positive budget rather than a detection threshold"
+    );
+  }
+  return { code, distribution, suggested: Number(suggested.toFixed(3)), gap, caveats };
+}
+function calibrate(samples, options = {}) {
+  const byCode = /* @__PURE__ */ new Map();
+  for (const sample of samples) {
+    for (const [code, score] of Object.entries(sample)) {
+      if (typeof score !== "number" || !Number.isFinite(score)) continue;
+      const list = byCode.get(code);
+      if (list) list.push(score);
+      else byCode.set(code, [score]);
+    }
+  }
+  const summaries = [...byCode.entries()].map(([code, scores]) => summarise(code, scores, options)).sort((a, b) => b.distribution.n - a.distribution.n);
+  return { n: samples.length, summaries };
+}
+
+// src/cli.ts
+var CODES = [
+  "EMPTY",
+  "TOO_SHORT",
+  "REPETITION",
+  "TAIL_LOOP",
+  "LOW_ENTROPY",
+  "TRUNCATED",
+  "INVALID_JSON",
+  "LANG_MISMATCH"
+];
+var OPTION_FOR = {
+  REPETITION: "maxRepetition",
+  TAIL_LOOP: "maxTailLoop",
+  LOW_ENTROPY: "maxCompressibility",
+  TRUNCATED: "maxTruncation",
+  LANG_MISMATCH: "maxLangMismatch"
+};
+var RATE_ONLY = {
+  EMPTY: "not configurable \u2014 this is how often you served nothing at all",
+  TOO_SHORT: "set by minLength, a character count, which a 0..1 score cannot suggest"
+};
+function extractScores(value) {
+  if (!value || typeof value !== "object") return null;
+  const record = value;
+  for (const nested of [record.scores, record.verdict, record.guard]) {
+    const found = nested ? extractScores(nested) : null;
+    if (found) return found;
+  }
+  const sample = {};
+  let hits = 0;
+  for (const code of CODES) {
+    const score = record[code];
+    if (typeof score === "number" && Number.isFinite(score)) {
+      sample[code] = score;
+      hits += 1;
+    }
+  }
+  return hits > 0 ? sample : null;
+}
+function parseLines(text) {
+  const samples = [];
+  let skipped = 0;
+  for (const line of text.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    try {
+      const scores = extractScores(JSON.parse(trimmed));
+      if (scores) samples.push(scores);
+      else skipped += 1;
+    } catch {
+      skipped += 1;
+    }
+  }
+  return { samples, skipped };
+}
+function readStdin() {
+  return new Promise((resolve, reject) => {
+    let data = "";
+    process.stdin.setEncoding("utf8");
+    process.stdin.on("data", (chunk) => data += chunk);
+    process.stdin.on("end", () => resolve(data));
+    process.stdin.on("error", reject);
+  });
+}
+var USAGE = `
+llm-output-guard calibrate \u2014 derive thresholds from your own logged scores
+
+  npx llm-output-guard calibrate scores.jsonl
+  cat scores.jsonl | npx llm-output-guard calibrate
+
+Options
+  --fpr <rate>   share of traffic you accept flagging (default 0.001)
+  --json         emit the calibration as JSON instead of a report
+
+Input is JSONL, one logged verdict per line. A bare scores object, a whole
+Verdict, or a wider log record containing either all work:
+
+  {"REPETITION":0.03,"TAIL_LOOP":0}
+  {"ok":true,"scores":{"REPETITION":0.03},"reasons":[]}
+  {"msg":"reply","verdict":{"scores":{"REPETITION":0.03}}}
+
+Log them with onVerdict:
+
+  outputGuard({ ...presets.chat, onDegenerate: 'ignore',
+                onVerdict: (v) => log.info({ scores: v.scores }) })
+`;
+var fmt = (n) => Number.isFinite(n) ? n.toFixed(3) : "  -  ";
+function report(text, fpr, asJson) {
+  const { samples, skipped } = parseLines(text);
+  if (samples.length === 0) {
+    process.stderr.write(
+      `No scores found${skipped ? ` (${skipped} lines had none)` : ""}.
+Expected JSONL with reason codes such as REPETITION or TAIL_LOOP.
+`
+    );
+    return 1;
+  }
+  const result = calibrate(samples, { falsePositiveRate: fpr });
+  if (asJson) {
+    process.stdout.write(`${JSON.stringify(result, null, 2)}
+`);
+    return 0;
+  }
+  const out = [
+    "",
+    `${result.n.toLocaleString()} verdicts` + (skipped ? `, ${skipped.toLocaleString()} lines skipped` : "") + ` \u2014 flagging budget ${(fpr * 100).toFixed(2)}% of traffic`
+  ];
+  const shared = result.summaries[0].caveats.filter(
+    (c) => result.summaries.every((s) => s.caveats.includes(c))
+  );
+  for (const caveat of shared) out.push(`! ${caveat}`);
+  for (const s of result.summaries) {
+    const d = s.distribution;
+    out.push("", `${s.code}   n=${d.n.toLocaleString()}`);
+    out.push(
+      `  p50 ${fmt(d.p50)}   p90 ${fmt(d.p90)}   p99 ${fmt(d.p99)}   p99.9 ${fmt(d.p999)}   max ${fmt(d.max)}`
+    );
+    if (s.gap) {
+      out.push(
+        `  gap ${fmt(s.gap.below)} -> ${fmt(s.gap.above)}  (${s.gap.count.toLocaleString()} above, ${(s.gap.share * 100).toFixed(2)}% of traffic)`
+      );
+    }
+    const rateOnly = RATE_ONLY[s.code];
+    if (rateOnly) {
+      out.push(
+        `  fired on ${d.nonZero.toLocaleString()} of ${d.n.toLocaleString()} (${(d.nonZero / d.n * 100).toFixed(2)}%) \u2014 ${rateOnly}`
+      );
+    } else {
+      const option = OPTION_FOR[s.code];
+      out.push(`  suggest ${option ? `${option}: ` : ""}${fmt(s.suggested)}`);
+    }
+    for (const caveat of s.caveats) {
+      if (!shared.includes(caveat)) out.push(`    ! ${caveat}`);
+    }
+  }
+  out.push(
+    "",
+    "These thresholds bound FALSE POSITIVES, not misses. They describe the shape",
+    "of your traffic on the assumption that degeneration is rare in it. Nothing",
+    "here shows a threshold catches anything \u2014 that needs labelled samples, which",
+    "a log does not have. A `gap` line is the exception worth trusting: it is real",
+    "separation observed in your own data.",
+    ""
+  );
+  process.stdout.write(`${out.join("\n")}
+`);
+  return 0;
+}
+async function main(argv) {
+  const args = argv.slice(2);
+  if (args.includes("--help") || args.includes("-h") || args.length === 0) {
+    process.stdout.write(USAGE);
+    return args.length === 0 ? 1 : 0;
+  }
+  const command = args[0] === "calibrate" ? args.slice(1) : args;
+  const asJson = command.includes("--json");
+  let fpr = 1e-3;
+  const fprAt = command.indexOf("--fpr");
+  if (fprAt !== -1) {
+    const parsed = Number(command[fprAt + 1]);
+    if (!Number.isFinite(parsed) || parsed <= 0 || parsed >= 1) {
+      process.stderr.write("--fpr expects a rate between 0 and 1, e.g. 0.001\n");
+      return 1;
+    }
+    fpr = parsed;
+  }
+  const file = command.find((arg, i) => !arg.startsWith("--") && command[i - 1] !== "--fpr");
+  let text;
+  try {
+    text = file ? fs.readFileSync(file, "utf8") : await readStdin();
+  } catch (error) {
+    process.stderr.write(`Could not read ${file ?? "stdin"}: ${error.message}
+`);
+    return 1;
+  }
+  return report(text, fpr, asJson);
+}
+var invokedDirectly = process.argv[1] && /llm-output-guard[/\\]dist[/\\]cli|cli\.(ts|js|cjs)$/.test(process.argv[1]);
+if (invokedDirectly) {
+  main(process.argv).then(
+    (code) => {
+      process.exitCode = code;
+    },
+    (error) => {
+      process.stderr.write(`${error.message}
+`);
+      process.exitCode = 1;
+    }
+  );
+}
+
+exports.extractScores = extractScores;
+exports.main = main;
+//# sourceMappingURL=cli.cjs.map
+//# sourceMappingURL=cli.cjs.map

package/dist/cli.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/calibrate.ts","../src/cli.ts"],"names":["readFileSync"],"mappings":";;;;;;AAuEO,SAAS,UAAA,CAAW,QAAkB,CAAA,EAAmB;AAC9D,EAAA,IAAI,MAAA,CAAO,MAAA,KAAW,CAAA,EAAG,OAAO,GAAA;AAChC,EAAA,IAAI,MAAA,CAAO,MAAA,KAAW,CAAA,EAAG,OAAO,OAAO,CAAC,CAAA;AACxC,EAAA,MAAM,IAAA,GAAA,CAAQ,MAAA,CAAO,MAAA,GAAS,CAAA,IAAK,CAAA;AACnC,EAAA,MAAM,GAAA,GAAM,IAAA,CAAK,KAAA,CAAM,IAAI,CAAA;AAC3B,EAAA,MAAM,IAAA,GAAO,IAAA,CAAK,IAAA,CAAK,IAAI,CAAA;AAC3B,EAAA,IAAI,GAAA,KAAQ,IAAA,EAAM,OAAO,MAAA,CAAO,GAAG,CAAA;AACnC,EAAA,OAAO,MAAA,CAAO,GAAG,CAAA,GAAA,CAAK,MAAA,CAAO,IAAI,CAAA,GAAI,MAAA,CAAO,GAAG,CAAA,KAAM,IAAA,GAAO,GAAA,CAAA;AAC9D;AAYO,SAAS,OAAA,CAAQ,MAAA,EAAkB,QAAA,GAAW,IAAA,EAAkB;AACrE,EAAA,IAAI,MAAA,CAAO,MAAA,GAAS,EAAA,EAAI,OAAO,IAAA;AAE/B,EAAA,MAAM,KAAA,GAAQ,IAAA,CAAK,KAAA,CAAM,MAAA,CAAO,SAAS,GAAG,CAAA;AAC5C,EAAA,IAAI,IAAA,GAAmB,IAAA;AAEvB,EAAA,KAAA,IAAS,IAAI,KAAA,EAAO,CAAA,GAAI,OAAO,MAAA,GAAS,CAAA,EAAG,KAAK,CAAA,EAAG;AACjD,IAAA,MAAM,QAAQ,MAAA,CAAO,CAAA,GAAI,CAAC,CAAA,GAAI,OAAO,CAAC,CAAA;AACtC,IAAA,IAAI,QAAQ,QAAA,IAAa,IAAA,IAAQ,SAAS,IAAA,CAAK,KAAA,GAAQ,KAAK,KAAA,EAAQ;AACpE,IAAA,MAAM,KAAA,GAAQ,MAAA,CAAO,MAAA,IAAU,CAAA,GAAI,CAAA,CAAA;AACnC,IAAA,IAAA,GAAO;AAAA,MACL,KAAA,EAAO,OAAO,CAAC,CAAA;AAAA,MACf,KAAA,EAAO,MAAA,CAAO,CAAA,GAAI,CAAC,CAAA;AAAA,MACnB,KAAA;AAAA,MACA,KAAA,EAAO,QAAQ,MAAA,CAAO;AAAA,KACxB;AAAA,EACF;AAEA,EAAA,OAAO,IAAA;AACT;AASO,SAAS,SAAA,CACd,IAAA,EACA,MAAA,EACA,OAAA,GAA8B,EAAC,EACtB;AACT,EAAA,MAAM,EAAE,iBAAA,GAAoB,IAAA,EAAM,GAAI,OAAA;AACtC,EAAA,MAAM,MAAA,GAAS,CAAC,GAAG,MAAM,CAAA,CAAE,KAAK,CAAC,CAAA,EAAG,CAAA,KAAM,CAAA,GAAI,CAAC,CAAA;AAC/C,EAAA,MAAM,IAAI,MAAA,CAAO,MAAA;AAEjB,EAAA,MAAM,YAAA,GAA6B;AAAA,IACjC,CAAA;AAAA,IACA,SAAS,MAAA,CAAO,MAAA,CAAO,CAAC,CAAA,KAAM,CAAA,GAAI,CAAC,CAAA,CAAE,MAAA;AAAA,IACrC,GAAA,EAAK,MAAA,CAAO,CAAC,CAAA,IAAK,GAAA;AAAA,IAClB,GAAA,EAAK,MAAA,CAAO,CAAA,GAAI,CAAC,CAAA,IAAK,GAAA;AAAA,IACtB,GAAA,EAAK,UAAA,CAAW,MAAA,EAAQ,GAAG,CAAA;AAAA,IAC3B,GAAA,EAAK,UAAA,CAAW,MAAA,EAAQ,GAAG,CAAA;AAAA,IAC3B,GAAA,EAAK,UAAA,CAAW,MAAA,EAAQ,IAAI,CAAA;AAAA,IAC5B,IAAA,EAAM,UAAA,CAAW,MAAA,EAAQ,KAAK;AAAA,GAChC;AAEA,EAAA,MAAM,UAAoB,EAAC;AAC3B,EAAA,MAAM,GAAA,GAAM,QAAQ,MAAM,CAAA;AAQ1B,EAAA,MAAM,cAAc,CAAA,GAAI,iBAAA;AACxB,EAAA,IAAI,cAAc,EAAA,EAAI;AAMpB,IAAA,OAAA,CAAQ,IAAA;AAAA,MACN,8BAA8B,iBAAA,GAAoB,GAAA,EAAK,QAAQ,CAAC,CAAC,gCACvC,WAAA,CAAY,OAAA,CAAQ,CAAC,CAAC,iBAC1C,IAAA,CAAK,IAAA,CAAK,KAAK,iBAAiB,CAAA,CAAE,gBAAgB,CAAA,oDAAA;AAAA,KAC1D;AAAA,EACF;AAEA,EAAA,IAAI,YAAA,CAAa,QAAQ,CAAA,EAAG;AAC1B,IAAA,OAAA,CAAQ,KAAK,iGAAiG,CAAA;AAAA,EAChH,CAAA,MAAA,IAAW,YAAA,CAAa,GAAA,KAAQ,YAAA,CAAa,GAAA,EAAK;AAChD,IAAA,OAAA,CAAQ,KAAK,2EAA2E,CAAA;AAAA,EAC1F;AAEA,EAAA,IAAI,SAAA;AACJ,EAAA,IAAI,GAAA,EAAK;AAEP,IAAA,SAAA,GAAA,CAAa,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,KAAA,IAAS,CAAA;AAStC,IAAA,IAAI,GAAA,CAAI,QAAQ,CAAA,EAAG;AACjB,MAAA,OAAA,CAAQ,IAAA;AAAA,QACN,CAAA,wBAAA,EAA2B,IAAI,KAAK,CAAA,OAAA,EAAU,IAAI,KAAA,KAAU,CAAA,GAAI,KAAK,GAAG,CAAA,2DAAA;AAAA,OAE1E;AAAA,IACF;AAAA,EACF,CAAA,MAAO;AACL,IAAA,SAAA,GAAY,UAAA,CAAW,MAAA,EAAQ,CAAA,GAAI,iBAAiB,CAAA;AACpD,IAAA,OAAA,CAAQ,IAAA;AAAA,MACN;AAAA,KACF;AAAA,EACF;AAEA,EAAA,OAAO,EAAE,IAAA,EAAM,YAAA,EAAc,SAAA,EAAW,MAAA,CAAO,SAAA,CAAU,OAAA,CAAQ,CAAC,CAAC,CAAA,EAAG,GAAA,EAAK,OAAA,EAAQ;AACrF;AAmBO,SAAS,SAAA,CAAU,OAAA,EAAwB,OAAA,GAA8B,EAAC,EAAgB;AAC/F,EAAA,MAAM,MAAA,uBAAa,GAAA,EAA0B;AAE7C,EAAA,KAAA,MAAW,UAAU,OAAA,EAAS;AAC5B,IAAA,KAAA,MAAW,CAAC,IAAA,EAAM,KAAK,KAAK,MAAA,CAAO,OAAA,CAAQ,MAAM,CAAA,EAA8B;AAC7E,MAAA,IAAI,OAAO,KAAA,KAAU,QAAA,IAAY,CAAC,MAAA,CAAO,QAAA,CAAS,KAAK,CAAA,EAAG;AAC1D,MAAA,MAAM,IAAA,GAAO,MAAA,CAAO,GAAA,CAAI,IAAI,CAAA;AAC5B,MAAA,IAAI,IAAA,EAAM,IAAA,CAAK,IAAA,CAAK,KAAK,CAAA;AAAA,WACpB,MAAA,CAAO,GAAA,CAAI,IAAA,EAAM,CAAC,KAAK,CAAC,CAAA;AAAA,IAC/B;AAAA,EACF;AAEA,EAAA,MAAM,SAAA,GAAY,CAAC,GAAG,MAAA,CAAO,OAAA,EAAS,CAAA,CACnC,GAAA,CAAI,CAAC,CAAC,IAAA,EAAM,MAAM,CAAA,KAAM,SAAA,CAAU,IAAA,EAAM,MAAA,EAAQ,OAAO,CAAC,CAAA,CACxD,IAAA,CAAK,CAAC,CAAA,EAAG,CAAA,KAAM,CAAA,CAAE,YAAA,CAAa,CAAA,GAAI,CAAA,CAAE,YAAA,CAAa,CAAC,CAAA;AAErD,EAAA,OAAO,EAAE,CAAA,EAAG,OAAA,CAAQ,MAAA,EAAQ,SAAA,EAAU;AACxC;;;AC1NA,IAAM,KAAA,GAAsB;AAAA,EAC1B,OAAA;AAAA,EACA,WAAA;AAAA,EACA,YAAA;AAAA,EACA,WAAA;AAAA,EACA,aAAA;AAAA,EACA,WAAA;AAAA,EACA,cAAA;AAAA,EACA;AACF,CAAA;AAYA,IAAM,UAAA,GAAkD;AAAA,EACtD,UAAA,EAAY,eAAA;AAAA,EACZ,SAAA,EAAW,aAAA;AAAA,EACX,WAAA,EAAa,oBAAA;AAAA,EACb,SAAA,EAAW,eAAA;AAAA,EACX,aAAA,EAAe;AACjB,CAAA;AAGA,IAAM,SAAA,GAAiD;AAAA,EACrD,KAAA,EAAO,qEAAA;AAAA,EACP,SAAA,EAAW;AACb,CAAA;AASO,SAAS,cAAc,KAAA,EAAoC;AAChE,EAAA,IAAI,CAAC,KAAA,IAAS,OAAO,KAAA,KAAU,UAAU,OAAO,IAAA;AAEhD,EAAA,MAAM,MAAA,GAAS,KAAA;AACf,EAAA,KAAA,MAAW,MAAA,IAAU,CAAC,MAAA,CAAO,MAAA,EAAQ,OAAO,OAAA,EAAS,MAAA,CAAO,KAAK,CAAA,EAAG;AAClE,IAAA,MAAM,KAAA,GAAQ,MAAA,GAAS,aAAA,CAAc,MAAM,CAAA,GAAI,IAAA;AAC/C,IAAA,IAAI,OAAO,OAAO,KAAA;AAAA,EACpB;AAEA,EAAA,MAAM,SAAsB,EAAC;AAC7B,EAAA,IAAI,IAAA,GAAO,CAAA;AACX,EAAA,KAAA,MAAW,QAAQ,KAAA,EAAO;AACxB,IAAA,MAAM,KAAA,GAAQ,OAAO,IAAI,CAAA;AACzB,IAAA,IAAI,OAAO,KAAA,KAAU,QAAA,IAAY,MAAA,CAAO,QAAA,CAAS,KAAK,CAAA,EAAG;AACvD,MAAA,MAAA,CAAO,IAAI,CAAA,GAAI,KAAA;AACf,MAAA,IAAA,IAAQ,CAAA;AAAA,IACV;AAAA,EACF;AAEA,EAAA,OAAO,IAAA,GAAO,IAAI,MAAA,GAAS,IAAA;AAC7B;AAEA,SAAS,WAAW,IAAA,EAA2D;AAC7E,EAAA,MAAM,UAAyB,EAAC;AAChC,EAAA,IAAI,OAAA,GAAU,CAAA;AAEd,EAAA,KAAA,MAAW,IAAA,IAAQ,IAAA,CAAK,KAAA,CAAM,IAAI,CAAA,EAAG;AACnC,IAAA,MAAM,OAAA,GAAU,KAAK,IAAA,EAAK;AAC1B,IAAA,IAAI,CAAC,OAAA,EAAS;AACd,IAAA,IAAI;AACF,MAAA,MAAM,MAAA,GAAS,aAAA,CAAc,IAAA,CAAK,KAAA,CAAM,OAAO,CAAC,CAAA;AAChD,MAAA,IAAI,MAAA,EAAQ,OAAA,CAAQ,IAAA,CAAK,MAAM,CAAA;AAAA,WAC1B,OAAA,IAAW,CAAA;AAAA,IAClB,CAAA,CAAA,MAAQ;AACN,MAAA,OAAA,IAAW,CAAA;AAAA,IACb;AAAA,EACF;AAEA,EAAA,OAAO,EAAE,SAAS,OAAA,EAAQ;AAC5B;AAEA,SAAS,SAAA,GAA6B;AACpC,EAAA,OAAO,IAAI,OAAA,CAAQ,CAAC,OAAA,EAAS,MAAA,KAAW;AACtC,IAAA,IAAI,IAAA,GAAO,EAAA;AACX,IAAA,OAAA,CAAQ,KAAA,CAAM,YAAY,MAAM,CAAA;AAChC,IAAA,OAAA,CAAQ,MAAM,EAAA,CAAG,MAAA,EAAQ,CAAC,KAAA,KAAW,QAAQ,KAAM,CAAA;AACnD,IAAA,OAAA,CAAQ,MAAM,EAAA,CAAG,KAAA,EAAO,MAAM,OAAA,CAAQ,IAAI,CAAC,CAAA;AAC3C,IAAA,OAAA,CAAQ,KAAA,CAAM,EAAA,CAAG,OAAA,EAAS,MAAM,CAAA;AAAA,EAClC,CAAC,CAAA;AACH;AAEA,IAAM,KAAA,GAAQ;AAAA;;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;;AAAA;AAAA;AAAA,CAAA;AAuBd,IAAM,GAAA,GAAM,CAAC,CAAA,KAAe,MAAA,CAAO,QAAA,CAAS,CAAC,CAAA,GAAI,CAAA,CAAE,OAAA,CAAQ,CAAC,CAAA,GAAI,OAAA;AAEhE,SAAS,MAAA,CAAO,IAAA,EAAc,GAAA,EAAa,MAAA,EAAyB;AAClE,EAAA,MAAM,EAAE,OAAA,EAAS,OAAA,EAAQ,GAAI,WAAW,IAAI,CAAA;AAE5C,EAAA,IAAI,OAAA,CAAQ,WAAW,CAAA,EAAG;AACxB,IAAA,OAAA,CAAQ,MAAA,CAAO,KAAA;AAAA,MACb,CAAA,eAAA,EAAkB,OAAA,GAAU,CAAA,EAAA,EAAK,OAAO,qBAAqB,EAAE,CAAA;AAAA;AAAA;AAAA,KAEjE;AACA,IAAA,OAAO,CAAA;AAAA,EACT;AAEA,EAAA,MAAM,SAAS,SAAA,CAAU,OAAA,EAAS,EAAE,iBAAA,EAAmB,KAAK,CAAA;AAE5D,EAAA,IAAI,MAAA,EAAQ;AACV,IAAA,OAAA,CAAQ,MAAA,CAAO,MAAM,CAAA,EAAG,IAAA,CAAK,UAAU,MAAA,EAAQ,IAAA,EAAM,CAAC,CAAC;AAAA,CAAI,CAAA;AAC3D,IAAA,OAAO,CAAA;AAAA,EACT;AAEA,EAAA,MAAM,GAAA,GAAgB;AAAA,IACpB,EAAA;AAAA,IACA,GAAG,MAAA,CAAO,CAAA,CAAE,gBAAgB,CAAA,SAAA,CAAA,IACzB,UAAU,CAAA,EAAA,EAAK,OAAA,CAAQ,cAAA,EAAgB,mBAAmB,EAAA,CAAA,GAC3D,CAAA,wBAAA,EAAA,CAAuB,MAAM,GAAA,EAAK,OAAA,CAAQ,CAAC,CAAC,CAAA,YAAA;AAAA,GAChD;AAOA,EAAA,MAAM,MAAA,GAAS,MAAA,CAAO,SAAA,CAAU,CAAC,EAAE,OAAA,CAAQ,MAAA;AAAA,IAAO,CAAC,CAAA,KACjD,MAAA,CAAO,SAAA,CAAU,KAAA,CAAM,CAAC,CAAA,KAAM,CAAA,CAAE,OAAA,CAAQ,QAAA,CAAS,CAAC,CAAC;AAAA,GACrD;AACA,EAAA,KAAA,MAAW,UAAU,MAAA,EAAQ,GAAA,CAAI,IAAA,CAAK,CAAA,EAAA,EAAK,MAAM,CAAA,CAAE,CAAA;AAEnD,EAAA,KAAA,MAAW,CAAA,IAAK,OAAO,SAAA,EAAW;AAChC,IAAA,MAAM,IAAI,CAAA,CAAE,YAAA;AACZ,IAAA,GAAA,CAAI,IAAA,CAAK,EAAA,EAAI,CAAA,EAAG,CAAA,CAAE,IAAI,QAAQ,CAAA,CAAE,CAAA,CAAE,cAAA,EAAgB,CAAA,CAAE,CAAA;AACpD,IAAA,GAAA,CAAI,IAAA;AAAA,MACF,CAAA,MAAA,EAAS,GAAA,CAAI,CAAA,CAAE,GAAG,CAAC,UAAU,GAAA,CAAI,CAAA,CAAE,GAAG,CAAC,CAAA,OAAA,EAAU,GAAA,CAAI,EAAE,GAAG,CAAC,CAAA,SAAA,EAChD,GAAA,CAAI,CAAA,CAAE,IAAI,CAAC,CAAA,OAAA,EAAU,GAAA,CAAI,CAAA,CAAE,GAAG,CAAC,CAAA;AAAA,KAC5C;AAEA,IAAA,IAAI,EAAE,GAAA,EAAK;AACT,MAAA,GAAA,CAAI,IAAA;AAAA,QACF,CAAA,MAAA,EAAS,GAAA,CAAI,CAAA,CAAE,GAAA,CAAI,KAAK,CAAC,CAAA,IAAA,EAAO,GAAA,CAAI,CAAA,CAAE,GAAA,CAAI,KAAK,CAAC,MAC1C,CAAA,CAAE,GAAA,CAAI,KAAA,CAAM,cAAA,EAAgB,CAAA,QAAA,EAAA,CAAY,CAAA,CAAE,GAAA,CAAI,KAAA,GAAQ,GAAA,EAAK,OAAA,CAAQ,CAAC,CAAC,CAAA,aAAA;AAAA,OAC7E;AAAA,IACF;AAEA,IAAA,MAAM,QAAA,GAAW,SAAA,CAAU,CAAA,CAAE,IAAI,CAAA;AACjC,IAAA,IAAI,QAAA,EAAU;AACZ,MAAA,GAAA,CAAI,IAAA;AAAA,QACF,CAAA,WAAA,EAAc,EAAE,OAAA,CAAQ,cAAA,EAAgB,CAAA,IAAA,EAAO,CAAA,CAAE,EAAE,cAAA,EAAgB,MAC3D,CAAA,CAAE,OAAA,GAAU,EAAE,CAAA,GAAK,GAAA,EAAK,QAAQ,CAAC,CAAC,aAAQ,QAAQ,CAAA;AAAA,OAC5D;AAAA,IACF,CAAA,MAAO;AACL,MAAA,MAAM,MAAA,GAAS,UAAA,CAAW,CAAA,CAAE,IAAI,CAAA;AAChC,MAAA,GAAA,CAAI,IAAA,CAAK,CAAA,UAAA,EAAa,MAAA,GAAS,CAAA,EAAG,MAAM,CAAA,EAAA,CAAA,GAAO,EAAE,CAAA,EAAG,GAAA,CAAI,CAAA,CAAE,SAAS,CAAC,CAAA,CAAE,CAAA;AAAA,IACxE;AAEA,IAAA,KAAA,MAAW,MAAA,IAAU,EAAE,OAAA,EAAS;AAC9B,MAAA,IAAI,CAAC,OAAO,QAAA,CAAS,MAAM,GAAG,GAAA,CAAI,IAAA,CAAK,CAAA,MAAA,EAAS,MAAM,CAAA,CAAE,CAAA;AAAA,IAC1D;AAAA,EACF;AAEA,EAAA,GAAA,CAAI,IAAA;AAAA,IACF,EAAA;AAAA,IACA,6EAAA;AAAA,IACA,4EAAA;AAAA,IACA,mFAAA;AAAA,IACA,+EAAA;AAAA,IACA,uCAAA;AAAA,IACA;AAAA,GACF;AAEA,EAAA,OAAA,CAAQ,OAAO,KAAA,CAAM,CAAA,EAAG,GAAA,CAAI,IAAA,CAAK,IAAI,CAAC;AAAA,CAAI,CAAA;AAC1C,EAAA,OAAO,CAAA;AACT;AAEA,eAAsB,KAAK,IAAA,EAAiC;AAC1D,EAAA,MAAM,IAAA,GAAO,IAAA,CAAK,KAAA,CAAM,CAAC,CAAA;AAEzB,EAAA,IAAI,IAAA,CAAK,QAAA,CAAS,QAAQ,CAAA,IAAK,IAAA,CAAK,SAAS,IAAI,CAAA,IAAK,IAAA,CAAK,MAAA,KAAW,CAAA,EAAG;AACvE,IAAA,OAAA,CAAQ,MAAA,CAAO,MAAM,KAAK,CAAA;AAC1B,IAAA,OAAO,IAAA,CAAK,MAAA,KAAW,CAAA,GAAI,CAAA,GAAI,CAAA;AAAA,EACjC;AAEA,EAAA,MAAM,OAAA,GAAU,KAAK,CAAC,CAAA,KAAM,cAAc,IAAA,CAAK,KAAA,CAAM,CAAC,CAAA,GAAI,IAAA;AAC1D,EAAA,MAAM,MAAA,GAAS,OAAA,CAAQ,QAAA,CAAS,QAAQ,CAAA;AAExC,EAAA,IAAI,GAAA,GAAM,IAAA;AACV,EAAA,MAAM,KAAA,GAAQ,OAAA,CAAQ,OAAA,CAAQ,OAAO,CAAA;AACrC,EAAA,IAAI,UAAU,EAAA,EAAI;AAChB,IAAA,MAAM,MAAA,GAAS,MAAA,CAAO,OAAA,CAAQ,KAAA,GAAQ,CAAC,CAAC,CAAA;AACxC,IAAA,IAAI,CAAC,OAAO,QAAA,CAAS,MAAM,KAAK,MAAA,IAAU,CAAA,IAAK,UAAU,CAAA,EAAG;AAC1D,MAAA,OAAA,CAAQ,MAAA,CAAO,MAAM,oDAAoD,CAAA;AACzE,MAAA,OAAO,CAAA;AAAA,IACT;AACA,IAAA,GAAA,GAAM,MAAA;AAAA,EACR;AAEA,EAAA,MAAM,IAAA,GAAO,OAAA,CAAQ,IAAA,CAAK,CAAC,KAAK,CAAA,KAAM,CAAC,GAAA,CAAI,UAAA,CAAW,IAAI,CAAA,IAAK,OAAA,CAAQ,CAAA,GAAI,CAAC,MAAM,OAAO,CAAA;AAEzF,EAAA,IAAI,IAAA;AACJ,EAAA,IAAI;AACF,IAAA,IAAA,GAAO,OAAOA,eAAA,CAAa,IAAA,EAAM,MAAM,CAAA,GAAI,MAAM,SAAA,EAAU;AAAA,EAC7D,SAAS,KAAA,EAAO;AACd,IAAA,OAAA,CAAQ,OAAO,KAAA,CAAM,CAAA,eAAA,EAAkB,QAAQ,OAAO,CAAA,EAAA,EAAM,MAAgB,OAAO;AAAA,CAAI,CAAA;AACvF,IAAA,OAAO,CAAA;AAAA,EACT;AAEA,EAAA,OAAO,MAAA,CAAO,IAAA,EAAM,GAAA,EAAK,MAAM,CAAA;AACjC;AAGA,IAAM,eAAA,GACJ,OAAA,CAAQ,IAAA,CAAK,CAAC,CAAA,IAAK,sDAAsD,IAAA,CAAK,OAAA,CAAQ,IAAA,CAAK,CAAC,CAAC,CAAA;AAE/F,IAAI,eAAA,EAAiB;AACnB,EAAA,IAAA,CAAK,OAAA,CAAQ,IAAI,CAAA,CAAE,IAAA;AAAA,IACjB,CAAC,IAAA,KAAS;AACR,MAAA,OAAA,CAAQ,QAAA,GAAW,IAAA;AAAA,IACrB,CAAA;AAAA,IACA,CAAC,KAAA,KAAU;AACT,MAAA,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,CAAA,EAAI,KAAA,CAAgB,OAAO;AAAA,CAAI,CAAA;AACpD,MAAA,OAAA,CAAQ,QAAA,GAAW,CAAA;AAAA,IACrB;AAAA,GACF;AACF","file":"cli.cjs","sourcesContent":["/**\n * Threshold calibration from your own logged scores.\n *\n * The fixture corpus can compute a real margin because every sample is\n * labelled. Production logs are not labelled, and no amount of arithmetic\n * recovers a label that was never recorded -- so this deliberately answers a\n * smaller question than `scripts/calibrate.ts` does.\n *\n * What it can answer: *how much of your own traffic would a given threshold\n * flag?* That works because degeneration is rare, which puts healthy output in\n * the bulk of the distribution and makes a high percentile a decent stand-in\n * for \"the most extreme thing my healthy traffic does\". Set a threshold above\n * it and you know your false-positive cost.\n *\n * What it cannot answer: whether that threshold catches anything. A detector\n * that never fires has a perfect false-positive rate. Every number here bounds\n * false positives only, and `Summary.caveats` says so wherever it matters.\n */\nimport type { ReasonCode } from './types.js';\n\nexport interface CalibrationOptions {\n  /**\n   * Share of your traffic you are willing to have flagged. Default 0.001.\n   *\n   * This is the real knob: a threshold is only ever a trade between the\n   * responses you discard wrongly and the ones you let through.\n   */\n  falsePositiveRate?: number;\n}\n\nexport interface Distribution {\n  n: number;\n  /**\n   * How many scores were above zero.\n   *\n   * For the detectors whose threshold is not a 0..1 score -- `EMPTY`, which is\n   * not configurable, and `TOO_SHORT`, which is set in characters -- this is\n   * the whole useful answer: how often the thing happened at all.\n   */\n  nonZero: number;\n  min: number;\n  max: number;\n  p50: number;\n  p90: number;\n  p99: number;\n  p999: number;\n}\n\nexport interface Gap {\n  /** Highest score below the gap -- the top of the bulk. */\n  below: number;\n  /** Lowest score above it -- the bottom of the outlier cluster. */\n  above: number;\n  /** How many samples sit above the gap. */\n  count: number;\n  /** Their share of the sample. */\n  share: number;\n}\n\nexport interface Summary {\n  code: ReasonCode;\n  distribution: Distribution;\n  /** Threshold that would flag `falsePositiveRate` of this sample. */\n  suggested: number;\n  /** A clean separation in the upper tail, when one exists. Stronger evidence. */\n  gap: Gap | null;\n  /** Everything that would make the number above untrustworthy. */\n  caveats: string[];\n}\n\n/** Linear-interpolated percentile. `sorted` must be ascending. */\nexport function percentile(sorted: number[], p: number): number {\n  if (sorted.length === 0) return NaN;\n  if (sorted.length === 1) return sorted[0];\n  const rank = (sorted.length - 1) * p;\n  const low = Math.floor(rank);\n  const high = Math.ceil(rank);\n  if (low === high) return sorted[low];\n  return sorted[low] + (sorted[high] - sorted[low]) * (rank - low);\n}\n\n/**\n * The widest empty stretch in the upper tail, if there is one.\n *\n * A genuinely bimodal detector -- healthy output clustered near zero, a\n * handful of failures far above it -- leaves a visible hole between the two.\n * That hole is worth far more than a percentile, because it is evidence about\n * *this* detector on *your* traffic rather than an assumption about rarity.\n * Searching only above the median keeps the ordinary spread of healthy scores\n * from being mistaken for a separation.\n */\nexport function findGap(sorted: number[], minWidth = 0.15): Gap | null {\n  if (sorted.length < 20) return null;\n\n  const start = Math.floor(sorted.length * 0.5);\n  let best: Gap | null = null;\n\n  for (let i = start; i < sorted.length - 1; i += 1) {\n    const width = sorted[i + 1] - sorted[i];\n    if (width < minWidth || (best && width <= best.above - best.below)) continue;\n    const count = sorted.length - (i + 1);\n    best = {\n      below: sorted[i],\n      above: sorted[i + 1],\n      count,\n      share: count / sorted.length,\n    };\n  }\n\n  return best;\n}\n\n/**\n * Summarise one detector's scores.\n *\n * The suggestion prefers a gap when one exists and falls back to the\n * percentile otherwise, because the two are not equally good evidence and\n * pretending they are is how a calibrated-looking number gets trusted.\n */\nexport function summarise(\n  code: ReasonCode,\n  scores: number[],\n  options: CalibrationOptions = {},\n): Summary {\n  const { falsePositiveRate = 0.001 } = options;\n  const sorted = [...scores].sort((a, b) => a - b);\n  const n = sorted.length;\n\n  const distribution: Distribution = {\n    n,\n    nonZero: sorted.filter((s) => s > 0).length,\n    min: sorted[0] ?? NaN,\n    max: sorted[n - 1] ?? NaN,\n    p50: percentile(sorted, 0.5),\n    p90: percentile(sorted, 0.9),\n    p99: percentile(sorted, 0.99),\n    p999: percentile(sorted, 0.999),\n  };\n\n  const caveats: string[] = [];\n  const gap = findGap(sorted);\n\n  /*\n   * A percentile cannot be estimated from samples that are not there. At a\n   * false-positive rate of 0.1% the estimate rests on the top 0.1% of the\n   * sample, so ten of them is the bare minimum before the number means\n   * anything -- below that it is one unusual response wearing a decimal point.\n   */\n  const tailSamples = n * falsePositiveRate;\n  if (tailSamples < 10) {\n    /*\n     * Phrased without this detector's own `n` so that the identical sentence\n     * comes back for every detector sharing the shortfall, and the reporter\n     * can hoist it to a single line. The sample size is already on the header.\n     */\n    caveats.push(\n      `sample is too small for a ${(falsePositiveRate * 100).toFixed(2)}% rate: ` +\n        `it rests on the top ~${tailSamples.toFixed(0)} scores, and ` +\n        `~${Math.ceil(10 / falsePositiveRate).toLocaleString()} verdicts are needed before that tail means anything`,\n    );\n  }\n\n  if (distribution.max === 0) {\n    caveats.push('every score is 0 -- this detector never moved on your traffic, so there is nothing to calibrate');\n  } else if (distribution.p50 === distribution.max) {\n    caveats.push('the distribution is a single value; a threshold from it describes nothing');\n  }\n\n  let suggested: number;\n  if (gap) {\n    // Midpoint of the hole: as far from the healthy bulk as from the outliers.\n    suggested = (gap.below + gap.above) / 2;\n\n    /*\n     * A hole with one or two things past it is an outlier, not a cluster, and\n     * \"clean separation\" is too strong a word for it. The suggestion is still\n     * the best available -- one real failure is exactly the signal you are\n     * looking for when the failure is rare -- but it rests on that one sample,\n     * and a reader deciding whether to trust it should be told so.\n     */\n    if (gap.count < 5) {\n      caveats.push(\n        `the separation rests on ${gap.count} sample${gap.count === 1 ? '' : 's'}; ` +\n          'treat it as a lead to confirm, not a calibrated threshold',\n      );\n    }\n  } else {\n    suggested = percentile(sorted, 1 - falsePositiveRate);\n    caveats.push(\n      'no clean separation in the tail, so this is a false-positive budget rather than a detection threshold',\n    );\n  }\n\n  return { code, distribution, suggested: Number(suggested.toFixed(3)), gap, caveats };\n}\n\n/** One logged verdict's worth of scores. */\nexport type ScoreSample = Partial<Record<ReasonCode, number>>;\n\nexport interface Calibration {\n  /** How many samples were read. */\n  n: number;\n  summaries: Summary[];\n}\n\n/**\n * Summarise every detector present in the samples.\n *\n * Detectors are summarised independently and only over the samples that\n * actually carry them, because a disabled detector logs nothing and counting\n * that as a zero would drag every percentile down and quietly recommend a\n * threshold far tighter than the traffic supports.\n */\nexport function calibrate(samples: ScoreSample[], options: CalibrationOptions = {}): Calibration {\n  const byCode = new Map<ReasonCode, number[]>();\n\n  for (const sample of samples) {\n    for (const [code, score] of Object.entries(sample) as [ReasonCode, unknown][]) {\n      if (typeof score !== 'number' || !Number.isFinite(score)) continue;\n      const list = byCode.get(code);\n      if (list) list.push(score);\n      else byCode.set(code, [score]);\n    }\n  }\n\n  const summaries = [...byCode.entries()]\n    .map(([code, scores]) => summarise(code, scores, options))\n    .sort((a, b) => b.distribution.n - a.distribution.n);\n\n  return { n: samples.length, summaries };\n}\n","/**\n * `llm-output-guard calibrate` -- turn a week of logged scores into thresholds.\n *\n * Reads JSONL from a file or stdin. Deliberately liberal about shape: the\n * whole point is to accept whatever you already log rather than making you\n * reshape it first, because a calibration step you have to prepare for is one\n * you do not run.\n */\nimport { readFileSync } from 'node:fs';\nimport { calibrate, type ScoreSample } from './calibrate.js';\nimport type { ReasonCode } from './types.js';\n\nconst CODES: ReasonCode[] = [\n  'EMPTY',\n  'TOO_SHORT',\n  'REPETITION',\n  'TAIL_LOOP',\n  'LOW_ENTROPY',\n  'TRUNCATED',\n  'INVALID_JSON',\n  'LANG_MISMATCH',\n];\n\n/**\n * The option each detector's threshold is set with, for the detectors whose\n * threshold is a 0..1 score.\n *\n * `EMPTY` is absent because its threshold is not configurable, and `TOO_SHORT`\n * because `minLength` is a character count -- suggesting a 0..1 score for it\n * would print a confidently wrong number in the right-looking place. Both are\n * still reported below as incidence rates, which is the useful thing to know\n * about them anyway.\n */\nconst OPTION_FOR: Partial<Record<ReasonCode, string>> = {\n  REPETITION: 'maxRepetition',\n  TAIL_LOOP: 'maxTailLoop',\n  LOW_ENTROPY: 'maxCompressibility',\n  TRUNCATED: 'maxTruncation',\n  LANG_MISMATCH: 'maxLangMismatch',\n};\n\n/** Detectors reported by how often they fired rather than by threshold. */\nconst RATE_ONLY: Partial<Record<ReasonCode, string>> = {\n  EMPTY: 'not configurable — this is how often you served nothing at all',\n  TOO_SHORT: 'set by minLength, a character count, which a 0..1 score cannot suggest',\n};\n\n/**\n * Dig a scores object out of a logged line.\n *\n * Handles the bare object, a whole `Verdict`, and the common case of a verdict\n * buried in a wider log record. Anything with at least one known reason code\n * mapped to a number counts; anything else is skipped rather than guessed at.\n */\nexport function extractScores(value: unknown): ScoreSample | null {\n  if (!value || typeof value !== 'object') return null;\n\n  const record = value as Record<string, unknown>;\n  for (const nested of [record.scores, record.verdict, record.guard]) {\n    const found = nested ? extractScores(nested) : null;\n    if (found) return found;\n  }\n\n  const sample: ScoreSample = {};\n  let hits = 0;\n  for (const code of CODES) {\n    const score = record[code];\n    if (typeof score === 'number' && Number.isFinite(score)) {\n      sample[code] = score;\n      hits += 1;\n    }\n  }\n\n  return hits > 0 ? sample : null;\n}\n\nfunction parseLines(text: string): { samples: ScoreSample[]; skipped: number } {\n  const samples: ScoreSample[] = [];\n  let skipped = 0;\n\n  for (const line of text.split('\\n')) {\n    const trimmed = line.trim();\n    if (!trimmed) continue;\n    try {\n      const scores = extractScores(JSON.parse(trimmed));\n      if (scores) samples.push(scores);\n      else skipped += 1;\n    } catch {\n      skipped += 1;\n    }\n  }\n\n  return { samples, skipped };\n}\n\nfunction readStdin(): Promise<string> {\n  return new Promise((resolve, reject) => {\n    let data = '';\n    process.stdin.setEncoding('utf8');\n    process.stdin.on('data', (chunk) => (data += chunk));\n    process.stdin.on('end', () => resolve(data));\n    process.stdin.on('error', reject);\n  });\n}\n\nconst USAGE = `\nllm-output-guard calibrate — derive thresholds from your own logged scores\n\n  npx llm-output-guard calibrate scores.jsonl\n  cat scores.jsonl | npx llm-output-guard calibrate\n\nOptions\n  --fpr <rate>   share of traffic you accept flagging (default 0.001)\n  --json         emit the calibration as JSON instead of a report\n\nInput is JSONL, one logged verdict per line. A bare scores object, a whole\nVerdict, or a wider log record containing either all work:\n\n  {\"REPETITION\":0.03,\"TAIL_LOOP\":0}\n  {\"ok\":true,\"scores\":{\"REPETITION\":0.03},\"reasons\":[]}\n  {\"msg\":\"reply\",\"verdict\":{\"scores\":{\"REPETITION\":0.03}}}\n\nLog them with onVerdict:\n\n  outputGuard({ ...presets.chat, onDegenerate: 'ignore',\n                onVerdict: (v) => log.info({ scores: v.scores }) })\n`;\n\nconst fmt = (n: number) => (Number.isFinite(n) ? n.toFixed(3) : '  -  ');\n\nfunction report(text: string, fpr: number, asJson: boolean): number {\n  const { samples, skipped } = parseLines(text);\n\n  if (samples.length === 0) {\n    process.stderr.write(\n      `No scores found${skipped ? ` (${skipped} lines had none)` : ''}.\\n` +\n        'Expected JSONL with reason codes such as REPETITION or TAIL_LOOP.\\n',\n    );\n    return 1;\n  }\n\n  const result = calibrate(samples, { falsePositiveRate: fpr });\n\n  if (asJson) {\n    process.stdout.write(`${JSON.stringify(result, null, 2)}\\n`);\n    return 0;\n  }\n\n  const out: string[] = [\n    '',\n    `${result.n.toLocaleString()} verdicts` +\n      (skipped ? `, ${skipped.toLocaleString()} lines skipped` : '') +\n      ` — flagging budget ${(fpr * 100).toFixed(2)}% of traffic`,\n  ];\n\n  /*\n   * A caveat every detector shares is a fact about the sample, not about any\n   * one of them. Repeating it per section buries the ones that are specific,\n   * which are the ones worth reading.\n   */\n  const shared = result.summaries[0].caveats.filter((c) =>\n    result.summaries.every((s) => s.caveats.includes(c)),\n  );\n  for (const caveat of shared) out.push(`! ${caveat}`);\n\n  for (const s of result.summaries) {\n    const d = s.distribution;\n    out.push('', `${s.code}   n=${d.n.toLocaleString()}`);\n    out.push(\n      `  p50 ${fmt(d.p50)}   p90 ${fmt(d.p90)}   p99 ${fmt(d.p99)}   ` +\n        `p99.9 ${fmt(d.p999)}   max ${fmt(d.max)}`,\n    );\n\n    if (s.gap) {\n      out.push(\n        `  gap ${fmt(s.gap.below)} -> ${fmt(s.gap.above)}  ` +\n          `(${s.gap.count.toLocaleString()} above, ${(s.gap.share * 100).toFixed(2)}% of traffic)`,\n      );\n    }\n\n    const rateOnly = RATE_ONLY[s.code];\n    if (rateOnly) {\n      out.push(\n        `  fired on ${d.nonZero.toLocaleString()} of ${d.n.toLocaleString()} ` +\n          `(${((d.nonZero / d.n) * 100).toFixed(2)}%) — ${rateOnly}`,\n      );\n    } else {\n      const option = OPTION_FOR[s.code];\n      out.push(`  suggest ${option ? `${option}: ` : ''}${fmt(s.suggested)}`);\n    }\n\n    for (const caveat of s.caveats) {\n      if (!shared.includes(caveat)) out.push(`    ! ${caveat}`);\n    }\n  }\n\n  out.push(\n    '',\n    'These thresholds bound FALSE POSITIVES, not misses. They describe the shape',\n    'of your traffic on the assumption that degeneration is rare in it. Nothing',\n    'here shows a threshold catches anything — that needs labelled samples, which',\n    'a log does not have. A `gap` line is the exception worth trusting: it is real',\n    'separation observed in your own data.',\n    '',\n  );\n\n  process.stdout.write(`${out.join('\\n')}\\n`);\n  return 0;\n}\n\nexport async function main(argv: string[]): Promise<number> {\n  const args = argv.slice(2);\n\n  if (args.includes('--help') || args.includes('-h') || args.length === 0) {\n    process.stdout.write(USAGE);\n    return args.length === 0 ? 1 : 0;\n  }\n\n  const command = args[0] === 'calibrate' ? args.slice(1) : args;\n  const asJson = command.includes('--json');\n\n  let fpr = 0.001;\n  const fprAt = command.indexOf('--fpr');\n  if (fprAt !== -1) {\n    const parsed = Number(command[fprAt + 1]);\n    if (!Number.isFinite(parsed) || parsed <= 0 || parsed >= 1) {\n      process.stderr.write('--fpr expects a rate between 0 and 1, e.g. 0.001\\n');\n      return 1;\n    }\n    fpr = parsed;\n  }\n\n  const file = command.find((arg, i) => !arg.startsWith('--') && command[i - 1] !== '--fpr');\n\n  let text: string;\n  try {\n    text = file ? readFileSync(file, 'utf8') : await readStdin();\n  } catch (error) {\n    process.stderr.write(`Could not read ${file ?? 'stdin'}: ${(error as Error).message}\\n`);\n    return 1;\n  }\n\n  return report(text, fpr, asJson);\n}\n\n/* c8 ignore start -- entry point, exercised by the CLI tests through main() */\nconst invokedDirectly =\n  process.argv[1] && /llm-output-guard[/\\\\]dist[/\\\\]cli|cli\\.(ts|js|cjs)$/.test(process.argv[1]);\n\nif (invokedDirectly) {\n  main(process.argv).then(\n    (code) => {\n      process.exitCode = code;\n    },\n    (error) => {\n      process.stderr.write(`${(error as Error).message}\\n`);\n      process.exitCode = 1;\n    },\n  );\n}\n/* c8 ignore stop */\n"]}