llm-output-guard 1.5.0 → 1.7.0

package/README.md

@@ -95,8 +95,10 @@ Adapters for the **OpenAI SDK** (both `chat.completions` and `responses`),
 protocol: Groq, Together, OpenRouter, Fireworks, vLLM, Ollama.
 
 On a stream this **cancels the HTTP request** the moment a loop is detectable, so
-you stop paying for the rest of it. Running agents? Add `checkToolArguments: true`
-to measure the arguments as well as the prose beside them.
+you stop paying for the rest of it. Two switches worth knowing:
+`checkToolArguments: true` measures the arguments a model passes to a tool, and
+`checkPromptEcho: true` reads the prompt out of each request so a model that
+replays it instead of answering is caught.
 
 **[docs/adapters.md](docs/adapters.md)** · **[docs/streaming.md](docs/streaming.md)**
 
@@ -154,6 +156,8 @@ length:
 
 ```ts
 checkOutput(raw, { ...presets.chat, prompt });
+// or, from an adapter, which reads the prompt out of the request itself:
+withOutputGuard(new OpenAI(), { ...presets.chat, checkPromptEcho: true });
 ```
 
 Not for rewrite, translate or summarise endpoints, where copying the input is
@@ -192,13 +196,16 @@ truths.
 
 ## Calibrate against your own traffic
 
-The shipped presets are tuned on a corpus that is **not your traffic**. Log your
-scores for a week, then derive thresholds you can defend:
+The shipped presets are tuned on a corpus that is **not your traffic**. Score the
+responses you already have, then derive thresholds you can defend:
 
 ```bash
-npx llm-output-guard calibrate scores.jsonl --fpr 0.001
+npx llm-output-guard check logs/*.txt --json | npx llm-output-guard calibrate --fpr 0.001
 ```
 
+`check` also works as a CI assertion — it exits 1 when anything is degenerate,
+2 when the input cannot be read.
+
 The report tells you when your sample is too small to support the rate you asked
 for, and distinguishes real separation in your data from a false-positive budget —
 because only one of those is evidence.
@@ -261,9 +268,9 @@ re-released as 0.5.0. The rule it broke is the one in the table above.
 
 - Not a hallucination detector. It measures *shape*, never truth.
 - `REPETITION` does not work on Chinese, Japanese or Thai — a known, measured gap, not an oversight.
-- `LANG_MISMATCH` is a function-word heuristic covering `id`/`en`/`es`, and is unreliable under 25 words. `expectScript` is the stronger check wherever the alphabets differ.
+- `LANG_MISMATCH` is a function-word heuristic covering `id`/`en`/`es`/`pt`/`it`/`fr`/`de`/`nl`, and is unreliable under 25 words. `expectScript` is the stronger check wherever the alphabets differ.
 - `PROMPT_ECHO` cannot tell a degenerate echo from a rewrite or translation — the difference is in what you asked for, not in the text.
-- `SCRIPT_MISMATCH` and `PROMPT_ECHO` do not run mid-stream: both measure a property of the whole response, and a mid-stream check reads a trailing window.
+- `SCRIPT_MISMATCH` and `PROMPT_ECHO` do not run mid-stream by default: both measure a property of the whole response, and a mid-stream check reads a trailing window. `earlyDocumentChecks: true` opts in, with a measured false-positive risk — see **[docs/streaming.md](docs/streaming.md)**.
 - Tool *arguments* are measured only with `checkToolArguments: true`, non-streaming responses only.
 - `openai`'s `responses.stream()` helper is not wrapped; `create({ stream: true })` is.
 - Truncation from a missing full stop is weak evidence, scored 0.55 and deliberately left below the defaults. Lower `maxTruncation` to ~0.5 to catch it, and expect false positives.

package/dist/{adapter-options-v9jM76u7.d.ts → adapter-options-B-fgYd--.d.cts}

@@ -1,4 +1,4 @@
-import { V as Verdict } from './stream-BSCIXfIH.js';
+import { V as Verdict } from './stream-BYy6vuGy.cjs';
 
 /**
  * The option surface every provider adapter shares.
@@ -82,6 +82,20 @@ interface AdapterGuardOptions {
      * measure mid-stream.
      */
     checkToolArguments?: boolean;
+    /**
+     * Check the response against the prompt that produced it, so a model that
+     * replays your prompt instead of answering is caught. Default `false`.
+     *
+     * `PROMPT_ECHO` needs the prompt, and a guard is configured once when you
+     * wrap the client while the prompt changes on every call -- so this is a
+     * switch rather than a value. Turn it on and the adapter reads the prompt out
+     * of each request it is already forwarding.
+     *
+     * **Do not enable this on a rewrite, translate, summarise or extract
+     * endpoint.** Copying from the input is the job on those, so a correct answer
+     * scores high and the detector measures the task rather than a failure.
+     */
+    checkPromptEcho?: boolean;
 }
 
 export type { AdapterGuardOptions as A, DegenerateAction as D };

package/dist/{adapter-options-h2gysBrj.d.cts → adapter-options-DNkMNS92.d.ts}

@@ -1,4 +1,4 @@
-import { V as Verdict } from './stream-BSCIXfIH.cjs';
+import { V as Verdict } from './stream-BYy6vuGy.js';
 
 /**
  * The option surface every provider adapter shares.
@@ -82,6 +82,20 @@ interface AdapterGuardOptions {
      * measure mid-stream.
      */
     checkToolArguments?: boolean;
+    /**
+     * Check the response against the prompt that produced it, so a model that
+     * replays your prompt instead of answering is caught. Default `false`.
+     *
+     * `PROMPT_ECHO` needs the prompt, and a guard is configured once when you
+     * wrap the client while the prompt changes on every call -- so this is a
+     * switch rather than a value. Turn it on and the adapter reads the prompt out
+     * of each request it is already forwarding.
+     *
+     * **Do not enable this on a rewrite, translate, summarise or extract
+     * endpoint.** Copying from the input is the job on those, so a correct answer
+     * scores high and the detector measures the task rather than a failure.
+     */
+    checkPromptEcho?: boolean;
 }
 
 export type { AdapterGuardOptions as A, DegenerateAction as D };

package/dist/ai-sdk.cjs

@@ -210,7 +210,12 @@ function jsonScore(text, options = {}) {
 var PROFILES = {
   id: /* @__PURE__ */ new Set(["yang", "dan", "di", "untuk", "dengan", "ini", "itu", "dari", "pada", "tidak", "adalah", "akan", "bisa", "kita", "saya", "atau", "juga", "dalam", "sudah", "ke"]),
   en: /* @__PURE__ */ new Set(["the", "and", "of", "to", "in", "is", "that", "for", "it", "with", "as", "this", "are", "be", "you", "on", "not", "or", "can", "we"]),
-  es: /* @__PURE__ */ new Set(["el", "la", "de", "que", "y", "en", "los", "un", "por", "con", "las", "para", "una", "es", "no", "se", "del", "al", "lo", "como"])
+  es: /* @__PURE__ */ new Set(["el", "los", "las", "y", "es", "no", "muy", "pero", "este", "esta", "sus", "cuando", "donde", "porque", "sin", "hasta", "aunque", "mismo", "otro", "todos"]),
+  pt: /* @__PURE__ */ new Set(["n\xE3o", "\xE9", "s\xE3o", "uma", "com", "em", "do", "da", "dos", "das", "ao", "voc\xEA", "tamb\xE9m", "muito", "mais", "j\xE1", "pelo", "isso", "seu", "sua"]),
+  it: /* @__PURE__ */ new Set(["il", "di", "che", "non", "per", "della", "nel", "nella", "gli", "pi\xF9", "anche", "perch\xE9", "questo", "sono", "essere", "dei", "delle", "agli", "per\xF2", "sia"]),
+  fr: /* @__PURE__ */ new Set(["les", "des", "du", "qui", "dans", "pour", "sur", "pas", "ce", "est", "plus", "nous", "vous", "avec", "cette", "aux", "\xEAtre", "mais", "leur", "tout"]),
+  de: /* @__PURE__ */ new Set(["der", "die", "das", "und", "ist", "nicht", "ein", "eine", "sich", "mit", "den", "f\xFCr", "auf", "von", "dem", "werden", "oder", "auch", "wird", "kann"]),
+  nl: /* @__PURE__ */ new Set(["het", "een", "niet", "zijn", "worden", "maar", "deze", "wordt", "ook", "naar", "bij", "over", "door", "tussen", "omdat", "moet", "kan", "heeft", "dat", "van"])
 };
 function languageProfile(text) {
   const w = words(text);
@@ -570,8 +575,44 @@ var DEFERRED_TO_END = {
    */
   maxCompressibility: null
 };
+var EARLY_DOCUMENT_MIN = 600;
+var EARLY_DOCUMENT_CERTAINTY = 0.9;
+var EARLY_DOCUMENT_SAMPLE = 8e3;
+var DOCUMENT_ONLY = {
+  minLength: 0,
+  maxRepetition: null,
+  maxTailLoop: null,
+  maxCharTailLoop: null,
+  maxCompressibility: null,
+  maxTruncation: null,
+  expectJson: false,
+  expectLang: null,
+  finishReason: void 0,
+  maxScriptMismatch: EARLY_DOCUMENT_CERTAINTY,
+  maxPromptEcho: EARLY_DOCUMENT_CERTAINTY
+};
+function mergeEarly(windowVerdict, document) {
+  if (!document) return windowVerdict;
+  const reasons = [...windowVerdict.reasons, ...document.reasons];
+  return {
+    ...windowVerdict,
+    ok: reasons.length === 0,
+    reasons,
+    scores: { ...windowVerdict.scores, ...document.scores },
+    ...document.modes || windowVerdict.modes ? { modes: { ...windowVerdict.modes, ...document.modes } } : {}
+  };
+}
 function createStreamGuard(options = {}) {
-  const { checkEvery = 400, warmup = 240, window = 2e3, ...checkOptions } = options;
+  const {
+    checkEvery = 400,
+    warmup = 240,
+    window = 2e3,
+    earlyDocumentChecks = false,
+    ...checkOptions
+  } = options;
+  const earlyDocument = earlyDocumentChecks && Boolean(checkOptions.expectScript || checkOptions.prompt);
+  let nextDocumentAt = EARLY_DOCUMENT_MIN;
+  let documentSettled = false;
   let text = "";
   let sinceCheck = 0;
   let checks = 0;
@@ -591,7 +632,12 @@ function createStreamGuard(options = {}) {
       sinceCheck = 0;
       checks += 1;
       const recent = text.length > window ? text.slice(-window) : text;
-      return checkOutput(recent, { ...checkOptions, ...DEFERRED_TO_END });
+      const verdict = checkOutput(recent, { ...checkOptions, ...DEFERRED_TO_END });
+      if (!earlyDocument || documentSettled || text.length < nextDocumentAt) return verdict;
+      nextDocumentAt = text.length * 2;
+      const head = text.length > EARLY_DOCUMENT_SAMPLE ? text.slice(0, EARLY_DOCUMENT_SAMPLE) : text;
+      if (head.length >= EARLY_DOCUMENT_SAMPLE) documentSettled = true;
+      return mergeEarly(verdict, checkOutput(head, { ...checkOptions, ...DOCUMENT_ONLY }));
     },
     end(finishReason) {
       return checkOutput(text, {
@@ -668,6 +714,33 @@ function mergeVerdicts(preamble, args) {
   };
 }
 
+// src/internal/prompt-text.ts
+var INPUT_ROLES = /* @__PURE__ */ new Set(["system", "developer", "user", "human"]);
+function contentText(content) {
+  if (typeof content === "string") return content;
+  if (!Array.isArray(content)) return "";
+  return content.map((part) => {
+    if (typeof part === "string") return part;
+    if (part && typeof part === "object") {
+      const text = part.text;
+      if (typeof text === "string") return text;
+    }
+    return "";
+  }).filter(Boolean).join("\n");
+}
+function promptFromMessages(messages) {
+  if (!Array.isArray(messages)) return "";
+  const parts = [];
+  for (const message of messages) {
+    if (!message || typeof message !== "object") continue;
+    const role = message.role;
+    if (typeof role === "string" && !INPUT_ROLES.has(role)) continue;
+    const text = contentText(message.content);
+    if (text.trim().length > 0) parts.push(text);
+  }
+  return parts.join("\n\n");
+}
+
 // src/ai-sdk.ts
 var isToolPart = (part) => part.type.startsWith("tool-");
 function finishReasonOf(value) {
@@ -676,7 +749,18 @@ function finishReasonOf(value) {
   return void 0;
 }
 function outputGuard(options = {}) {
-  const { onDegenerate = "throw", onVerdict, checkToolArguments = false, ...guardOptions } = options;
+  const {
+    onDegenerate = "throw",
+    onVerdict,
+    checkToolArguments = false,
+    checkPromptEcho = false,
+    ...guardOptions
+  } = options;
+  const optionsFor = (params) => {
+    if (!checkPromptEcho || guardOptions.prompt) return guardOptions;
+    const prompt = promptFromMessages(params?.prompt);
+    return prompt ? { ...guardOptions, prompt } : guardOptions;
+  };
   const act = (verdict, streaming) => {
     onVerdict?.(verdict, { streaming });
     if (verdict.ok || onDegenerate === "ignore") return;
@@ -704,16 +788,18 @@ function outputGuard(options = {}) {
      * all it can do is stop a bad answer from being used as a good one.
      */
     async wrapGenerate({
-      doGenerate
+      doGenerate,
+      params
     }) {
       const result = await doGenerate();
+      const callOptions = optionsFor(params);
       const content = result.content ?? [];
       const text = content.filter((part) => part.type === "text").map((part) => part.text ?? "").join("");
       if (content.some(isToolPart)) {
-        const preamble = checkPreamble(text, guardOptions);
+        const preamble = checkPreamble(text, callOptions);
         const args = checkToolArguments ? checkArguments(
           content.filter((part) => part.type === "tool-call").map((part) => part.input ?? part.args),
-          guardOptions
+          callOptions
         ) : null;
         const verdict = mergeVerdicts(preamble, args);
         if (verdict) act(verdict, false);
@@ -721,18 +807,19 @@ function outputGuard(options = {}) {
       }
       act(
         checkOutput(text, {
-          ...guardOptions,
-          finishReason: finishReasonOf(result.finishReason) ?? guardOptions.finishReason
+          ...callOptions,
+          finishReason: finishReasonOf(result.finishReason) ?? callOptions.finishReason
         }),
         false
       );
       return result;
     },
     async wrapStream({
-      doStream
+      doStream,
+      params
     }) {
       const result = await doStream();
-      const guard = createStreamGuard(guardOptions);
+      const guard = createStreamGuard(optionsFor(params));
       let fired = false;
       let sawToolCall = false;
       let finishReason;