llm-output-guard 1.4.0 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/README.md +115 -141
  2. package/dist/{adapter-options-CQNPNjES.d.cts → adapter-options-h2gysBrj.d.cts} +21 -1
  3. package/dist/{adapter-options-BUOzVp3T.d.ts → adapter-options-v9jM76u7.d.ts} +21 -1
  4. package/dist/ai-sdk.cjs +118 -2
  5. package/dist/ai-sdk.cjs.map +1 -1
  6. package/dist/ai-sdk.d.cts +11 -3
  7. package/dist/ai-sdk.d.ts +11 -3
  8. package/dist/ai-sdk.js +9 -4
  9. package/dist/ai-sdk.js.map +1 -1
  10. package/dist/anthropic.cjs +121 -2
  11. package/dist/anthropic.cjs.map +1 -1
  12. package/dist/anthropic.d.cts +3 -3
  13. package/dist/anthropic.d.ts +3 -3
  14. package/dist/anthropic.js +9 -3
  15. package/dist/anthropic.js.map +1 -1
  16. package/dist/bin.cjs +4 -2
  17. package/dist/bin.cjs.map +1 -1
  18. package/dist/bin.js +4 -2
  19. package/dist/bin.js.map +1 -1
  20. package/dist/chunk-JYDJYJEY.js +71 -0
  21. package/dist/chunk-JYDJYJEY.js.map +1 -0
  22. package/dist/{chunk-NHVIUA4Z.js → chunk-PQXX7OEN.js} +8 -6
  23. package/dist/chunk-PQXX7OEN.js.map +1 -0
  24. package/dist/{chunk-R6QMUCVR.js → chunk-RDIQ3HKR.js} +63 -3
  25. package/dist/chunk-RDIQ3HKR.js.map +1 -0
  26. package/dist/index.cjs +62 -0
  27. package/dist/index.cjs.map +1 -1
  28. package/dist/index.d.cts +81 -3
  29. package/dist/index.d.ts +81 -3
  30. package/dist/index.js +1 -1
  31. package/dist/openai.cjs +128 -3
  32. package/dist/openai.cjs.map +1 -1
  33. package/dist/openai.d.cts +3 -3
  34. package/dist/openai.d.ts +3 -3
  35. package/dist/openai.js +16 -4
  36. package/dist/openai.js.map +1 -1
  37. package/dist/{stream-NO-TTGYs.d.cts → stream-BSCIXfIH.d.cts} +26 -1
  38. package/dist/{stream-NO-TTGYs.d.ts → stream-BSCIXfIH.d.ts} +26 -1
  39. package/package.json +3 -2
  40. package/dist/chunk-NHVIUA4Z.js.map +0 -1
  41. package/dist/chunk-OHF5SN7W.js +0 -17
  42. package/dist/chunk-OHF5SN7W.js.map +0 -1
  43. package/dist/chunk-R6QMUCVR.js.map +0 -1
package/README.md CHANGED
@@ -7,12 +7,26 @@
7
7
 
8
8
  **Detect LLM responses that failed while returning `200 OK`.**
9
9
 
10
+ <!--
11
+ Two rows on purpose: what you are installing, then whether it is looked after.
12
+
13
+ Every GitHub-backed badge carries an explicit cacheSeconds. Shields defaults
14
+ these to max-age=120, which has GitHub's camo proxy refetch 720 times a day
15
+ and gives 720 daily chances to catch the upstream API rate limited and cache
16
+ the error for the whole TTL. That is not hypothetical: it happened to the
17
+ downloads badge, which is why it carries one too.
18
+ -->
10
19
  [![npm](https://img.shields.io/npm/v/llm-output-guard?color=0b7285)](https://www.npmjs.com/package/llm-output-guard)
11
- [![downloads](https://img.shields.io/npm/dm/llm-output-guard?color=0b7285)](https://www.npmjs.com/package/llm-output-guard)
20
+ [![downloads](https://img.shields.io/npm/dm/llm-output-guard?color=0b7285&cacheSeconds=86400)](https://www.npmjs.com/package/llm-output-guard)
12
21
  [![minzipped](https://img.shields.io/bundlejs/size/llm-output-guard?color=0b7285&label=min%2Bgzip)](https://bundlejs.com/?q=llm-output-guard)
13
22
  [![dependencies](https://img.shields.io/badge/dependencies-0-brightgreen)](https://github.com/edwinsatya/llm-output-guard/blob/main/package.json)
23
+ [![types](https://img.shields.io/npm/types/llm-output-guard?color=0b7285&cacheSeconds=86400)](https://www.npmjs.com/package/llm-output-guard)
24
+ [![node](https://img.shields.io/node/v/llm-output-guard?color=0b7285&cacheSeconds=86400)](https://nodejs.org)
25
+
14
26
  [![CI](https://github.com/edwinsatya/llm-output-guard/actions/workflows/ci.yml/badge.svg)](https://github.com/edwinsatya/llm-output-guard/actions/workflows/ci.yml)
15
- [![license](https://img.shields.io/npm/l/llm-output-guard?color=0b7285)](./LICENSE)
27
+ [![last commit](https://img.shields.io/github/last-commit/edwinsatya/llm-output-guard?color=0b7285&cacheSeconds=21600)](https://github.com/edwinsatya/llm-output-guard/commits/main)
28
+ [![commit activity](https://img.shields.io/github/commit-activity/m/edwinsatya/llm-output-guard?color=0b7285&cacheSeconds=86400)](https://github.com/edwinsatya/llm-output-guard/graphs/commit-activity)
29
+ [![license](https://img.shields.io/npm/l/llm-output-guard?color=0b7285&cacheSeconds=86400)](./LICENSE)
16
30
 
17
31
  Your retry layer watches for `429`, `5xx` and timeouts. It cannot see a model that
18
32
  looped until `max_tokens`, returned `{}`, stopped mid-sentence, or answered in the
@@ -56,12 +70,16 @@ every detector, running on your own pasted output. No API key, no request.
56
70
  | `LOW_ENTROPY` | Character-level collapse, token artifacts | Compression ratio |
57
71
  | `TRUNCATED` | Cut off mid-thought | `finish_reason`, unbalanced fences |
58
72
  | `INVALID_JSON` | Prose around the payload, wrong types | Parse + key + schema contract |
59
- | `SCRIPT_MISMATCH` | Answered in the wrong alphabet | Share of letters outside the expected scripts (opt-in) |
60
- | `LANG_MISMATCH` | Answered in the wrong language, same alphabet | Function-word profile (opt-in) |
73
+ | `SCRIPT_MISMATCH` | Answered in the wrong alphabet | Share of letters outside expected scripts · opt-in |
74
+ | `LANG_MISMATCH` | Wrong language, same alphabet | Function-word profile · opt-in |
75
+ | `PROMPT_ECHO` | Returned your prompt instead of an answer | Share of output copied from the prompt · opt-in |
61
76
 
62
77
  Every detector runs even after one fails, so a verdict shows the whole picture
63
78
  rather than whichever check happened to be ordered first. Each returns **0–1, not
64
- a boolean** — you pick the line. Full reference: **[docs/detectors.md](docs/detectors.md)**.
79
+ a boolean** — you pick the line.
80
+
81
+ **Full reference, with the measurements behind every default:
82
+ [docs/detectors.md](docs/detectors.md)**
65
83
 
66
84
  ## Guard your provider in one wrap
67
85
 
@@ -77,8 +95,10 @@ Adapters for the **OpenAI SDK** (both `chat.completions` and `responses`),
77
95
  protocol: Groq, Together, OpenRouter, Fireworks, vLLM, Ollama.
78
96
 
79
97
  On a stream this **cancels the HTTP request** the moment a loop is detectable, so
80
- you stop paying for the rest of it. See **[docs/adapters.md](docs/adapters.md)**
81
- and **[docs/streaming.md](docs/streaming.md)**.
98
+ you stop paying for the rest of it. Running agents? Add `checkToolArguments: true`
99
+ to measure the arguments as well as the prose beside them.
100
+
101
+ **[docs/adapters.md](docs/adapters.md)** · **[docs/streaming.md](docs/streaming.md)**
82
102
 
83
103
  ## The hard part is not catching loops
84
104
 
@@ -86,89 +106,58 @@ A miss is annoying. **A false positive is worse** — a healthy response gets
86
106
  discarded and retried against a slower provider for nothing.
87
107
 
88
108
  So the corpus carries deliberate traps: markdown tables, repeated-prefix lists,
89
- code blocks, rhetorical refrains, a Chinese poem refrain. All repetitive, all
90
- fine, all flagged by a naive detector. Paste one into the
109
+ code blocks, rhetorical refrains, a Chinese poem refrain, a Chinese answer
110
+ wrapped around a TypeScript block. All of them look degenerate to a naive
111
+ detector, and all of them are fine. Paste one into the
91
112
  [playground](https://edwinsatya.github.io/llm-output-guard/) and watch it pass.
92
113
 
93
114
  ---
94
115
 
95
- ### Structured output
116
+ ## Common setups
96
117
 
97
- ```ts
98
- const verdict = checkOutput(raw, {
99
- ...presets.strictJson,
100
- requiredKeys: ['score', 'notes', 'followUp'],
101
- });
102
-
103
- if (verdict.ok) use(verdict.json); // already parsed, fence stripped
104
- ```
105
-
106
- `requiredKeys` only asks whether a name is present. A model returning
107
- `{ "score": "very good" }` where you wanted a number satisfies it and still
108
- breaks everything downstream that does arithmetic. Pass a **schema** to check
109
- the shape rather than the spelling:
118
+ **Structured output** — parse, check keys, and validate against a schema you
119
+ already have:
110
120
 
111
121
  ```ts
112
122
  import { z } from 'zod';
113
123
 
114
- const Review = z.object({
115
- score: z.number().min(0).max(10),
116
- notes: z.string(),
117
- followUp: z.array(z.string()),
124
+ const verdict = checkOutput(raw, {
125
+ ...presets.strictJson,
126
+ schema: z.object({ score: z.number(), notes: z.string() }),
118
127
  });
119
128
 
120
- const verdict = checkOutput(raw, { ...presets.strictJson, schema: Review });
121
-
122
129
  if (verdict.ok) use(verdict.json); // parsed, validated, defaults applied
123
130
  ```
124
131
 
125
- Any [Standard Schema](https://standardschema.dev) validator works — **Zod 4,
126
- Valibot, ArkType**, or your own. The spec is types-only, so this costs an
127
- interface and **no dependency**: your validator is one you already have, and
128
- `llm-output-guard` still installs with nothing behind it.
129
-
130
- On success `verdict.json` is the schema's *output*, so defaults, coercions and
131
- transforms are applied and the value matches the type you declared. On failure
132
- you get `INVALID_JSON` with the failing path in the message —
133
- `score: Expected number, received string`. It is the same reason code as a
134
- missing key or an unparseable payload because it wants the same handling: retry,
135
- or fall through to another provider.
136
-
137
- The two compose, and keys are checked first, so a missing key is still reported
138
- as a missing key rather than as whatever the schema calls it.
132
+ Any [Standard Schema](https://standardschema.dev) validator works — Zod 4,
133
+ Valibot, ArkType and the spec is types-only, so this still costs **no
134
+ dependency**. Details, and the one case that throws:
135
+ [docs/detectors.md](docs/detectors.md#structured-output)
139
136
 
140
- > **The schema must validate synchronously.** `checkOutput` is synchronous by
141
- > design that is what makes it safe on a hot path — so a schema carrying an
142
- > async refinement throws a `TypeError` telling you so, rather than silently
143
- > passing. Everything Zod, Valibot and ArkType produce otherwise is synchronous.
144
- > This is the one thing in the package that throws about your configuration; it
145
- > still never throws about a response.
146
-
147
- See **[docs/detectors.md](docs/detectors.md)** for arrays of repeated records —
148
- a JSON array of identical rows reads as a loop under the default scope, and
149
- `redundancyScope: 'jsonValues'` is the fix.
150
-
151
- ### Answering in the wrong language
137
+ **Answered in the wrong language** a model that ignores "answer in English"
138
+ returns fluent Chinese, not broken English:
152
139
 
153
140
  ```ts
154
141
  checkOutput(raw, { ...presets.chat, expectScript: 'latin' });
155
142
  ```
156
143
 
157
- A model that ignores "answer in English" does not produce broken English, it
158
- produces fluent Chinese. That is a `200 OK` your retry layer cannot see, and it
159
- is detectable by counting characters no word list, no model, decisive from
160
- about a dozen letters. A response answered entirely in the wrong script scores
161
- **1.000**; a healthy response measured against its own script scores
162
- **0.000–0.028**.
144
+ Detectable by counting characters no word list, decisive from about a dozen
145
+ letters. A wrong-script answer scores **1.000**; a healthy one measured against
146
+ its own script scores **0.000–0.028**. Pass every script the answer may
147
+ legitimately contain (`['han', 'latin']` for Chinese). Code fences and URLs are
148
+ excluded, so a TypeScript block never counts as answering in English.
149
+ [More →](docs/detectors.md#answering-in-the-wrong-language)
163
150
 
164
- Pass every script the answer may legitimately contain `['han', 'latin']` for
165
- Chinese, `['han', 'kana', 'latin']` for Japanese. `'latin'` belongs in nearly all
166
- of them, because a Chinese answer about React still contains `useEffect`. Code
167
- fences, inline code and URLs are removed before measuring, so a TypeScript block
168
- never counts as answering in English.
151
+ **Returned your prompt instead of an answer**invisible to every other
152
+ detector here, because such a response is fluent, well-formed and the right
153
+ length:
154
+
155
+ ```ts
156
+ checkOutput(raw, { ...presets.chat, prompt });
157
+ ```
169
158
 
170
- Same script means no signal: Spanish against English scores 0. That is what
171
- `expectLang` is for, and the two compose under separate codes.
159
+ Not for rewrite, translate or summarise endpoints, where copying the input is
160
+ the job. [More →](docs/detectors.md#returning-the-prompt-instead-of-an-answer)
172
161
 
173
162
  ## The verdict
174
163
 
@@ -185,26 +174,26 @@ Same script means no signal: Spanish against English scores 0. That is what
185
174
  }
186
175
  ```
187
176
 
188
- Every detector runs even after one fails, so `reasons` shows the whole picture instead of whichever check happened to be ordered first. `scores` includes passing detectors too — send them to your metrics and you will know your real degeneration rate within a day.
177
+ `reasons` carries every failing signal, not just the first. `scores` includes the
178
+ passing detectors too — send them to your metrics and you will know your real
179
+ degeneration rate within a day.
189
180
 
190
- `modes` says which tokenizer produced a score, for the detectors that have more
191
- than one. **Log it next to `scores`.** `TAIL_LOOP` measures words on spaced
192
- scripts and characters on Chinese, Japanese and Thai; those are two
193
- distributions with different base rates, and aggregating them into one histogram
194
- gives you a number that describes neither.
181
+ `modes` says which tokenizer produced a score. **Log it next to `scores`:**
182
+ `TAIL_LOOP` measures words on spaced scripts and characters on Chinese, Japanese
183
+ and Thai, and pooling two distributions into one histogram gives you a number
184
+ that describes neither.
195
185
 
196
186
  ## Presets
197
187
 
198
188
  `chat` · `strictJson` · `longForm` · `lenient`
199
189
 
200
- They are starting points calibrated against the fixture corpus in this repo — not universal truths. Log your scores for a week, then set your own thresholds.
201
-
202
- ---
190
+ Starting points calibrated against this repo's fixture corpus — not universal
191
+ truths.
203
192
 
204
- ## Calibrating against your own traffic
193
+ ## Calibrate against your own traffic
205
194
 
206
- The shipped presets are tuned on this repo's fixture corpus, which is **not your
207
- traffic**. Log your scores for a week, then derive thresholds you can defend:
195
+ The shipped presets are tuned on a corpus that is **not your traffic**. Log your
196
+ scores for a week, then derive thresholds you can defend:
208
197
 
209
198
  ```bash
210
199
  npx llm-output-guard calibrate scores.jsonl --fpr 0.001
@@ -212,44 +201,33 @@ npx llm-output-guard calibrate scores.jsonl --fpr 0.001
212
201
 
213
202
  The report tells you when your sample is too small to support the rate you asked
214
203
  for, and distinguishes real separation in your data from a false-positive budget —
215
- because only one of those is evidence. Full guide:
216
- **[docs/calibration.md](docs/calibration.md)**.
217
-
218
- ## Script coverage
219
-
220
- Korean, Cyrillic, Greek, Arabic and Devanagari separate words and are handled like
221
- English. **Chinese, Japanese and Thai do not**, so `TAIL_LOOP` switches to
222
- character mode and reads its own threshold. `REPETITION` is blind on those scripts —
223
- a known, measured gap. `SCRIPT_MISMATCH` covers all ten scripts and is the one
224
- detector these are *not* the weak case for. Numbers behind both in
225
- **[docs/script-coverage.md](docs/script-coverage.md)**.
204
+ because only one of those is evidence.
205
+ **[docs/calibration.md](docs/calibration.md)**
226
206
 
227
207
  ## Design notes
228
208
 
229
- - **Zero runtime dependencies**, enforced in CI. Node ≥ 18, works on edge, browser, Deno, Bun.
230
- - **Hand-rolled LZ77** rather than `node:zlib`, so the package stays runtime-agnostic. It is not a real compressor; it only needs to move monotonically with redundancy.
209
+ - **Zero runtime dependencies**, enforced in CI. Node ≥ 18; works on edge, browser, Deno, Bun.
231
210
  - **Pure and synchronous.** No network, no clock, no randomness — safe on a hot path, trivial to test.
232
211
  - **Scores, not booleans.** Detectors report 0–1 and leave the threshold decision to you.
233
212
  - **Abstains rather than guesses.** Samples too short to judge score 0.
213
+ - **Hand-rolled LZ77** rather than `node:zlib`, so the package stays runtime-agnostic.
214
+ - **Sub-millisecond**, 0.383 ms at 500 B and 1.014 ms at 32 KB, with one detector accounting for most of it. `npm run bench` reproduces it — **[docs/performance.md](docs/performance.md)**
215
+ - **Chinese, Japanese and Thai** are handled where they differ: `TAIL_LOOP` switches to character mode, `REPETITION` is blind and says so — **[docs/script-coverage.md](docs/script-coverage.md)**
234
216
 
235
217
  ## Stability
236
218
 
237
- What semver means for this package specifically. These rules bind from **1.0.0**
238
- onward; under `0.x` they described an intent, and the surface was frozen — export
239
- by export — in the 1.0.0 release.
240
-
241
- **The public API is:** everything exported from `llm-output-guard`, plus
242
- `outputGuard` / `OutputGuardOptions` / `DegenerateAction` from `./ai-sdk`, and
243
- `withOutputGuard` / `OutputGuardOptions` / `DegenerateAction` from each of
244
- `./openai` and `./anthropic`. Each subpath is its own contract; the adapters
245
- share internal base types today and are free to diverge, so an option added to
246
- one is not a promise about the others. Anything not exported from those four
247
- entry points is internal, has no stability guarantee, and may move in any release
248
- `internal/proxy-guard.ts` and `internal/tool-calls.ts` included, however much
249
- behaviour they carry. The list is asserted in `test/surface.test.ts`, so an
250
- export cannot join it by accident.
251
-
252
- **Threshold and preset values are behaviour, not implementation.** This is the
219
+ What semver means here specifically. These rules bind from **1.0.0** onward, and
220
+ the public surface was frozen export by export in that release.
221
+
222
+ **The public API is** everything exported from `llm-output-guard`, plus
223
+ `outputGuard` / `OutputGuardOptions` / `DegenerateAction` from `./ai-sdk` and
224
+ `withOutputGuard` / `OutputGuardOptions` / `DegenerateAction` from `./openai` and
225
+ `./anthropic`. Each subpath is its own contract, so an option added to one is not
226
+ a promise about the others. Anything else is internal and may move in any
227
+ release. The list is asserted in `test/surface.test.ts`, so an export cannot join
228
+ it by accident.
229
+
230
+ **Threshold and preset values are behaviour, not implementation.** That is the
253
231
  interesting case, so it gets a rule of its own:
254
232
 
255
233
  | Change | Release type |
@@ -259,42 +237,38 @@ interesting case, so it gets a rule of its own:
259
237
  | Adding a new *option*, defaulted so nothing changes | minor |
260
238
  | Adding a new detector that is opt-in | minor |
261
239
  | Making an existing detector strictly more accurate on its own axis | minor |
262
- | Docs, internals, performance, fixing a detector that was returning a wrong score | patch |
263
-
264
- The reasoning: a threshold change does not break your build, it changes which of
265
- your production responses get discarded and retried. That is a larger event than
266
- a signature change, and it is invisible until your traffic hits it. A number in
267
- `presets.chat` is part of the contract in the same way a function name is.
268
-
269
- **Semver does not cover:** the exact scores a detector returns (only their
270
- direction and the thresholds that act on them), the contents of the fixture
271
- corpus, `message` strings in `Reason`, or the output format of the `calibrate`
272
- CLI's human-readable report. `--json` output *is* covered.
273
-
274
- **Peer ranges** are narrowed only in a major. They are verified rather than
275
- assumed `npm run check:peers` installs the packed tarball against each end of
276
- each declared range and both typechecks and runs the adapter.
277
-
278
- **Why this is written down.** Version 0.4.2 shipped the `ai` peer narrowing, the
279
- new `./openai` subpath, and character-mode `TAIL_LOOP` one breaking change, one
280
- feature, and one behaviour change under a **patch** number, which every default
281
- version range upgrades into automatically. It was withdrawn from npm within the
282
- 72-hour unpublish window and re-released as 0.5.0, where `^0.4.1` correctly
283
- resolves away from it. The rule it broke is the one in the table above: threshold
284
- and preset changes are behaviour changes, and behaviour changes are never
285
- patches.
240
+ | Docs, internals, performance, fixing a detector that returned a wrong score | patch |
241
+
242
+ A threshold change does not break your build. It changes which of your production
243
+ responses get discarded and retried, which is a larger event than a signature
244
+ change and invisible until your traffic hits it. A number in `presets.chat` is
245
+ part of the contract in the same way a function name is.
246
+
247
+ **Not covered:** the exact scores a detector returns (only their direction and
248
+ the thresholds acting on them), the fixture corpus, `message` strings in
249
+ `Reason`, or the human-readable `calibrate` report. `--json` output *is* covered.
250
+
251
+ **Peer ranges** narrow only in a major, and are verified rather than assumed —
252
+ `npm run check:peers` installs the packed tarball against each end of each
253
+ declared range, then typechecks and runs the adapter.
254
+
255
+ **Why this is written down:** 0.4.2 shipped a peer narrowing, a new subpath, and
256
+ a behaviour change under a **patch** number, which every default version range
257
+ upgrades into automatically. It was unpublished within the 72-hour window and
258
+ re-released as 0.5.0. The rule it broke is the one in the table above.
286
259
 
287
260
  ## Limitations
288
261
 
289
262
  - Not a hallucination detector. It measures *shape*, never truth.
290
- - Tool *arguments* are not checked, only the prose beside them. A model that loops inside a JSON argument string is invisible here — your provider validates those against the schema you gave it.
291
- - `openai`'s `responses.stream()` helper is not wrapped. See the note above; `create({ stream: true })` is.
292
- - `REPETITION` does not work on Chinese, Japanese or Thai. See abovethis is a known, measured gap, not an oversight.
293
- - Language detection is a function-word heuristic covering `id`/`en`/`es`. Opt-in, and unreliable under 25 words. `expectScript` is the stronger check where the languages differ in alphabet, and says nothing where they do not.
294
- - `SCRIPT_MISMATCH` does not run mid-stream. A mid-stream check reads a trailing window, and the language of a window is not the language of the response — an English answer quoting a Chinese passage measures 0.114 whole and 0.500 over its last 400 characters.
295
- - Truncation from a missing full stop is weak evidence, scored 0.55 and left below the default thresholds on purpose. Lower `maxTruncation` to ~0.5 to catch it, and expect false positives.
296
- - A JSON array of repeated identical records reads as a loop under the default scope, and fails from three records up. Set `redundancyScope: 'jsonValues'` see **Structured output**.
297
- - Thresholds calibrated on the bundled corpus. Yours will differ and the word and character thresholds need calibrating **separately**, because they are separate distributions.
263
+ - `REPETITION` does not work on Chinese, Japanese or Thai a known, measured gap, not an oversight.
264
+ - `LANG_MISMATCH` is a function-word heuristic covering `id`/`en`/`es`, and is unreliable under 25 words. `expectScript` is the stronger check wherever the alphabets differ.
265
+ - `PROMPT_ECHO` cannot tell a degenerate echo from a rewrite or translationthe difference is in what you asked for, not in the text.
266
+ - `SCRIPT_MISMATCH` and `PROMPT_ECHO` do not run mid-stream: both measure a property of the whole response, and a mid-stream check reads a trailing window.
267
+ - Tool *arguments* are measured only with `checkToolArguments: true`, non-streaming responses only.
268
+ - `openai`'s `responses.stream()` helper is not wrapped; `create({ stream: true })` is.
269
+ - Truncation from a missing full stop is weak evidence, scored 0.55 and deliberately left below the defaults. Lower `maxTruncation` to ~0.5 to catch it, and expect false positives.
270
+ - A JSON array of repeated identical records reads as a loop and fails from three records up. Set `redundancyScope: 'jsonValues'`.
271
+ - Thresholds are calibrated on the bundled corpus. Yours will differ — and the word and character thresholds need calibrating **separately**.
298
272
 
299
273
  ## License
300
274
 
@@ -1,4 +1,4 @@
1
- import { V as Verdict } from './stream-NO-TTGYs.cjs';
1
+ import { V as Verdict } from './stream-BSCIXfIH.cjs';
2
2
 
3
3
  /**
4
4
  * The option surface every provider adapter shares.
@@ -62,6 +62,26 @@ interface AdapterGuardOptions {
62
62
  onVerdict?: (verdict: Verdict, context: {
63
63
  streaming: boolean;
64
64
  }) => void;
65
+ /**
66
+ * Also measure the arguments the model passed to a tool. Default `false`.
67
+ *
68
+ * A tool-calling turn is judged by its preamble, because the text beside a
69
+ * tool call is not the answer -- and that leaves the answer itself
70
+ * unmeasured. Your provider validates arguments against the schema you gave
71
+ * it, which covers types and not content: `{ "query": "site reliability site
72
+ * reliability site reliability ..." }` is a schema-valid string, and it is
73
+ * still a garbage query.
74
+ *
75
+ * Switching this on measures each string value of each call's arguments for
76
+ * redundancy, and folds the result into the same verdict the preamble
77
+ * produces. Reason codes are unchanged, so existing handling works; the
78
+ * `message` says the loop was found in an argument.
79
+ *
80
+ * **Non-streaming responses only.** Arguments arrive as JSON fragments that
81
+ * do not parse until the call is complete, so there is nothing meaningful to
82
+ * measure mid-stream.
83
+ */
84
+ checkToolArguments?: boolean;
65
85
  }
66
86
 
67
87
  export type { AdapterGuardOptions as A, DegenerateAction as D };
@@ -1,4 +1,4 @@
1
- import { V as Verdict } from './stream-NO-TTGYs.js';
1
+ import { V as Verdict } from './stream-BSCIXfIH.js';
2
2
 
3
3
  /**
4
4
  * The option surface every provider adapter shares.
@@ -62,6 +62,26 @@ interface AdapterGuardOptions {
62
62
  onVerdict?: (verdict: Verdict, context: {
63
63
  streaming: boolean;
64
64
  }) => void;
65
+ /**
66
+ * Also measure the arguments the model passed to a tool. Default `false`.
67
+ *
68
+ * A tool-calling turn is judged by its preamble, because the text beside a
69
+ * tool call is not the answer -- and that leaves the answer itself
70
+ * unmeasured. Your provider validates arguments against the schema you gave
71
+ * it, which covers types and not content: `{ "query": "site reliability site
72
+ * reliability site reliability ..." }` is a schema-valid string, and it is
73
+ * still a garbage query.
74
+ *
75
+ * Switching this on measures each string value of each call's arguments for
76
+ * redundancy, and folds the result into the same verdict the preamble
77
+ * produces. Reason codes are unchanged, so existing handling works; the
78
+ * `message` says the loop was found in an argument.
79
+ *
80
+ * **Non-streaming responses only.** Arguments arrive as JSON fragments that
81
+ * do not parse until the call is complete, so there is nothing meaningful to
82
+ * measure mid-stream.
83
+ */
84
+ checkToolArguments?: boolean;
65
85
  }
66
86
 
67
87
  export type { AdapterGuardOptions as A, DegenerateAction as D };
package/dist/ai-sdk.cjs CHANGED
@@ -282,6 +282,37 @@ function judgeableLetters(text, options) {
282
282
  return found.filter((ch) => !SCRIPT_NEUTRAL.test(ch));
283
283
  }
284
284
 
285
+ // src/detectors/prompt-echo.ts
286
+ function promptEchoDetail(text, prompt, options = {}) {
287
+ const {
288
+ n = 5,
289
+ charN = 12,
290
+ minTokens = 40,
291
+ nonSpacedCutoff = 0.5,
292
+ maxSample = 8e3
293
+ } = options;
294
+ const output = text.slice(0, maxSample);
295
+ const mode = options.mode ?? tokenModeOf(output, nonSpacedCutoff);
296
+ if (typeof prompt !== "string" || prompt.length === 0) return { score: 0, mode };
297
+ const tokenize = mode === "char" ? chars : words;
298
+ const run = mode === "char" ? charN : n;
299
+ const out = tokenize(output);
300
+ const src = tokenize(prompt.slice(0, maxSample));
301
+ if (out.length < minTokens) return { score: 0, mode };
302
+ if (out.length < run || src.length < run) return { score: 0, mode };
303
+ const seen = /* @__PURE__ */ new Set();
304
+ for (let i = 0; i + run <= src.length; i++) {
305
+ seen.add(src.slice(i, i + run).join(" "));
306
+ }
307
+ let matched = 0;
308
+ let total = 0;
309
+ for (let i = 0; i + run <= out.length; i++) {
310
+ if (seen.has(out.slice(i, i + run).join(" "))) matched++;
311
+ total++;
312
+ }
313
+ return { score: total === 0 ? 0 : clamp01(matched / total), mode };
314
+ }
315
+
285
316
  // src/internal/json-scope.ts
286
317
  function stringValues(value, out = []) {
287
318
  if (typeof value === "string") out.push(value);
@@ -316,6 +347,7 @@ var DEFAULTS = {
316
347
  allowJsonFence: true,
317
348
  maxLangMismatch: 0.6,
318
349
  maxScriptMismatch: 0.5,
350
+ maxPromptEcho: 0.6,
319
351
  ngram: 3,
320
352
  redundancyScope: "document"
321
353
  };
@@ -423,6 +455,18 @@ function checkOutput(text, options = {}) {
423
455
  `${Math.round(s * 100)}% of letters are not in ${wanted.join(" or ")}.`
424
456
  );
425
457
  }
458
+ if (opts.prompt) {
459
+ const { score, mode } = promptEchoDetail(text, opts.prompt, {
460
+ nonSpacedCutoff: opts.nonSpacedCutoff
461
+ });
462
+ add(
463
+ "PROMPT_ECHO",
464
+ score,
465
+ opts.maxPromptEcho,
466
+ `${Math.round(score * 100)}% of the response is copied from the prompt.`,
467
+ mode
468
+ );
469
+ }
426
470
  if (opts.expectLang) {
427
471
  const s = languageMismatchScore(text, opts.expectLang);
428
472
  add(
@@ -482,6 +526,19 @@ var DEFERRED_TO_END = {
482
526
  * and this is one line for the callers who need it.
483
527
  */
484
528
  expectScript: null,
529
+ /*
530
+ * PROMPT_ECHO is deferred for the same reason as SCRIPT_MISMATCH, and it
531
+ * bites harder. The score is the share of the *whole* output copied from the
532
+ * prompt, so a trailing window measures the share of that window -- and a
533
+ * response that opens by echoing the system prompt and then answers properly
534
+ * reads as 1.000 over its first window and 0.000 over its last. Neither
535
+ * number describes the response.
536
+ *
537
+ * Dilution is the point of the measure: an output that is 10% leaked prompt
538
+ * and 90% answer should score 0.1, and only the finished response knows what
539
+ * that ratio is.
540
+ */
541
+ prompt: null,
485
542
  /*
486
543
  * LOW_ENTROPY is deferred for a second reason: cost. The LZ77 pass is
487
544
  * 0.4ms at 500 characters and 11ms at its 4000-character sample cap, which
@@ -557,6 +614,60 @@ function checkPreamble(text, options) {
557
614
  return checkOutput(text, { ...options, ...TOOL_CALL_PREAMBLE });
558
615
  }
559
616
 
617
+ // src/internal/tool-arguments.ts
618
+ var ARGUMENT_SCOPE = {
619
+ minLength: 0,
620
+ maxTruncation: null,
621
+ maxCompressibility: null,
622
+ expectJson: false,
623
+ expectLang: null,
624
+ expectScript: null,
625
+ finishReason: void 0,
626
+ redundancyScope: "jsonValues"
627
+ };
628
+ function argumentsToText(args) {
629
+ if (typeof args === "string") return args;
630
+ if (args === null || args === void 0) return "";
631
+ try {
632
+ return JSON.stringify(args) ?? "";
633
+ } catch {
634
+ return "";
635
+ }
636
+ }
637
+ function checkArguments(calls, options) {
638
+ const texts = calls.map(argumentsToText).filter(hasMeasurableContent);
639
+ if (texts.length === 0) return null;
640
+ let worst = null;
641
+ for (const text of texts) {
642
+ const verdict = checkOutput(text, { ...options, ...ARGUMENT_SCOPE });
643
+ if (!worst || worst.ok && !verdict.ok) worst = verdict;
644
+ else if (!verdict.ok && !worst.ok && maxScore(verdict) > maxScore(worst)) worst = verdict;
645
+ }
646
+ return worst;
647
+ }
648
+ var maxScore = (verdict) => verdict.reasons.reduce((hi, reason) => Math.max(hi, reason.score), 0);
649
+ var hasMeasurableContent = (text) => redundancySpans(text, "jsonValues").some((span) => span.trim().length > 0);
650
+ function mergeVerdicts(preamble, args) {
651
+ if (!args) return preamble;
652
+ const labelled = args.reasons.map((reason) => ({
653
+ ...reason,
654
+ message: `In a tool call argument: ${reason.message}`
655
+ }));
656
+ if (!preamble) return { ...args, reasons: labelled };
657
+ const scores = { ...preamble.scores };
658
+ for (const [code, score] of Object.entries(args.scores)) {
659
+ const key = code;
660
+ if (scores[key] === void 0 || score > scores[key]) scores[key] = score;
661
+ }
662
+ const reasons = [...preamble.reasons, ...labelled];
663
+ return {
664
+ ...preamble,
665
+ ok: reasons.length === 0,
666
+ reasons,
667
+ scores
668
+ };
669
+ }
670
+
560
671
  // src/ai-sdk.ts
561
672
  var isToolPart = (part) => part.type.startsWith("tool-");
562
673
  function finishReasonOf(value) {
@@ -565,7 +676,7 @@ function finishReasonOf(value) {
565
676
  return void 0;
566
677
  }
567
678
  function outputGuard(options = {}) {
568
- const { onDegenerate = "throw", onVerdict, ...guardOptions } = options;
679
+ const { onDegenerate = "throw", onVerdict, checkToolArguments = false, ...guardOptions } = options;
569
680
  const act = (verdict, streaming) => {
570
681
  onVerdict?.(verdict, { streaming });
571
682
  if (verdict.ok || onDegenerate === "ignore") return;
@@ -599,7 +710,12 @@ function outputGuard(options = {}) {
599
710
  const content = result.content ?? [];
600
711
  const text = content.filter((part) => part.type === "text").map((part) => part.text ?? "").join("");
601
712
  if (content.some(isToolPart)) {
602
- const verdict = checkPreamble(text, guardOptions);
713
+ const preamble = checkPreamble(text, guardOptions);
714
+ const args = checkToolArguments ? checkArguments(
715
+ content.filter((part) => part.type === "tool-call").map((part) => part.input ?? part.args),
716
+ guardOptions
717
+ ) : null;
718
+ const verdict = mergeVerdicts(preamble, args);
603
719
  if (verdict) act(verdict, false);
604
720
  return result;
605
721
  }