llm-output-guard 1.4.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +115 -141
- package/dist/{adapter-options-CQNPNjES.d.cts → adapter-options-h2gysBrj.d.cts} +21 -1
- package/dist/{adapter-options-BUOzVp3T.d.ts → adapter-options-v9jM76u7.d.ts} +21 -1
- package/dist/ai-sdk.cjs +118 -2
- package/dist/ai-sdk.cjs.map +1 -1
- package/dist/ai-sdk.d.cts +11 -3
- package/dist/ai-sdk.d.ts +11 -3
- package/dist/ai-sdk.js +9 -4
- package/dist/ai-sdk.js.map +1 -1
- package/dist/anthropic.cjs +121 -2
- package/dist/anthropic.cjs.map +1 -1
- package/dist/anthropic.d.cts +3 -3
- package/dist/anthropic.d.ts +3 -3
- package/dist/anthropic.js +9 -3
- package/dist/anthropic.js.map +1 -1
- package/dist/bin.cjs +4 -2
- package/dist/bin.cjs.map +1 -1
- package/dist/bin.js +4 -2
- package/dist/bin.js.map +1 -1
- package/dist/chunk-JYDJYJEY.js +71 -0
- package/dist/chunk-JYDJYJEY.js.map +1 -0
- package/dist/{chunk-NHVIUA4Z.js → chunk-PQXX7OEN.js} +8 -6
- package/dist/chunk-PQXX7OEN.js.map +1 -0
- package/dist/{chunk-R6QMUCVR.js → chunk-RDIQ3HKR.js} +63 -3
- package/dist/chunk-RDIQ3HKR.js.map +1 -0
- package/dist/index.cjs +62 -0
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +81 -3
- package/dist/index.d.ts +81 -3
- package/dist/index.js +1 -1
- package/dist/openai.cjs +128 -3
- package/dist/openai.cjs.map +1 -1
- package/dist/openai.d.cts +3 -3
- package/dist/openai.d.ts +3 -3
- package/dist/openai.js +16 -4
- package/dist/openai.js.map +1 -1
- package/dist/{stream-NO-TTGYs.d.cts → stream-BSCIXfIH.d.cts} +26 -1
- package/dist/{stream-NO-TTGYs.d.ts → stream-BSCIXfIH.d.ts} +26 -1
- package/package.json +3 -2
- package/dist/chunk-NHVIUA4Z.js.map +0 -1
- package/dist/chunk-OHF5SN7W.js +0 -17
- package/dist/chunk-OHF5SN7W.js.map +0 -1
- package/dist/chunk-R6QMUCVR.js.map +0 -1
package/README.md
CHANGED
|
@@ -7,12 +7,26 @@
|
|
|
7
7
|
|
|
8
8
|
**Detect LLM responses that failed while returning `200 OK`.**
|
|
9
9
|
|
|
10
|
+
<!--
|
|
11
|
+
Two rows on purpose: what you are installing, then whether it is looked after.
|
|
12
|
+
|
|
13
|
+
Every GitHub-backed badge carries an explicit cacheSeconds. Shields defaults
|
|
14
|
+
these to max-age=120, which has GitHub's camo proxy refetch 720 times a day
|
|
15
|
+
and gives 720 daily chances to catch the upstream API rate limited and cache
|
|
16
|
+
the error for the whole TTL. That is not hypothetical: it happened to the
|
|
17
|
+
downloads badge, which is why it carries one too.
|
|
18
|
+
-->
|
|
10
19
|
[](https://www.npmjs.com/package/llm-output-guard)
|
|
11
|
-
[](https://www.npmjs.com/package/llm-output-guard)
|
|
20
|
+
[](https://www.npmjs.com/package/llm-output-guard)
|
|
12
21
|
[](https://bundlejs.com/?q=llm-output-guard)
|
|
13
22
|
[](https://github.com/edwinsatya/llm-output-guard/blob/main/package.json)
|
|
23
|
+
[](https://www.npmjs.com/package/llm-output-guard)
|
|
24
|
+
[](https://nodejs.org)
|
|
25
|
+
|
|
14
26
|
[](https://github.com/edwinsatya/llm-output-guard/actions/workflows/ci.yml)
|
|
15
|
-
[](https://github.com/edwinsatya/llm-output-guard/commits/main)
|
|
28
|
+
[](https://github.com/edwinsatya/llm-output-guard/graphs/commit-activity)
|
|
29
|
+
[](./LICENSE)
|
|
16
30
|
|
|
17
31
|
Your retry layer watches for `429`, `5xx` and timeouts. It cannot see a model that
|
|
18
32
|
looped until `max_tokens`, returned `{}`, stopped mid-sentence, or answered in the
|
|
@@ -56,12 +70,16 @@ every detector, running on your own pasted output. No API key, no request.
|
|
|
56
70
|
| `LOW_ENTROPY` | Character-level collapse, token artifacts | Compression ratio |
|
|
57
71
|
| `TRUNCATED` | Cut off mid-thought | `finish_reason`, unbalanced fences |
|
|
58
72
|
| `INVALID_JSON` | Prose around the payload, wrong types | Parse + key + schema contract |
|
|
59
|
-
| `SCRIPT_MISMATCH` | Answered in the wrong alphabet | Share of letters outside
|
|
60
|
-
| `LANG_MISMATCH` |
|
|
73
|
+
| `SCRIPT_MISMATCH` | Answered in the wrong alphabet | Share of letters outside expected scripts · opt-in |
|
|
74
|
+
| `LANG_MISMATCH` | Wrong language, same alphabet | Function-word profile · opt-in |
|
|
75
|
+
| `PROMPT_ECHO` | Returned your prompt instead of an answer | Share of output copied from the prompt · opt-in |
|
|
61
76
|
|
|
62
77
|
Every detector runs even after one fails, so a verdict shows the whole picture
|
|
63
78
|
rather than whichever check happened to be ordered first. Each returns **0–1, not
|
|
64
|
-
a boolean** — you pick the line.
|
|
79
|
+
a boolean** — you pick the line.
|
|
80
|
+
|
|
81
|
+
**Full reference, with the measurements behind every default:
|
|
82
|
+
[docs/detectors.md](docs/detectors.md)**
|
|
65
83
|
|
|
66
84
|
## Guard your provider in one wrap
|
|
67
85
|
|
|
@@ -77,8 +95,10 @@ Adapters for the **OpenAI SDK** (both `chat.completions` and `responses`),
|
|
|
77
95
|
protocol: Groq, Together, OpenRouter, Fireworks, vLLM, Ollama.
|
|
78
96
|
|
|
79
97
|
On a stream this **cancels the HTTP request** the moment a loop is detectable, so
|
|
80
|
-
you stop paying for the rest of it.
|
|
81
|
-
|
|
98
|
+
you stop paying for the rest of it. Running agents? Add `checkToolArguments: true`
|
|
99
|
+
to measure the arguments as well as the prose beside them.
|
|
100
|
+
|
|
101
|
+
**[docs/adapters.md](docs/adapters.md)** · **[docs/streaming.md](docs/streaming.md)**
|
|
82
102
|
|
|
83
103
|
## The hard part is not catching loops
|
|
84
104
|
|
|
@@ -86,89 +106,58 @@ A miss is annoying. **A false positive is worse** — a healthy response gets
|
|
|
86
106
|
discarded and retried against a slower provider for nothing.
|
|
87
107
|
|
|
88
108
|
So the corpus carries deliberate traps: markdown tables, repeated-prefix lists,
|
|
89
|
-
code blocks, rhetorical refrains, a Chinese poem refrain
|
|
90
|
-
|
|
109
|
+
code blocks, rhetorical refrains, a Chinese poem refrain, a Chinese answer
|
|
110
|
+
wrapped around a TypeScript block. All of them look degenerate to a naive
|
|
111
|
+
detector, and all of them are fine. Paste one into the
|
|
91
112
|
[playground](https://edwinsatya.github.io/llm-output-guard/) and watch it pass.
|
|
92
113
|
|
|
93
114
|
---
|
|
94
115
|
|
|
95
|
-
|
|
116
|
+
## Common setups
|
|
96
117
|
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
...presets.strictJson,
|
|
100
|
-
requiredKeys: ['score', 'notes', 'followUp'],
|
|
101
|
-
});
|
|
102
|
-
|
|
103
|
-
if (verdict.ok) use(verdict.json); // already parsed, fence stripped
|
|
104
|
-
```
|
|
105
|
-
|
|
106
|
-
`requiredKeys` only asks whether a name is present. A model returning
|
|
107
|
-
`{ "score": "very good" }` where you wanted a number satisfies it and still
|
|
108
|
-
breaks everything downstream that does arithmetic. Pass a **schema** to check
|
|
109
|
-
the shape rather than the spelling:
|
|
118
|
+
**Structured output** — parse, check keys, and validate against a schema you
|
|
119
|
+
already have:
|
|
110
120
|
|
|
111
121
|
```ts
|
|
112
122
|
import { z } from 'zod';
|
|
113
123
|
|
|
114
|
-
const
|
|
115
|
-
|
|
116
|
-
notes: z.string(),
|
|
117
|
-
followUp: z.array(z.string()),
|
|
124
|
+
const verdict = checkOutput(raw, {
|
|
125
|
+
...presets.strictJson,
|
|
126
|
+
schema: z.object({ score: z.number(), notes: z.string() }),
|
|
118
127
|
});
|
|
119
128
|
|
|
120
|
-
const verdict = checkOutput(raw, { ...presets.strictJson, schema: Review });
|
|
121
|
-
|
|
122
129
|
if (verdict.ok) use(verdict.json); // parsed, validated, defaults applied
|
|
123
130
|
```
|
|
124
131
|
|
|
125
|
-
Any [Standard Schema](https://standardschema.dev) validator works —
|
|
126
|
-
Valibot, ArkType
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
On success `verdict.json` is the schema's *output*, so defaults, coercions and
|
|
131
|
-
transforms are applied and the value matches the type you declared. On failure
|
|
132
|
-
you get `INVALID_JSON` with the failing path in the message —
|
|
133
|
-
`score: Expected number, received string`. It is the same reason code as a
|
|
134
|
-
missing key or an unparseable payload because it wants the same handling: retry,
|
|
135
|
-
or fall through to another provider.
|
|
136
|
-
|
|
137
|
-
The two compose, and keys are checked first, so a missing key is still reported
|
|
138
|
-
as a missing key rather than as whatever the schema calls it.
|
|
132
|
+
Any [Standard Schema](https://standardschema.dev) validator works — Zod 4,
|
|
133
|
+
Valibot, ArkType — and the spec is types-only, so this still costs **no
|
|
134
|
+
dependency**. Details, and the one case that throws:
|
|
135
|
+
[docs/detectors.md](docs/detectors.md#structured-output)
|
|
139
136
|
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
> async refinement throws a `TypeError` telling you so, rather than silently
|
|
143
|
-
> passing. Everything Zod, Valibot and ArkType produce otherwise is synchronous.
|
|
144
|
-
> This is the one thing in the package that throws about your configuration; it
|
|
145
|
-
> still never throws about a response.
|
|
146
|
-
|
|
147
|
-
See **[docs/detectors.md](docs/detectors.md)** for arrays of repeated records —
|
|
148
|
-
a JSON array of identical rows reads as a loop under the default scope, and
|
|
149
|
-
`redundancyScope: 'jsonValues'` is the fix.
|
|
150
|
-
|
|
151
|
-
### Answering in the wrong language
|
|
137
|
+
**Answered in the wrong language** — a model that ignores "answer in English"
|
|
138
|
+
returns fluent Chinese, not broken English:
|
|
152
139
|
|
|
153
140
|
```ts
|
|
154
141
|
checkOutput(raw, { ...presets.chat, expectScript: 'latin' });
|
|
155
142
|
```
|
|
156
143
|
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
144
|
+
Detectable by counting characters — no word list, decisive from about a dozen
|
|
145
|
+
letters. A wrong-script answer scores **1.000**; a healthy one measured against
|
|
146
|
+
its own script scores **0.000–0.028**. Pass every script the answer may
|
|
147
|
+
legitimately contain (`['han', 'latin']` for Chinese). Code fences and URLs are
|
|
148
|
+
excluded, so a TypeScript block never counts as answering in English.
|
|
149
|
+
[More →](docs/detectors.md#answering-in-the-wrong-language)
|
|
163
150
|
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
151
|
+
**Returned your prompt instead of an answer** — invisible to every other
|
|
152
|
+
detector here, because such a response is fluent, well-formed and the right
|
|
153
|
+
length:
|
|
154
|
+
|
|
155
|
+
```ts
|
|
156
|
+
checkOutput(raw, { ...presets.chat, prompt });
|
|
157
|
+
```
|
|
169
158
|
|
|
170
|
-
|
|
171
|
-
|
|
159
|
+
Not for rewrite, translate or summarise endpoints, where copying the input is
|
|
160
|
+
the job. [More →](docs/detectors.md#returning-the-prompt-instead-of-an-answer)
|
|
172
161
|
|
|
173
162
|
## The verdict
|
|
174
163
|
|
|
@@ -185,26 +174,26 @@ Same script means no signal: Spanish against English scores 0. That is what
|
|
|
185
174
|
}
|
|
186
175
|
```
|
|
187
176
|
|
|
188
|
-
|
|
177
|
+
`reasons` carries every failing signal, not just the first. `scores` includes the
|
|
178
|
+
passing detectors too — send them to your metrics and you will know your real
|
|
179
|
+
degeneration rate within a day.
|
|
189
180
|
|
|
190
|
-
`modes` says which tokenizer produced a score
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
gives you a number that describes neither.
|
|
181
|
+
`modes` says which tokenizer produced a score. **Log it next to `scores`:**
|
|
182
|
+
`TAIL_LOOP` measures words on spaced scripts and characters on Chinese, Japanese
|
|
183
|
+
and Thai, and pooling two distributions into one histogram gives you a number
|
|
184
|
+
that describes neither.
|
|
195
185
|
|
|
196
186
|
## Presets
|
|
197
187
|
|
|
198
188
|
`chat` · `strictJson` · `longForm` · `lenient`
|
|
199
189
|
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
---
|
|
190
|
+
Starting points calibrated against this repo's fixture corpus — not universal
|
|
191
|
+
truths.
|
|
203
192
|
|
|
204
|
-
##
|
|
193
|
+
## Calibrate against your own traffic
|
|
205
194
|
|
|
206
|
-
The shipped presets are tuned on
|
|
207
|
-
|
|
195
|
+
The shipped presets are tuned on a corpus that is **not your traffic**. Log your
|
|
196
|
+
scores for a week, then derive thresholds you can defend:
|
|
208
197
|
|
|
209
198
|
```bash
|
|
210
199
|
npx llm-output-guard calibrate scores.jsonl --fpr 0.001
|
|
@@ -212,44 +201,33 @@ npx llm-output-guard calibrate scores.jsonl --fpr 0.001
|
|
|
212
201
|
|
|
213
202
|
The report tells you when your sample is too small to support the rate you asked
|
|
214
203
|
for, and distinguishes real separation in your data from a false-positive budget —
|
|
215
|
-
because only one of those is evidence.
|
|
216
|
-
**[docs/calibration.md](docs/calibration.md)
|
|
217
|
-
|
|
218
|
-
## Script coverage
|
|
219
|
-
|
|
220
|
-
Korean, Cyrillic, Greek, Arabic and Devanagari separate words and are handled like
|
|
221
|
-
English. **Chinese, Japanese and Thai do not**, so `TAIL_LOOP` switches to
|
|
222
|
-
character mode and reads its own threshold. `REPETITION` is blind on those scripts —
|
|
223
|
-
a known, measured gap. `SCRIPT_MISMATCH` covers all ten scripts and is the one
|
|
224
|
-
detector these are *not* the weak case for. Numbers behind both in
|
|
225
|
-
**[docs/script-coverage.md](docs/script-coverage.md)**.
|
|
204
|
+
because only one of those is evidence.
|
|
205
|
+
**[docs/calibration.md](docs/calibration.md)**
|
|
226
206
|
|
|
227
207
|
## Design notes
|
|
228
208
|
|
|
229
|
-
- **Zero runtime dependencies**, enforced in CI. Node ≥ 18
|
|
230
|
-
- **Hand-rolled LZ77** rather than `node:zlib`, so the package stays runtime-agnostic. It is not a real compressor; it only needs to move monotonically with redundancy.
|
|
209
|
+
- **Zero runtime dependencies**, enforced in CI. Node ≥ 18; works on edge, browser, Deno, Bun.
|
|
231
210
|
- **Pure and synchronous.** No network, no clock, no randomness — safe on a hot path, trivial to test.
|
|
232
211
|
- **Scores, not booleans.** Detectors report 0–1 and leave the threshold decision to you.
|
|
233
212
|
- **Abstains rather than guesses.** Samples too short to judge score 0.
|
|
213
|
+
- **Hand-rolled LZ77** rather than `node:zlib`, so the package stays runtime-agnostic.
|
|
214
|
+
- **Sub-millisecond**, 0.383 ms at 500 B and 1.014 ms at 32 KB, with one detector accounting for most of it. `npm run bench` reproduces it — **[docs/performance.md](docs/performance.md)**
|
|
215
|
+
- **Chinese, Japanese and Thai** are handled where they differ: `TAIL_LOOP` switches to character mode, `REPETITION` is blind and says so — **[docs/script-coverage.md](docs/script-coverage.md)**
|
|
234
216
|
|
|
235
217
|
## Stability
|
|
236
218
|
|
|
237
|
-
What semver means
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
`
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
behaviour they carry. The list is asserted in `test/surface.test.ts`, so an
|
|
250
|
-
export cannot join it by accident.
|
|
251
|
-
|
|
252
|
-
**Threshold and preset values are behaviour, not implementation.** This is the
|
|
219
|
+
What semver means here specifically. These rules bind from **1.0.0** onward, and
|
|
220
|
+
the public surface was frozen export by export in that release.
|
|
221
|
+
|
|
222
|
+
**The public API is** everything exported from `llm-output-guard`, plus
|
|
223
|
+
`outputGuard` / `OutputGuardOptions` / `DegenerateAction` from `./ai-sdk` and
|
|
224
|
+
`withOutputGuard` / `OutputGuardOptions` / `DegenerateAction` from `./openai` and
|
|
225
|
+
`./anthropic`. Each subpath is its own contract, so an option added to one is not
|
|
226
|
+
a promise about the others. Anything else is internal and may move in any
|
|
227
|
+
release. The list is asserted in `test/surface.test.ts`, so an export cannot join
|
|
228
|
+
it by accident.
|
|
229
|
+
|
|
230
|
+
**Threshold and preset values are behaviour, not implementation.** That is the
|
|
253
231
|
interesting case, so it gets a rule of its own:
|
|
254
232
|
|
|
255
233
|
| Change | Release type |
|
|
@@ -259,42 +237,38 @@ interesting case, so it gets a rule of its own:
|
|
|
259
237
|
| Adding a new *option*, defaulted so nothing changes | minor |
|
|
260
238
|
| Adding a new detector that is opt-in | minor |
|
|
261
239
|
| Making an existing detector strictly more accurate on its own axis | minor |
|
|
262
|
-
| Docs, internals, performance, fixing a detector that
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
**
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
version range upgrades into automatically. It was withdrawn from npm within the
|
|
282
|
-
72-hour unpublish window and re-released as 0.5.0, where `^0.4.1` correctly
|
|
283
|
-
resolves away from it. The rule it broke is the one in the table above: threshold
|
|
284
|
-
and preset changes are behaviour changes, and behaviour changes are never
|
|
285
|
-
patches.
|
|
240
|
+
| Docs, internals, performance, fixing a detector that returned a wrong score | patch |
|
|
241
|
+
|
|
242
|
+
A threshold change does not break your build. It changes which of your production
|
|
243
|
+
responses get discarded and retried, which is a larger event than a signature
|
|
244
|
+
change and invisible until your traffic hits it. A number in `presets.chat` is
|
|
245
|
+
part of the contract in the same way a function name is.
|
|
246
|
+
|
|
247
|
+
**Not covered:** the exact scores a detector returns (only their direction and
|
|
248
|
+
the thresholds acting on them), the fixture corpus, `message` strings in
|
|
249
|
+
`Reason`, or the human-readable `calibrate` report. `--json` output *is* covered.
|
|
250
|
+
|
|
251
|
+
**Peer ranges** narrow only in a major, and are verified rather than assumed —
|
|
252
|
+
`npm run check:peers` installs the packed tarball against each end of each
|
|
253
|
+
declared range, then typechecks and runs the adapter.
|
|
254
|
+
|
|
255
|
+
**Why this is written down:** 0.4.2 shipped a peer narrowing, a new subpath, and
|
|
256
|
+
a behaviour change under a **patch** number, which every default version range
|
|
257
|
+
upgrades into automatically. It was unpublished within the 72-hour window and
|
|
258
|
+
re-released as 0.5.0. The rule it broke is the one in the table above.
|
|
286
259
|
|
|
287
260
|
## Limitations
|
|
288
261
|
|
|
289
262
|
- Not a hallucination detector. It measures *shape*, never truth.
|
|
290
|
-
-
|
|
291
|
-
- `
|
|
292
|
-
- `
|
|
293
|
-
-
|
|
294
|
-
-
|
|
295
|
-
-
|
|
296
|
-
-
|
|
297
|
-
-
|
|
263
|
+
- `REPETITION` does not work on Chinese, Japanese or Thai — a known, measured gap, not an oversight.
|
|
264
|
+
- `LANG_MISMATCH` is a function-word heuristic covering `id`/`en`/`es`, and is unreliable under 25 words. `expectScript` is the stronger check wherever the alphabets differ.
|
|
265
|
+
- `PROMPT_ECHO` cannot tell a degenerate echo from a rewrite or translation — the difference is in what you asked for, not in the text.
|
|
266
|
+
- `SCRIPT_MISMATCH` and `PROMPT_ECHO` do not run mid-stream: both measure a property of the whole response, and a mid-stream check reads a trailing window.
|
|
267
|
+
- Tool *arguments* are measured only with `checkToolArguments: true`, non-streaming responses only.
|
|
268
|
+
- `openai`'s `responses.stream()` helper is not wrapped; `create({ stream: true })` is.
|
|
269
|
+
- Truncation from a missing full stop is weak evidence, scored 0.55 and deliberately left below the defaults. Lower `maxTruncation` to ~0.5 to catch it, and expect false positives.
|
|
270
|
+
- A JSON array of repeated identical records reads as a loop and fails from three records up. Set `redundancyScope: 'jsonValues'`.
|
|
271
|
+
- Thresholds are calibrated on the bundled corpus. Yours will differ — and the word and character thresholds need calibrating **separately**.
|
|
298
272
|
|
|
299
273
|
## License
|
|
300
274
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { V as Verdict } from './stream-
|
|
1
|
+
import { V as Verdict } from './stream-BSCIXfIH.cjs';
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
4
|
* The option surface every provider adapter shares.
|
|
@@ -62,6 +62,26 @@ interface AdapterGuardOptions {
|
|
|
62
62
|
onVerdict?: (verdict: Verdict, context: {
|
|
63
63
|
streaming: boolean;
|
|
64
64
|
}) => void;
|
|
65
|
+
/**
|
|
66
|
+
* Also measure the arguments the model passed to a tool. Default `false`.
|
|
67
|
+
*
|
|
68
|
+
* A tool-calling turn is judged by its preamble, because the text beside a
|
|
69
|
+
* tool call is not the answer -- and that leaves the answer itself
|
|
70
|
+
* unmeasured. Your provider validates arguments against the schema you gave
|
|
71
|
+
* it, which covers types and not content: `{ "query": "site reliability site
|
|
72
|
+
* reliability site reliability ..." }` is a schema-valid string, and it is
|
|
73
|
+
* still a garbage query.
|
|
74
|
+
*
|
|
75
|
+
* Switching this on measures each string value of each call's arguments for
|
|
76
|
+
* redundancy, and folds the result into the same verdict the preamble
|
|
77
|
+
* produces. Reason codes are unchanged, so existing handling works; the
|
|
78
|
+
* `message` says the loop was found in an argument.
|
|
79
|
+
*
|
|
80
|
+
* **Non-streaming responses only.** Arguments arrive as JSON fragments that
|
|
81
|
+
* do not parse until the call is complete, so there is nothing meaningful to
|
|
82
|
+
* measure mid-stream.
|
|
83
|
+
*/
|
|
84
|
+
checkToolArguments?: boolean;
|
|
65
85
|
}
|
|
66
86
|
|
|
67
87
|
export type { AdapterGuardOptions as A, DegenerateAction as D };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { V as Verdict } from './stream-
|
|
1
|
+
import { V as Verdict } from './stream-BSCIXfIH.js';
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
4
|
* The option surface every provider adapter shares.
|
|
@@ -62,6 +62,26 @@ interface AdapterGuardOptions {
|
|
|
62
62
|
onVerdict?: (verdict: Verdict, context: {
|
|
63
63
|
streaming: boolean;
|
|
64
64
|
}) => void;
|
|
65
|
+
/**
|
|
66
|
+
* Also measure the arguments the model passed to a tool. Default `false`.
|
|
67
|
+
*
|
|
68
|
+
* A tool-calling turn is judged by its preamble, because the text beside a
|
|
69
|
+
* tool call is not the answer -- and that leaves the answer itself
|
|
70
|
+
* unmeasured. Your provider validates arguments against the schema you gave
|
|
71
|
+
* it, which covers types and not content: `{ "query": "site reliability site
|
|
72
|
+
* reliability site reliability ..." }` is a schema-valid string, and it is
|
|
73
|
+
* still a garbage query.
|
|
74
|
+
*
|
|
75
|
+
* Switching this on measures each string value of each call's arguments for
|
|
76
|
+
* redundancy, and folds the result into the same verdict the preamble
|
|
77
|
+
* produces. Reason codes are unchanged, so existing handling works; the
|
|
78
|
+
* `message` says the loop was found in an argument.
|
|
79
|
+
*
|
|
80
|
+
* **Non-streaming responses only.** Arguments arrive as JSON fragments that
|
|
81
|
+
* do not parse until the call is complete, so there is nothing meaningful to
|
|
82
|
+
* measure mid-stream.
|
|
83
|
+
*/
|
|
84
|
+
checkToolArguments?: boolean;
|
|
65
85
|
}
|
|
66
86
|
|
|
67
87
|
export type { AdapterGuardOptions as A, DegenerateAction as D };
|
package/dist/ai-sdk.cjs
CHANGED
|
@@ -282,6 +282,37 @@ function judgeableLetters(text, options) {
|
|
|
282
282
|
return found.filter((ch) => !SCRIPT_NEUTRAL.test(ch));
|
|
283
283
|
}
|
|
284
284
|
|
|
285
|
+
// src/detectors/prompt-echo.ts
|
|
286
|
+
function promptEchoDetail(text, prompt, options = {}) {
|
|
287
|
+
const {
|
|
288
|
+
n = 5,
|
|
289
|
+
charN = 12,
|
|
290
|
+
minTokens = 40,
|
|
291
|
+
nonSpacedCutoff = 0.5,
|
|
292
|
+
maxSample = 8e3
|
|
293
|
+
} = options;
|
|
294
|
+
const output = text.slice(0, maxSample);
|
|
295
|
+
const mode = options.mode ?? tokenModeOf(output, nonSpacedCutoff);
|
|
296
|
+
if (typeof prompt !== "string" || prompt.length === 0) return { score: 0, mode };
|
|
297
|
+
const tokenize = mode === "char" ? chars : words;
|
|
298
|
+
const run = mode === "char" ? charN : n;
|
|
299
|
+
const out = tokenize(output);
|
|
300
|
+
const src = tokenize(prompt.slice(0, maxSample));
|
|
301
|
+
if (out.length < minTokens) return { score: 0, mode };
|
|
302
|
+
if (out.length < run || src.length < run) return { score: 0, mode };
|
|
303
|
+
const seen = /* @__PURE__ */ new Set();
|
|
304
|
+
for (let i = 0; i + run <= src.length; i++) {
|
|
305
|
+
seen.add(src.slice(i, i + run).join(" "));
|
|
306
|
+
}
|
|
307
|
+
let matched = 0;
|
|
308
|
+
let total = 0;
|
|
309
|
+
for (let i = 0; i + run <= out.length; i++) {
|
|
310
|
+
if (seen.has(out.slice(i, i + run).join(" "))) matched++;
|
|
311
|
+
total++;
|
|
312
|
+
}
|
|
313
|
+
return { score: total === 0 ? 0 : clamp01(matched / total), mode };
|
|
314
|
+
}
|
|
315
|
+
|
|
285
316
|
// src/internal/json-scope.ts
|
|
286
317
|
function stringValues(value, out = []) {
|
|
287
318
|
if (typeof value === "string") out.push(value);
|
|
@@ -316,6 +347,7 @@ var DEFAULTS = {
|
|
|
316
347
|
allowJsonFence: true,
|
|
317
348
|
maxLangMismatch: 0.6,
|
|
318
349
|
maxScriptMismatch: 0.5,
|
|
350
|
+
maxPromptEcho: 0.6,
|
|
319
351
|
ngram: 3,
|
|
320
352
|
redundancyScope: "document"
|
|
321
353
|
};
|
|
@@ -423,6 +455,18 @@ function checkOutput(text, options = {}) {
|
|
|
423
455
|
`${Math.round(s * 100)}% of letters are not in ${wanted.join(" or ")}.`
|
|
424
456
|
);
|
|
425
457
|
}
|
|
458
|
+
if (opts.prompt) {
|
|
459
|
+
const { score, mode } = promptEchoDetail(text, opts.prompt, {
|
|
460
|
+
nonSpacedCutoff: opts.nonSpacedCutoff
|
|
461
|
+
});
|
|
462
|
+
add(
|
|
463
|
+
"PROMPT_ECHO",
|
|
464
|
+
score,
|
|
465
|
+
opts.maxPromptEcho,
|
|
466
|
+
`${Math.round(score * 100)}% of the response is copied from the prompt.`,
|
|
467
|
+
mode
|
|
468
|
+
);
|
|
469
|
+
}
|
|
426
470
|
if (opts.expectLang) {
|
|
427
471
|
const s = languageMismatchScore(text, opts.expectLang);
|
|
428
472
|
add(
|
|
@@ -482,6 +526,19 @@ var DEFERRED_TO_END = {
|
|
|
482
526
|
* and this is one line for the callers who need it.
|
|
483
527
|
*/
|
|
484
528
|
expectScript: null,
|
|
529
|
+
/*
|
|
530
|
+
* PROMPT_ECHO is deferred for the same reason as SCRIPT_MISMATCH, and it
|
|
531
|
+
* bites harder. The score is the share of the *whole* output copied from the
|
|
532
|
+
* prompt, so a trailing window measures the share of that window -- and a
|
|
533
|
+
* response that opens by echoing the system prompt and then answers properly
|
|
534
|
+
* reads as 1.000 over its first window and 0.000 over its last. Neither
|
|
535
|
+
* number describes the response.
|
|
536
|
+
*
|
|
537
|
+
* Dilution is the point of the measure: an output that is 10% leaked prompt
|
|
538
|
+
* and 90% answer should score 0.1, and only the finished response knows what
|
|
539
|
+
* that ratio is.
|
|
540
|
+
*/
|
|
541
|
+
prompt: null,
|
|
485
542
|
/*
|
|
486
543
|
* LOW_ENTROPY is deferred for a second reason: cost. The LZ77 pass is
|
|
487
544
|
* 0.4ms at 500 characters and 11ms at its 4000-character sample cap, which
|
|
@@ -557,6 +614,60 @@ function checkPreamble(text, options) {
|
|
|
557
614
|
return checkOutput(text, { ...options, ...TOOL_CALL_PREAMBLE });
|
|
558
615
|
}
|
|
559
616
|
|
|
617
|
+
// src/internal/tool-arguments.ts
|
|
618
|
+
var ARGUMENT_SCOPE = {
|
|
619
|
+
minLength: 0,
|
|
620
|
+
maxTruncation: null,
|
|
621
|
+
maxCompressibility: null,
|
|
622
|
+
expectJson: false,
|
|
623
|
+
expectLang: null,
|
|
624
|
+
expectScript: null,
|
|
625
|
+
finishReason: void 0,
|
|
626
|
+
redundancyScope: "jsonValues"
|
|
627
|
+
};
|
|
628
|
+
function argumentsToText(args) {
|
|
629
|
+
if (typeof args === "string") return args;
|
|
630
|
+
if (args === null || args === void 0) return "";
|
|
631
|
+
try {
|
|
632
|
+
return JSON.stringify(args) ?? "";
|
|
633
|
+
} catch {
|
|
634
|
+
return "";
|
|
635
|
+
}
|
|
636
|
+
}
|
|
637
|
+
function checkArguments(calls, options) {
|
|
638
|
+
const texts = calls.map(argumentsToText).filter(hasMeasurableContent);
|
|
639
|
+
if (texts.length === 0) return null;
|
|
640
|
+
let worst = null;
|
|
641
|
+
for (const text of texts) {
|
|
642
|
+
const verdict = checkOutput(text, { ...options, ...ARGUMENT_SCOPE });
|
|
643
|
+
if (!worst || worst.ok && !verdict.ok) worst = verdict;
|
|
644
|
+
else if (!verdict.ok && !worst.ok && maxScore(verdict) > maxScore(worst)) worst = verdict;
|
|
645
|
+
}
|
|
646
|
+
return worst;
|
|
647
|
+
}
|
|
648
|
+
var maxScore = (verdict) => verdict.reasons.reduce((hi, reason) => Math.max(hi, reason.score), 0);
|
|
649
|
+
var hasMeasurableContent = (text) => redundancySpans(text, "jsonValues").some((span) => span.trim().length > 0);
|
|
650
|
+
function mergeVerdicts(preamble, args) {
|
|
651
|
+
if (!args) return preamble;
|
|
652
|
+
const labelled = args.reasons.map((reason) => ({
|
|
653
|
+
...reason,
|
|
654
|
+
message: `In a tool call argument: ${reason.message}`
|
|
655
|
+
}));
|
|
656
|
+
if (!preamble) return { ...args, reasons: labelled };
|
|
657
|
+
const scores = { ...preamble.scores };
|
|
658
|
+
for (const [code, score] of Object.entries(args.scores)) {
|
|
659
|
+
const key = code;
|
|
660
|
+
if (scores[key] === void 0 || score > scores[key]) scores[key] = score;
|
|
661
|
+
}
|
|
662
|
+
const reasons = [...preamble.reasons, ...labelled];
|
|
663
|
+
return {
|
|
664
|
+
...preamble,
|
|
665
|
+
ok: reasons.length === 0,
|
|
666
|
+
reasons,
|
|
667
|
+
scores
|
|
668
|
+
};
|
|
669
|
+
}
|
|
670
|
+
|
|
560
671
|
// src/ai-sdk.ts
|
|
561
672
|
var isToolPart = (part) => part.type.startsWith("tool-");
|
|
562
673
|
function finishReasonOf(value) {
|
|
@@ -565,7 +676,7 @@ function finishReasonOf(value) {
|
|
|
565
676
|
return void 0;
|
|
566
677
|
}
|
|
567
678
|
function outputGuard(options = {}) {
|
|
568
|
-
const { onDegenerate = "throw", onVerdict, ...guardOptions } = options;
|
|
679
|
+
const { onDegenerate = "throw", onVerdict, checkToolArguments = false, ...guardOptions } = options;
|
|
569
680
|
const act = (verdict, streaming) => {
|
|
570
681
|
onVerdict?.(verdict, { streaming });
|
|
571
682
|
if (verdict.ok || onDegenerate === "ignore") return;
|
|
@@ -599,7 +710,12 @@ function outputGuard(options = {}) {
|
|
|
599
710
|
const content = result.content ?? [];
|
|
600
711
|
const text = content.filter((part) => part.type === "text").map((part) => part.text ?? "").join("");
|
|
601
712
|
if (content.some(isToolPart)) {
|
|
602
|
-
const
|
|
713
|
+
const preamble = checkPreamble(text, guardOptions);
|
|
714
|
+
const args = checkToolArguments ? checkArguments(
|
|
715
|
+
content.filter((part) => part.type === "tool-call").map((part) => part.input ?? part.args),
|
|
716
|
+
guardOptions
|
|
717
|
+
) : null;
|
|
718
|
+
const verdict = mergeVerdicts(preamble, args);
|
|
603
719
|
if (verdict) act(verdict, false);
|
|
604
720
|
return result;
|
|
605
721
|
}
|