llm-output-guard 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +303 -21
- package/dist/adapter-options-DQQaELAX.d.cts +67 -0
- package/dist/adapter-options-DafMCHtz.d.ts +67 -0
- package/dist/ai-sdk.cjs +95 -20
- package/dist/ai-sdk.cjs.map +1 -1
- package/dist/ai-sdk.d.cts +26 -38
- package/dist/ai-sdk.d.ts +26 -38
- package/dist/ai-sdk.js +17 -1
- package/dist/ai-sdk.js.map +1 -1
- package/dist/{cli.cjs → bin.cjs} +59 -30
- package/dist/bin.cjs.map +1 -0
- package/dist/{cli.js → bin.js} +59 -29
- package/dist/bin.js.map +1 -0
- package/dist/{chunk-KJNWKUM7.js → chunk-P3VLZO6Y.js} +85 -23
- package/dist/chunk-P3VLZO6Y.js.map +1 -0
- package/dist/index.cjs +116 -29
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +120 -10
- package/dist/index.d.ts +120 -10
- package/dist/index.js +34 -10
- package/dist/index.js.map +1 -1
- package/dist/openai.cjs +493 -0
- package/dist/openai.cjs.map +1 -0
- package/dist/openai.d.cts +32 -0
- package/dist/openai.d.ts +32 -0
- package/dist/openai.js +88 -0
- package/dist/openai.js.map +1 -0
- package/dist/{stream-mT0r7Yhq.d.ts → stream-Be2WhoKn.d.cts} +47 -2
- package/dist/{stream-mT0r7Yhq.d.cts → stream-Be2WhoKn.d.ts} +47 -2
- package/package.json +15 -3
- package/dist/chunk-KJNWKUM7.js.map +0 -1
- package/dist/cli.cjs.map +0 -1
- package/dist/cli.js.map +0 -1
package/README.md
CHANGED
|
@@ -102,21 +102,49 @@ for await (const chunk of guardStream(stream, {
|
|
|
102
102
|
}
|
|
103
103
|
```
|
|
104
104
|
|
|
105
|
-
Against the degenerate fixtures,
|
|
106
|
-
|
|
107
|
-
produced:
|
|
105
|
+
Against the degenerate fixtures, the guard reports a failure after **8-52%** of
|
|
106
|
+
each fixture's characters:
|
|
108
107
|
|
|
109
108
|
```
|
|
110
|
-
repetition-word-stutter caught at 240/2999 chars -> 92%
|
|
111
|
-
repetition-clause-loop caught at 240/1680 chars -> 86%
|
|
112
|
-
tail-loop-after-good-start caught at 640/1569 chars -> 59%
|
|
113
|
-
tail-loop-trailing-phrase caught at 640/1238 chars -> 48%
|
|
109
|
+
repetition-word-stutter caught at 240/2999 chars -> 92% not yet read
|
|
110
|
+
repetition-clause-loop caught at 240/1680 chars -> 86% not yet read
|
|
111
|
+
tail-loop-after-good-start caught at 640/1569 chars -> 59% not yet read
|
|
112
|
+
tail-loop-trailing-phrase caught at 640/1238 chars -> 48% not yet read
|
|
114
113
|
```
|
|
115
114
|
|
|
115
|
+
**Read that as detection latency, not as a saving.** It is measured by feeding
|
|
116
|
+
fixture strings to `createStreamGuard` in-process — there is no provider and no
|
|
117
|
+
connection involved, so it says how early the signal is available and nothing
|
|
118
|
+
about tokens or cost. What you do with the signal is the part that saves money,
|
|
119
|
+
and how much it saves depends on your provider.
|
|
120
|
+
|
|
116
121
|
Zero of the healthy fixtures trip it, and the watching costs **~0.05ms per
|
|
117
122
|
check** — around 0.7ms across a 5,500 character response, flat as the stream
|
|
118
123
|
grows rather than quadratic in its length.
|
|
119
124
|
|
|
125
|
+
**Those numbers are measured on Latin-script fixtures and do not carry over
|
|
126
|
+
unchanged.** For Chinese, Japanese and Thai the same in-process measurement
|
|
127
|
+
gives **0-85%**, and the spread is the whole story:
|
|
128
|
+
|
|
129
|
+
```
|
|
130
|
+
cjk-tail-loop-th-nopunct caught at 240/1640 chars -> 85% not yet read
|
|
131
|
+
cjk-tail-loop-ja-nopunct caught at 240/800 chars -> 70% not yet read
|
|
132
|
+
cjk-tail-loop-zh-nopunct caught at 240/640 chars -> 63% not yet read
|
|
133
|
+
cjk-tail-loop-diluted caught at 1840/2303 chars -> 20% not yet read
|
|
134
|
+
cjk-tail-loop-short never mid-stream (128 chars, under the warmup)
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
A response that loops from the start saves what a Latin one saves. A response
|
|
138
|
+
that answers properly and *then* falls into a Chinese loop is caught late,
|
|
139
|
+
because there is nothing to detect until the loop begins — 20% on that fixture,
|
|
140
|
+
and less on a longer healthy prefix. Responses shorter than the 240-character
|
|
141
|
+
warmup are never judged mid-stream at all; they are caught by `end()`, after you
|
|
142
|
+
have paid for them.
|
|
143
|
+
|
|
144
|
+
Late detection is still worth having: it stops a broken response being cached,
|
|
145
|
+
returned, or counted as a success, which is the reason this package exists. It
|
|
146
|
+
is just not the token saving, and you should not budget for one.
|
|
147
|
+
|
|
120
148
|
For manual control over the loop, use the primitive:
|
|
121
149
|
|
|
122
150
|
```ts
|
|
@@ -147,9 +175,11 @@ const model = wrapLanguageModel({
|
|
|
147
175
|
```
|
|
148
176
|
|
|
149
177
|
On `streamText` this cancels the provider's stream mid-generation. Driven
|
|
150
|
-
through the real SDK
|
|
151
|
-
of 137 parts** before the guard cut it off
|
|
152
|
-
|
|
178
|
+
through the real SDK over a **mock part stream**, the source was pulled for **17
|
|
179
|
+
of 137 parts** before the guard cut it off. That figure is parts never requested
|
|
180
|
+
from a stub, not tokens never billed by a provider: the SDK's cancellation path
|
|
181
|
+
is exercised for real, the thing on the other end of it is not. On
|
|
182
|
+
`generateText` the tokens are already bought, so it throws
|
|
153
183
|
`DegenerateOutputError` instead, which your fallback layer can act on.
|
|
154
184
|
|
|
155
185
|
`onDegenerate` takes `'throw'` (default, also cancels the stream), `'abort'`
|
|
@@ -165,7 +195,90 @@ outputGuard({
|
|
|
165
195
|
```
|
|
166
196
|
|
|
167
197
|
`ai` is an **optional peer dependency** — importing the subpath does not pull it
|
|
168
|
-
in, and the main entry point has no peers at all.
|
|
198
|
+
in, and the main entry point has no peers at all. Supported: **`ai` v5, v6 and
|
|
199
|
+
v7**. CI installs the packed tarball against each of those and both typechecks
|
|
200
|
+
and runs the adapter, so the range is one that has been executed rather than
|
|
201
|
+
assumed.
|
|
202
|
+
|
|
203
|
+
**`ai` v4 is not supported, and forcing it will look like a bug in this
|
|
204
|
+
package.** v4's middleware hands back `text` where v5+ hands back a `content`
|
|
205
|
+
array, and streams `{ textDelta }` where v5+ streams `{ delta }`. This adapter
|
|
206
|
+
reads the v5+ shape, so on v4 it sees the empty string for every response —
|
|
207
|
+
which means **every healthy generation is flagged `EMPTY`, and under the default
|
|
208
|
+
`onDegenerate: 'throw'` every call throws `DegenerateOutputError`.** It is not
|
|
209
|
+
that the guard misses things on v4; it rejects everything. The peer range now
|
|
210
|
+
refuses the install so you find out at `npm install` rather than in production.
|
|
211
|
+
If you are pinned to v4, do not override it — stay on the core entry point and
|
|
212
|
+
call `checkOutput` on the result yourself.
|
|
213
|
+
|
|
214
|
+
### OpenAI SDK — and anything speaking its protocol
|
|
215
|
+
|
|
216
|
+
One wrap, and both call shapes are guarded:
|
|
217
|
+
|
|
218
|
+
```ts
|
|
219
|
+
import OpenAI from 'openai';
|
|
220
|
+
import { withOutputGuard } from 'llm-output-guard/openai';
|
|
221
|
+
import { presets } from 'llm-output-guard';
|
|
222
|
+
|
|
223
|
+
const client = withOutputGuard(new OpenAI(), {
|
|
224
|
+
...presets.chat,
|
|
225
|
+
onDegenerate: 'abort',
|
|
226
|
+
});
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
This is also how you guard **Groq, Together, OpenRouter, Fireworks, DeepInfra,
|
|
230
|
+
vLLM and Ollama** — anything you reach through an OpenAI-compatible `baseURL`
|
|
231
|
+
works, because the adapter is typed against the chat-completions shape rather
|
|
232
|
+
than against OpenAI the company.
|
|
233
|
+
|
|
234
|
+
Non-streaming calls are already paid for by the time anything can run, so a
|
|
235
|
+
degenerate one throws `DegenerateOutputError` for your fallback layer to catch:
|
|
236
|
+
|
|
237
|
+
```ts
|
|
238
|
+
const completion = await client.chat.completions.create({ model, messages });
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
Streaming is where it pays. The guard watches deltas and **cancels the HTTP
|
|
242
|
+
request** the moment a loop is detectable:
|
|
243
|
+
|
|
244
|
+
```ts
|
|
245
|
+
const stream = await client.chat.completions.create({ model, messages, stream: true });
|
|
246
|
+
for await (const chunk of stream) process.stdout.write(chunk.choices[0]?.delta?.content ?? '');
|
|
247
|
+
```
|
|
248
|
+
|
|
249
|
+
Driven through the real SDK against a looping model over a **mock transport**,
|
|
250
|
+
the response body was cancelled after **16 of 135 chunks** were generated — 88%
|
|
251
|
+
of the chunks were never produced.
|
|
252
|
+
|
|
253
|
+
**What that number is, precisely:** chunks a mock server was never asked to
|
|
254
|
+
produce after the client closed the connection, measured against an unguarded
|
|
255
|
+
baseline of the full 135. It is stronger than a "did abort fire" assertion —
|
|
256
|
+
the test observes cancellation at the response body, so a guard that stopped
|
|
257
|
+
iterating while the connection stayed open would fail it. It is **not** a
|
|
258
|
+
billing figure. A real provider sits behind buffering, its own chunking, and
|
|
259
|
+
server-side generation that may already have run ahead of what it has sent;
|
|
260
|
+
none of that exists in the mock. Treat 88% as evidence that cancellation
|
|
261
|
+
reaches the transport promptly, and measure your own provider before putting a
|
|
262
|
+
number in a budget.
|
|
263
|
+
|
|
264
|
+
`onDegenerate` and `onVerdict` are the same options as the Vercel adapter, from
|
|
265
|
+
the same type — `'throw'` (default, also cancels the stream), `'abort'` (stop
|
|
266
|
+
cleanly, keep what arrived), or `'ignore'`. Start with `'ignore'` plus
|
|
267
|
+
`onVerdict` to watch your own traffic first:
|
|
268
|
+
|
|
269
|
+
```ts
|
|
270
|
+
withOutputGuard(new OpenAI(), {
|
|
271
|
+
...presets.chat,
|
|
272
|
+
onDegenerate: 'ignore',
|
|
273
|
+
onVerdict: (verdict, { streaming }) =>
|
|
274
|
+
metrics.record(verdict.scores, { streaming, modes: verdict.modes }),
|
|
275
|
+
});
|
|
276
|
+
```
|
|
277
|
+
|
|
278
|
+
`openai` is an **optional peer dependency**, and the wrapper is a proxy: every
|
|
279
|
+
other method on the client, and `create()`'s own `.withResponse()`, pass through
|
|
280
|
+
untouched. `finish_reason: 'length'` is mapped into the final check, so
|
|
281
|
+
`TRUNCATED` fires on a response that hit `max_tokens`.
|
|
169
282
|
|
|
170
283
|
**What runs when.** Mid-stream only the redundancy detectors are meaningful:
|
|
171
284
|
partial output is genuinely short, genuinely cut off, and genuinely not valid
|
|
@@ -173,7 +286,10 @@ JSON, so `TOO_SHORT`, `TRUNCATED`, `INVALID_JSON` and `LANG_MISMATCH` would
|
|
|
173
286
|
fire on every healthy generation and teach you to ignore the guard. They are
|
|
174
287
|
deferred to `end()`. `LOW_ENTROPY` is deferred too, for cost — it is ~100x the
|
|
175
288
|
other detectors, and everything it would have caught early is caught by
|
|
176
|
-
`REPETITION`
|
|
289
|
+
`REPETITION`, or by `TAIL_LOOP`'s character mode on non-spaced scripts.
|
|
290
|
+
|
|
291
|
+
Both adapters share this behaviour because both drive the same
|
|
292
|
+
`createStreamGuard`. Neither reimplements it.
|
|
177
293
|
|
|
178
294
|
---
|
|
179
295
|
|
|
@@ -184,22 +300,30 @@ other detectors, and everything it would have caught early is caught by
|
|
|
184
300
|
ok: false,
|
|
185
301
|
reasons: [
|
|
186
302
|
{ code: 'REPETITION', score: 0.83, threshold: 0.4, message: '83% of 3-grams are duplicates.' },
|
|
187
|
-
{ code: 'TAIL_LOOP', score: 0.90, threshold: 0.5, message: 'Response ends in a repeating block…'
|
|
303
|
+
{ code: 'TAIL_LOOP', score: 0.90, threshold: 0.5, message: 'Response ends in a repeating block…',
|
|
304
|
+
mode: 'word' },
|
|
188
305
|
],
|
|
189
306
|
scores: { EMPTY: 0, TOO_SHORT: 0, REPETITION: 0.83, TAIL_LOOP: 0.90, LOW_ENTROPY: 0.41 },
|
|
307
|
+
modes: { TAIL_LOOP: 'word' },
|
|
190
308
|
}
|
|
191
309
|
```
|
|
192
310
|
|
|
193
311
|
Every detector runs even after one fails, so `reasons` shows the whole picture instead of whichever check happened to be ordered first. `scores` includes passing detectors too — send them to your metrics and you will know your real degeneration rate within a day.
|
|
194
312
|
|
|
313
|
+
`modes` says which tokenizer produced a score, for the detectors that have more
|
|
314
|
+
than one. **Log it next to `scores`.** `TAIL_LOOP` measures words on spaced
|
|
315
|
+
scripts and characters on Chinese, Japanese and Thai; those are two
|
|
316
|
+
distributions with different base rates, and aggregating them into one histogram
|
|
317
|
+
gives you a number that describes neither.
|
|
318
|
+
|
|
195
319
|
## Detectors
|
|
196
320
|
|
|
197
321
|
| Code | Catches | Signal |
|
|
198
322
|
|---|---|---|
|
|
199
323
|
| `EMPTY` | Whitespace, lone punctuation, `{}`, empty fences | Content presence |
|
|
200
324
|
| `TOO_SHORT` | Non-empty but useless | Length vs. minimum |
|
|
201
|
-
| `REPETITION` | Loops and stutters | Duplicate n-gram fraction |
|
|
202
|
-
| `TAIL_LOOP` | Good start, then a stuck ending | Periodicity in the trailing window |
|
|
325
|
+
| `REPETITION` | Loops and stutters | Duplicate word n-gram fraction |
|
|
326
|
+
| `TAIL_LOOP` | Good start, then a stuck ending | Periodicity in the trailing window, over words or characters |
|
|
203
327
|
| `LOW_ENTROPY` | Character-level collapse, token artifacts | Hand-rolled LZ77 compression ratio |
|
|
204
328
|
| `TRUNCATED` | Cut off mid-thought | `finish_reason`, unbalanced fences/brackets |
|
|
205
329
|
| `INVALID_JSON` | Prose around the payload, missing keys | Parse + key contract |
|
|
@@ -248,6 +372,13 @@ object, a whole `Verdict`, or either of those buried in a wider log record all
|
|
|
248
372
|
work, because a calibration step you have to reshape your logs for is one you
|
|
249
373
|
will not run. `--json` emits the same analysis as data.
|
|
250
374
|
|
|
375
|
+
If you log `modes` alongside `scores`, detectors are segmented by tokenizer and
|
|
376
|
+
reported as `TAIL_LOOP [word]` and `TAIL_LOOP [char]`, each suggesting its own
|
|
377
|
+
option. Do this if your traffic is not all one script: pooled, the two
|
|
378
|
+
distributions produce a single threshold that is wrong for both — word-mode
|
|
379
|
+
`TAIL_LOOP` on Indonesian traffic has a healthy maximum near 0.35 where character
|
|
380
|
+
mode's is near 0.06.
|
|
381
|
+
|
|
251
382
|
**What it can and cannot tell you.** The corpus can compute a real margin
|
|
252
383
|
because every fixture is labelled. Your logs are not, and no arithmetic
|
|
253
384
|
recovers a label that was never written down. So these numbers bound *false
|
|
@@ -266,13 +397,25 @@ A miss is annoying. **A false positive is worse**: a healthy response gets disca
|
|
|
266
397
|
So the corpus carries deliberate traps — markdown tables, repeated-prefix lists, code blocks, rhetorical refrains — all of which a naive detector flags. `npm run calibrate` prints the margin between the worst healthy score and the weakest degenerate one:
|
|
267
398
|
|
|
268
399
|
```
|
|
269
|
-
===
|
|
270
|
-
healthy max : 0.
|
|
271
|
-
degenerate min: 0.
|
|
272
|
-
margin : 0.
|
|
400
|
+
=== TAIL_LOOP [word] ===
|
|
401
|
+
healthy max : 0.000 (code-block-typescript)
|
|
402
|
+
degenerate min: 0.900 (tail-loop-after-good-start)
|
|
403
|
+
margin : 0.900 OK
|
|
404
|
+
|
|
405
|
+
=== TAIL_LOOP [char] ===
|
|
406
|
+
healthy max : 0.291 (prose-zh-poem-refrain)
|
|
407
|
+
degenerate min: 0.829 (cjk-refrain-x20)
|
|
408
|
+
margin : 0.538 OK
|
|
273
409
|
```
|
|
274
410
|
|
|
275
|
-
|
|
411
|
+
Detectors with two tokenizers are reported per mode, and each detector is scored
|
|
412
|
+
only against fixtures labelled for it — otherwise a tail loop that `LOW_ENTROPY`
|
|
413
|
+
was never meant to catch drags `LOW_ENTROPY`'s margin negative and the report
|
|
414
|
+
reads like a regression in something nobody touched.
|
|
415
|
+
|
|
416
|
+
If that margin ever goes thin, the answer is a better detector, not a nudged
|
|
417
|
+
threshold. That rule is why `REPETITION` has no character mode: the one that was
|
|
418
|
+
built came out with a *negative* margin, so it was deleted rather than tuned.
|
|
276
419
|
|
|
277
420
|
## Growing the corpus
|
|
278
421
|
|
|
@@ -290,12 +433,151 @@ Output lands in `test/fixtures/raw/` **unreviewed**. Read each one, label it, th
|
|
|
290
433
|
- **Scores, not booleans.** Detectors report 0–1 and leave the threshold decision to you.
|
|
291
434
|
- **Abstains rather than guesses.** Samples too short to judge score 0.
|
|
292
435
|
|
|
436
|
+
## Script coverage
|
|
437
|
+
|
|
438
|
+
The dividing line is **whether a script puts spaces between words**, not whether
|
|
439
|
+
it is Latin. Korean, Cyrillic, Greek, Arabic and Devanagari all separate words
|
|
440
|
+
and are handled exactly like English. Han, Hiragana, Katakana and Thai do not,
|
|
441
|
+
and get different treatment:
|
|
442
|
+
|
|
443
|
+
| | Chinese / Japanese / Thai | Everything else |
|
|
444
|
+
|---|---|---|
|
|
445
|
+
| `TAIL_LOOP` | **Character mode**, `maxCharTailLoop` (default 0.7) | Word mode, `maxTailLoop` (default 0.5) |
|
|
446
|
+
| `REPETITION` | **Blind — see below** | Word n-grams, works |
|
|
447
|
+
| `LOW_ENTROPY`, `TRUNCATED`, `INVALID_JSON`, `EMPTY`, `TOO_SHORT` | Character- or structure-based, unaffected | Same |
|
|
448
|
+
| `LANG_MISMATCH` | Not covered (`id`/`en`/`es` only) | `id`/`en`/`es` only |
|
|
449
|
+
|
|
450
|
+
Mode is chosen per detector, from the span that detector actually reads — so a
|
|
451
|
+
reply that answers in English and then loops in Chinese puts the *tail* detector
|
|
452
|
+
into character mode without moving anything else. It is reported in
|
|
453
|
+
`Verdict.modes`.
|
|
454
|
+
|
|
455
|
+
**`REPETITION` is blind on these scripts, and we could not fix it.** A word
|
|
456
|
+
tokenizer sees a punctuation-delimited Chinese clause as one token, and a loop
|
|
457
|
+
with no punctuation as one token for the entire response, so it scores 0.000 on
|
|
458
|
+
an obvious loop. A character n-gram fallback was built, measured, and rejected —
|
|
459
|
+
because **it would add no coverage and cost a false-positive surface**.
|
|
460
|
+
|
|
461
|
+
It adds nothing because `TAIL_LOOP`'s character mode already catches every
|
|
462
|
+
degenerate non-Latin sample in the corpus, at a margin of 0.538.
|
|
463
|
+
|
|
464
|
+
It costs something because healthy *structured* CJK output scores high under it.
|
|
465
|
+
Repeated key scaffolding around short CJK values is genuinely redundant
|
|
466
|
+
character-by-character:
|
|
467
|
+
|
|
468
|
+
```
|
|
469
|
+
healthy json-zh-keys-valid, char n-grams (n=4), all items distinct
|
|
470
|
+
8 items 0.396 20 items 0.543 40 items 0.597
|
|
471
|
+
12 items 0.474 30 items 0.577
|
|
472
|
+
```
|
|
473
|
+
|
|
474
|
+
That flattens rather than diverging — it converges on the scaffolding's own
|
|
475
|
+
proportion — so a threshold does exist. But the plateau near 0.6 against the
|
|
476
|
+
weakest pure loop at 0.872 leaves about **0.19**, under the 0.2 margin this
|
|
477
|
+
package holds itself to, and the healthy side climbs with the number of keys a
|
|
478
|
+
payload carries. A detector with nothing to add and a structure-sensitive margin
|
|
479
|
+
is a false positive waiting for someone's payload shape to change, which is the
|
|
480
|
+
wrong trade here.
|
|
481
|
+
|
|
482
|
+
`TAIL_LOOP`'s character mode covers the gap in practice — it requires *exact*
|
|
483
|
+
periodicity, which scaffolding never produces, and it catches every degenerate
|
|
484
|
+
non-Latin sample in the corpus. But a mid-response CJK loop that recovers before
|
|
485
|
+
the end is not detected by anything here. If that is your failure mode, log
|
|
486
|
+
`LOW_ENTROPY` and threshold it yourself.
|
|
487
|
+
|
|
488
|
+
Two more things worth knowing:
|
|
489
|
+
|
|
490
|
+
- **Character mode abstains below 80 characters.** Three identical short
|
|
491
|
+
sentences closing a 40-character reply look like total coverage and are not
|
|
492
|
+
evidence of anything.
|
|
493
|
+
|
|
494
|
+
### Character mode is deliberately slower to fire
|
|
495
|
+
|
|
496
|
+
The two modes do not flag the same shape at the same point, and the gap is
|
|
497
|
+
large. Taking the clearest case — a response ending in an identical repeated
|
|
498
|
+
line — measured on both:
|
|
499
|
+
|
|
500
|
+
| Repeats of an identical closing line | English (`maxTailLoop` 0.5) | Chinese (`maxCharTailLoop` 0.7) |
|
|
501
|
+
|---|---|---|
|
|
502
|
+
| 3 | **flagged** (0.563) | 0.000 — under the 80-character floor |
|
|
503
|
+
| 5 | flagged (0.682) | 0.000 |
|
|
504
|
+
| 9 | flagged (0.794) | 0.686 |
|
|
505
|
+
| 10 | flagged (0.811) | **flagged** (0.708) |
|
|
506
|
+
| 20 | flagged (0.900) | flagged (0.829) |
|
|
507
|
+
|
|
508
|
+
**English flags at 3 repeats, Chinese at about 10** — and nearer 30 when a long
|
|
509
|
+
healthy passage precedes the loop, because the score is coverage of the trailing
|
|
510
|
+
window rather than a count.
|
|
511
|
+
|
|
512
|
+
This is a decision, not an accident of two constants. Word mode counts tokens,
|
|
513
|
+
so a repeated clause is several tokens and accumulates fast. Character mode
|
|
514
|
+
measures how much of a fixed trailing window one repeating block covers, and a
|
|
515
|
+
short refrain takes many repeats to fill it. Tightening `maxCharTailLoop` toward
|
|
516
|
+
word-mode aggression would put it into the range where ordinary CJK structured
|
|
517
|
+
output sits, which is the trade this package refuses.
|
|
518
|
+
|
|
519
|
+
**The practical consequence: a looping model answering in Chinese, Japanese or
|
|
520
|
+
Thai generates several times more output before the guard fires than the same
|
|
521
|
+
model looping in English.** Detection is later and the token saving is smaller.
|
|
522
|
+
If you serve mostly non-spaced-script traffic and that cost matters more to you
|
|
523
|
+
than the false-positive risk, lower `maxCharTailLoop` toward 0.5 — and calibrate
|
|
524
|
+
it against your own traffic first, because that is the range healthy structured
|
|
525
|
+
output starts to reach.
|
|
526
|
+
|
|
527
|
+
## Stability
|
|
528
|
+
|
|
529
|
+
What semver means for this package specifically.
|
|
530
|
+
|
|
531
|
+
**The public API is:** everything exported from `llm-output-guard`, plus
|
|
532
|
+
`outputGuard` / `OutputGuardOptions` / `DegenerateAction` from `./ai-sdk` and
|
|
533
|
+
`withOutputGuard` / `OutputGuardOptions` / `DegenerateAction` from `./openai`.
|
|
534
|
+
Each subpath is its own contract; the two adapters share an internal base type
|
|
535
|
+
today and are free to diverge, so an option added to one is not a promise about
|
|
536
|
+
the other. Anything not exported from those three entry points is internal, has
|
|
537
|
+
no stability guarantee, and may move in any release.
|
|
538
|
+
|
|
539
|
+
**Threshold and preset values are behaviour, not implementation.** This is the
|
|
540
|
+
interesting case, so it gets a rule of its own:
|
|
541
|
+
|
|
542
|
+
| Change | Release type |
|
|
543
|
+
|---|---|
|
|
544
|
+
| Lowering a default threshold, or changing a preset's numbers | **major** |
|
|
545
|
+
| Adding a new detector that runs by default | **major** |
|
|
546
|
+
| Adding a new *option*, defaulted so nothing changes | minor |
|
|
547
|
+
| Adding a new detector that is opt-in | minor |
|
|
548
|
+
| Making an existing detector strictly more accurate on its own axis | minor |
|
|
549
|
+
| Docs, internals, performance, fixing a detector that was returning a wrong score | patch |
|
|
550
|
+
|
|
551
|
+
The reasoning: a threshold change does not break your build, it changes which of
|
|
552
|
+
your production responses get discarded and retried. That is a larger event than
|
|
553
|
+
a signature change, and it is invisible until your traffic hits it. A number in
|
|
554
|
+
`presets.chat` is part of the contract in the same way a function name is.
|
|
555
|
+
|
|
556
|
+
**Semver does not cover:** the exact scores a detector returns (only their
|
|
557
|
+
direction and the thresholds that act on them), the contents of the fixture
|
|
558
|
+
corpus, `message` strings in `Reason`, or the output format of the `calibrate`
|
|
559
|
+
CLI's human-readable report. `--json` output *is* covered.
|
|
560
|
+
|
|
561
|
+
**Peer ranges** are narrowed only in a major. They are verified rather than
|
|
562
|
+
assumed — `npm run check:peers` installs the packed tarball against each end of
|
|
563
|
+
each declared range and both typechecks and runs the adapter.
|
|
564
|
+
|
|
565
|
+
**Why this is written down.** Version 0.4.2 shipped the `ai` peer narrowing, the
|
|
566
|
+
new `./openai` subpath, and character-mode `TAIL_LOOP` — one breaking change, one
|
|
567
|
+
feature, and one behaviour change — under a **patch** number, which every default
|
|
568
|
+
version range upgrades into automatically. It was withdrawn from npm within the
|
|
569
|
+
72-hour unpublish window and re-released as 0.5.0, where `^0.4.1` correctly
|
|
570
|
+
resolves away from it. The rule it broke is the one in the table above: threshold
|
|
571
|
+
and preset changes are behaviour changes, and behaviour changes are never
|
|
572
|
+
patches.
|
|
573
|
+
|
|
293
574
|
## Limitations
|
|
294
575
|
|
|
295
576
|
- Not a hallucination detector. It measures *shape*, never truth.
|
|
577
|
+
- `REPETITION` does not work on Chinese, Japanese or Thai. See above — this is a known, measured gap, not an oversight.
|
|
296
578
|
- Language detection is a function-word heuristic covering `id`/`en`/`es`. Opt-in, and unreliable under 25 words.
|
|
297
579
|
- Truncation from a missing full stop is weak evidence, scored 0.55 and left below the default thresholds on purpose. Lower `maxTruncation` to ~0.5 to catch it, and expect false positives.
|
|
298
|
-
- Thresholds calibrated on the bundled corpus. Yours will differ.
|
|
580
|
+
- Thresholds calibrated on the bundled corpus. Yours will differ — and the word and character thresholds need calibrating **separately**, because they are separate distributions.
|
|
299
581
|
|
|
300
582
|
## License
|
|
301
583
|
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { V as Verdict } from './stream-Be2WhoKn.cjs';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* The option surface every provider adapter shares.
|
|
5
|
+
*
|
|
6
|
+
* Defined once rather than per subpath so the adapters cannot drift: the
|
|
7
|
+
* promise `./openai` makes is that someone who has read the `./ai-sdk` docs can
|
|
8
|
+
* guess it, and two hand-maintained copies of an interface is how that promise
|
|
9
|
+
* quietly stops being true.
|
|
10
|
+
*
|
|
11
|
+
* ## This type is INTERNAL. It is not public API, at 1.0 or after.
|
|
12
|
+
*
|
|
13
|
+
* It is exported from no subpath and is not reachable by any import path a user
|
|
14
|
+
* has. What is public is each adapter's own `OutputGuardOptions`, and those are
|
|
15
|
+
* separate contracts that happen to share a base today.
|
|
16
|
+
*
|
|
17
|
+
* The distinction is what keeps a future adapter-specific option cheap. When
|
|
18
|
+
* `./openai` needs something only OpenAI has, it goes on *that* subpath's
|
|
19
|
+
* `OutputGuardOptions` and nothing else moves. Nobody has to widen a shared type
|
|
20
|
+
* for one provider's concern, and nobody has to split a frozen public interface
|
|
21
|
+
* after 1.0 -- because the shared thing was never frozen in the first place.
|
|
22
|
+
*
|
|
23
|
+
* Rules that follow from that, worth stating because they are easy to erode:
|
|
24
|
+
*
|
|
25
|
+
* - Anything here must be genuinely common to every adapter. If it needs an
|
|
26
|
+
* "only applies to X" caveat, it belongs on X's own options instead.
|
|
27
|
+
* - Adapters are free to stop extending this. Convergence is the goal, not the
|
|
28
|
+
* constraint; if two adapters genuinely diverge, copying the fields into both
|
|
29
|
+
* is the correct move and breaks no promise.
|
|
30
|
+
* - `DegenerateAction` is re-exported from each subpath and therefore *is*
|
|
31
|
+
* public through those paths. Changing its members is a breaking change to
|
|
32
|
+
* both, and semver applies per subpath.
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
type DegenerateAction = 'throw' | 'abort' | 'ignore';
|
|
36
|
+
interface AdapterGuardOptions {
|
|
37
|
+
/**
|
|
38
|
+
* What to do when output is judged degenerate. Default `'throw'`.
|
|
39
|
+
*
|
|
40
|
+
* - `'throw'` errors the call with a `DegenerateOutputError`, which
|
|
41
|
+
* `.retryable` marks as safe for your fallback layer to act on. On a
|
|
42
|
+
* stream this also cancels the upstream request, so the tokens you have
|
|
43
|
+
* not been billed for yet never get generated.
|
|
44
|
+
* - `'abort'` ends the stream cleanly and keeps whatever arrived first.
|
|
45
|
+
* Same token saving, no exception to handle -- use it when a partial
|
|
46
|
+
* answer beats no answer.
|
|
47
|
+
* - `'ignore'` reports through `onVerdict` and changes nothing. This is the
|
|
48
|
+
* setting to roll out with: watch your own traffic before letting any
|
|
49
|
+
* threshold fail a request.
|
|
50
|
+
*/
|
|
51
|
+
onDegenerate?: DegenerateAction;
|
|
52
|
+
/**
|
|
53
|
+
* Every verdict, passing or failing, once per call. Send the scores to your
|
|
54
|
+
* metrics -- a week of them is what turns the shipped thresholds into
|
|
55
|
+
* thresholds you can defend for your own traffic.
|
|
56
|
+
*
|
|
57
|
+
* Log `verdict.modes` alongside `verdict.scores`. `TAIL_LOOP` measures words
|
|
58
|
+
* on spaced scripts and characters on Chinese, Japanese and Thai, and those
|
|
59
|
+
* are different distributions; pooled into one histogram they describe
|
|
60
|
+
* neither.
|
|
61
|
+
*/
|
|
62
|
+
onVerdict?: (verdict: Verdict, context: {
|
|
63
|
+
streaming: boolean;
|
|
64
|
+
}) => void;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export type { AdapterGuardOptions as A, DegenerateAction as D };
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { V as Verdict } from './stream-Be2WhoKn.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* The option surface every provider adapter shares.
|
|
5
|
+
*
|
|
6
|
+
* Defined once rather than per subpath so the adapters cannot drift: the
|
|
7
|
+
* promise `./openai` makes is that someone who has read the `./ai-sdk` docs can
|
|
8
|
+
* guess it, and two hand-maintained copies of an interface is how that promise
|
|
9
|
+
* quietly stops being true.
|
|
10
|
+
*
|
|
11
|
+
* ## This type is INTERNAL. It is not public API, at 1.0 or after.
|
|
12
|
+
*
|
|
13
|
+
* It is exported from no subpath and is not reachable by any import path a user
|
|
14
|
+
* has. What is public is each adapter's own `OutputGuardOptions`, and those are
|
|
15
|
+
* separate contracts that happen to share a base today.
|
|
16
|
+
*
|
|
17
|
+
* The distinction is what keeps a future adapter-specific option cheap. When
|
|
18
|
+
* `./openai` needs something only OpenAI has, it goes on *that* subpath's
|
|
19
|
+
* `OutputGuardOptions` and nothing else moves. Nobody has to widen a shared type
|
|
20
|
+
* for one provider's concern, and nobody has to split a frozen public interface
|
|
21
|
+
* after 1.0 -- because the shared thing was never frozen in the first place.
|
|
22
|
+
*
|
|
23
|
+
* Rules that follow from that, worth stating because they are easy to erode:
|
|
24
|
+
*
|
|
25
|
+
* - Anything here must be genuinely common to every adapter. If it needs an
|
|
26
|
+
* "only applies to X" caveat, it belongs on X's own options instead.
|
|
27
|
+
* - Adapters are free to stop extending this. Convergence is the goal, not the
|
|
28
|
+
* constraint; if two adapters genuinely diverge, copying the fields into both
|
|
29
|
+
* is the correct move and breaks no promise.
|
|
30
|
+
* - `DegenerateAction` is re-exported from each subpath and therefore *is*
|
|
31
|
+
* public through those paths. Changing its members is a breaking change to
|
|
32
|
+
* both, and semver applies per subpath.
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
type DegenerateAction = 'throw' | 'abort' | 'ignore';
|
|
36
|
+
interface AdapterGuardOptions {
|
|
37
|
+
/**
|
|
38
|
+
* What to do when output is judged degenerate. Default `'throw'`.
|
|
39
|
+
*
|
|
40
|
+
* - `'throw'` errors the call with a `DegenerateOutputError`, which
|
|
41
|
+
* `.retryable` marks as safe for your fallback layer to act on. On a
|
|
42
|
+
* stream this also cancels the upstream request, so the tokens you have
|
|
43
|
+
* not been billed for yet never get generated.
|
|
44
|
+
* - `'abort'` ends the stream cleanly and keeps whatever arrived first.
|
|
45
|
+
* Same token saving, no exception to handle -- use it when a partial
|
|
46
|
+
* answer beats no answer.
|
|
47
|
+
* - `'ignore'` reports through `onVerdict` and changes nothing. This is the
|
|
48
|
+
* setting to roll out with: watch your own traffic before letting any
|
|
49
|
+
* threshold fail a request.
|
|
50
|
+
*/
|
|
51
|
+
onDegenerate?: DegenerateAction;
|
|
52
|
+
/**
|
|
53
|
+
* Every verdict, passing or failing, once per call. Send the scores to your
|
|
54
|
+
* metrics -- a week of them is what turns the shipped thresholds into
|
|
55
|
+
* thresholds you can defend for your own traffic.
|
|
56
|
+
*
|
|
57
|
+
* Log `verdict.modes` alongside `verdict.scores`. `TAIL_LOOP` measures words
|
|
58
|
+
* on spaced scripts and characters on Chinese, Japanese and Thai, and those
|
|
59
|
+
* are different distributions; pooled into one histogram they describe
|
|
60
|
+
* neither.
|
|
61
|
+
*/
|
|
62
|
+
onVerdict?: (verdict: Verdict, context: {
|
|
63
|
+
streaming: boolean;
|
|
64
|
+
}) => void;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export type { AdapterGuardOptions as A, DegenerateAction as D };
|