llm-output-guard 0.1.0

package/dist/index.d.cts

@@ -0,0 +1,233 @@
+type ReasonCode = 'EMPTY' | 'TOO_SHORT' | 'REPETITION' | 'TAIL_LOOP' | 'LOW_ENTROPY' | 'TRUNCATED' | 'INVALID_JSON' | 'LANG_MISMATCH';
+interface Reason {
+    code: ReasonCode;
+    /** 0..1. Higher means more suspicious. */
+    score: number;
+    /** The threshold this score crossed. */
+    threshold: number;
+    /** Human-readable explanation, safe to log. */
+    message: string;
+}
+interface Verdict {
+    /** True when nothing crossed its threshold. */
+    ok: boolean;
+    /** Every failing signal, not just the first -- more useful when debugging. */
+    reasons: Reason[];
+    /** Every score computed, including passing ones. Feed these to your metrics. */
+    scores: Partial<Record<ReasonCode, number>>;
+    /** Parsed JSON payload when `json` was enabled and parsing succeeded. */
+    json?: unknown;
+}
+interface CheckOptions {
+    /** Minimum acceptable length in characters. Set 0 to disable. Default 1. */
+    minLength?: number;
+    /** Duplicate-n-gram threshold. Set null to disable. Default 0.35. */
+    maxRepetition?: number | null;
+    /** Tail-loop threshold. Set null to disable. Default 0.5. */
+    maxTailLoop?: number | null;
+    /** Compressibility threshold. Set null to disable. Default 0.75. */
+    maxCompressibility?: number | null;
+    /** Truncation threshold. Set null to disable. Default null. */
+    maxTruncation?: number | null;
+    /** Provider stop reason, used by the truncation detector when present. */
+    finishReason?: string;
+    /** Require a parseable JSON payload. Default false. */
+    expectJson?: boolean;
+    /** Allow the JSON payload to be wrapped in a fence. Default true. */
+    allowJsonFence?: boolean;
+    /** Top-level keys the JSON payload must contain. */
+    requiredKeys?: string[];
+    /** Expected language code ('id' | 'en' | 'es'). Off by default. */
+    expectLang?: string | null;
+    /** Language-mismatch threshold. Default 0.6. */
+    maxLangMismatch?: number;
+    /** N-gram size for the repetition detector. Default 3. */
+    ngram?: number;
+}
+
+/**
+ * Runs every enabled detector and returns a structured verdict.
+ *
+ * Pure and synchronous: no network, no clock, no randomness. The same input
+ * always produces the same verdict, which is what makes it safe to put on a
+ * hot path and easy to unit test.
+ *
+ * Every detector runs even after one fails, so `reasons` shows the full picture
+ * rather than whichever check happened to be ordered first.
+ */
+declare function checkOutput(text: string, options?: CheckOptions): Verdict;
+/** Error thrown by {@link assertOutput}, carrying the full verdict. */
+declare class DegenerateOutputError extends Error {
+    readonly verdict: Verdict;
+    /** Marks this as safe to retry against another provider. */
+    readonly retryable = true;
+    constructor(verdict: Verdict);
+}
+/**
+ * Throwing wrapper, for dropping straight into an existing retry or fallback
+ * chain that already keys off thrown errors.
+ */
+declare function assertOutput(text: string, options?: CheckOptions): string;
+
+/**
+ * Starting configurations. Tuned against the fixture corpus in test/fixtures,
+ * so treat them as calibrated defaults rather than arbitrary numbers -- and
+ * still re-tune against your own traffic before trusting them in production.
+ */
+declare const presets: {
+    /** Conversational replies. Tolerant of quoted or listed repetition. */
+    readonly chat: {
+        minLength: number;
+        maxRepetition: number;
+        maxTailLoop: number;
+        maxCompressibility: number;
+    };
+    /** Structured output. Parseability is non-negotiable; prose checks relax. */
+    readonly strictJson: {
+        minLength: number;
+        expectJson: true;
+        maxRepetition: number;
+        maxTailLoop: number;
+        maxCompressibility: null;
+        maxTruncation: number;
+    };
+    /** Long-form generation, where truncation matters most. */
+    readonly longForm: {
+        minLength: number;
+        maxRepetition: number;
+        maxTailLoop: number;
+        maxCompressibility: number;
+        maxTruncation: number;
+    };
+    /** Catches only unambiguous garbage. Use when false positives cost more than misses. */
+    readonly lenient: {
+        minLength: number;
+        maxRepetition: number;
+        maxTailLoop: number;
+        maxCompressibility: number;
+    };
+};
+
+interface RepetitionOptions {
+    /** N-gram size. 3 suits prose; 2 is noisy, 4 misses short loops. */
+    n?: number;
+    /** Only analyse the first N characters. Keeps cost bounded on long outputs. */
+    maxSample?: number;
+}
+/**
+ * Fraction of n-grams that are duplicates. 0 = every n-gram unique, 1 = total collapse.
+ *
+ * Healthy prose sits near 0.00-0.10. A model stuck in a loop passes 0.5 quickly.
+ * Returns 0 for text too short to judge rather than guessing.
+ */
+declare function repetitionScore(text: string, options?: RepetitionOptions): number;
+interface TailLoopOptions {
+    /** How many trailing words to inspect. */
+    tailWords?: number;
+    /** Longest loop period to look for, in words. */
+    maxPeriod?: number;
+    /** A block must repeat at least this many times to count as a loop. */
+    minRepeats?: number;
+}
+/**
+ * Detects the specific failure where a model terminates in a repeating tail --
+ * the same clause emitted over and over until max_tokens runs out.
+ *
+ * Whole-output repetition misses this when the first half of the response was fine.
+ * Returns the fraction of the inspected tail covered by the loop.
+ */
+declare function tailLoopScore(text: string, options?: TailLoopOptions): number;
+
+interface CompressibilityOptions {
+    /** Sliding window for back-references, in characters. */
+    window?: number;
+    /** Only analyse the first N characters. */
+    maxSample?: number;
+    /** Shortest back-reference worth emitting. */
+    minMatch?: number;
+}
+/**
+ * Greedy LZ77 pass returning emitted-tokens / input-characters.
+ *
+ * Deliberately hand-rolled instead of node:zlib so the package stays
+ * runtime-agnostic (browser, edge, Deno, Bun) and dependency-free.
+ * This is not a real compressor; it only needs to move monotonically
+ * with redundancy, which is all the score requires.
+ *
+ * Healthy prose lands around 0.30-0.55. Degenerate output collapses toward 0.
+ */
+declare function compressionRatio(text: string, options?: CompressibilityOptions): number;
+/**
+ * Suspicion score derived from {@link compressionRatio}.
+ * `pivot` is the ratio treated as fully healthy; lower ratios scale up toward 1.
+ */
+declare function compressibilityScore(text: string, options?: CompressibilityOptions & {
+    pivot?: number;
+}): number;
+
+/**
+ * 1 when the response carries no usable content at all.
+ *
+ * Covers the cases a plain `!text` check misses: whitespace-only, a lone
+ * punctuation mark, an empty code fence, or an empty JSON envelope.
+ */
+declare function emptinessScore(text: string): number;
+/** 1 when the response is shorter than `minChars`, scaling down to 0 at the threshold. */
+declare function shortnessScore(text: string, minChars: number): number;
+
+interface TruncationOptions {
+    /**
+     * The provider's own stop reason, if you have it. When this says the output
+     * hit the token ceiling, that is authoritative and the heuristics are skipped.
+     */
+    finishReason?: string;
+}
+/**
+ * Detects output that stopped mid-thought.
+ *
+ * Prefers the provider's finish_reason when supplied, because that is ground
+ * truth. Falls back to structural signals: unbalanced fences or brackets, or a
+ * final sentence with no terminal punctuation.
+ *
+ * Returns a graded score, not a boolean -- a missing full stop alone is weak
+ * evidence and should not sink a response on its own.
+ */
+declare function truncationScore(text: string, options?: TruncationOptions): number;
+
+interface JsonOptions {
+    /** Allow the payload to sit inside a ```json fence rather than being bare. */
+    allowFence?: boolean;
+    /** Top-level keys that must be present for the payload to count as valid. */
+    requiredKeys?: string[];
+}
+interface JsonResult {
+    /** 0 when the payload parses and satisfies requiredKeys, 1 otherwise. */
+    score: number;
+    /** The parsed value, when parsing succeeded. */
+    value?: unknown;
+    reason?: 'unparseable' | 'missing-keys';
+    missingKeys?: string[];
+}
+/** Pull a JSON payload out of a ```json fence, or return the text unchanged. */
+declare function stripFence(text: string): string;
+/**
+ * Structured-output check. Models that "succeed" while emitting prose around
+ * the JSON, or an object missing half its keys, fail here.
+ */
+declare function jsonScore(text: string, options?: JsonOptions): JsonResult;
+
+interface LanguageOptions {
+    /** Below this word count the signal is unreliable and the score is 0. */
+    minWords?: number;
+}
+/** Share of tokens matching each known profile. Not a full language detector. */
+declare function languageProfile(text: string): Record<string, number>;
+/**
+ * Suspicion that the response is not in `expected`.
+ * Returns 0 for unknown languages or samples too short to judge --
+ * silence is better than a confident wrong answer here.
+ */
+declare function languageMismatchScore(text: string, expected: string, options?: LanguageOptions): number;
+declare const supportedLanguages: string[];
+
+export { type CheckOptions, type CompressibilityOptions, DegenerateOutputError, type JsonOptions, type JsonResult, type LanguageOptions, type Reason, type ReasonCode, type RepetitionOptions, type TailLoopOptions, type TruncationOptions, type Verdict, assertOutput, checkOutput, compressibilityScore, compressionRatio, emptinessScore, jsonScore, languageMismatchScore, languageProfile, presets, repetitionScore, shortnessScore, stripFence, supportedLanguages, tailLoopScore, truncationScore };

package/dist/index.d.ts

@@ -0,0 +1,233 @@
+type ReasonCode = 'EMPTY' | 'TOO_SHORT' | 'REPETITION' | 'TAIL_LOOP' | 'LOW_ENTROPY' | 'TRUNCATED' | 'INVALID_JSON' | 'LANG_MISMATCH';
+interface Reason {
+    code: ReasonCode;
+    /** 0..1. Higher means more suspicious. */
+    score: number;
+    /** The threshold this score crossed. */
+    threshold: number;
+    /** Human-readable explanation, safe to log. */
+    message: string;
+}
+interface Verdict {
+    /** True when nothing crossed its threshold. */
+    ok: boolean;
+    /** Every failing signal, not just the first -- more useful when debugging. */
+    reasons: Reason[];
+    /** Every score computed, including passing ones. Feed these to your metrics. */
+    scores: Partial<Record<ReasonCode, number>>;
+    /** Parsed JSON payload when `json` was enabled and parsing succeeded. */
+    json?: unknown;
+}
+interface CheckOptions {
+    /** Minimum acceptable length in characters. Set 0 to disable. Default 1. */
+    minLength?: number;
+    /** Duplicate-n-gram threshold. Set null to disable. Default 0.35. */
+    maxRepetition?: number | null;
+    /** Tail-loop threshold. Set null to disable. Default 0.5. */
+    maxTailLoop?: number | null;
+    /** Compressibility threshold. Set null to disable. Default 0.75. */
+    maxCompressibility?: number | null;
+    /** Truncation threshold. Set null to disable. Default null. */
+    maxTruncation?: number | null;
+    /** Provider stop reason, used by the truncation detector when present. */
+    finishReason?: string;
+    /** Require a parseable JSON payload. Default false. */
+    expectJson?: boolean;
+    /** Allow the JSON payload to be wrapped in a fence. Default true. */
+    allowJsonFence?: boolean;
+    /** Top-level keys the JSON payload must contain. */
+    requiredKeys?: string[];
+    /** Expected language code ('id' | 'en' | 'es'). Off by default. */
+    expectLang?: string | null;
+    /** Language-mismatch threshold. Default 0.6. */
+    maxLangMismatch?: number;
+    /** N-gram size for the repetition detector. Default 3. */
+    ngram?: number;
+}
+
+/**
+ * Runs every enabled detector and returns a structured verdict.
+ *
+ * Pure and synchronous: no network, no clock, no randomness. The same input
+ * always produces the same verdict, which is what makes it safe to put on a
+ * hot path and easy to unit test.
+ *
+ * Every detector runs even after one fails, so `reasons` shows the full picture
+ * rather than whichever check happened to be ordered first.
+ */
+declare function checkOutput(text: string, options?: CheckOptions): Verdict;
+/** Error thrown by {@link assertOutput}, carrying the full verdict. */
+declare class DegenerateOutputError extends Error {
+    readonly verdict: Verdict;
+    /** Marks this as safe to retry against another provider. */
+    readonly retryable = true;
+    constructor(verdict: Verdict);
+}
+/**
+ * Throwing wrapper, for dropping straight into an existing retry or fallback
+ * chain that already keys off thrown errors.
+ */
+declare function assertOutput(text: string, options?: CheckOptions): string;
+
+/**
+ * Starting configurations. Tuned against the fixture corpus in test/fixtures,
+ * so treat them as calibrated defaults rather than arbitrary numbers -- and
+ * still re-tune against your own traffic before trusting them in production.
+ */
+declare const presets: {
+    /** Conversational replies. Tolerant of quoted or listed repetition. */
+    readonly chat: {
+        minLength: number;
+        maxRepetition: number;
+        maxTailLoop: number;
+        maxCompressibility: number;
+    };
+    /** Structured output. Parseability is non-negotiable; prose checks relax. */
+    readonly strictJson: {
+        minLength: number;
+        expectJson: true;
+        maxRepetition: number;
+        maxTailLoop: number;
+        maxCompressibility: null;
+        maxTruncation: number;
+    };
+    /** Long-form generation, where truncation matters most. */
+    readonly longForm: {
+        minLength: number;
+        maxRepetition: number;
+        maxTailLoop: number;
+        maxCompressibility: number;
+        maxTruncation: number;
+    };
+    /** Catches only unambiguous garbage. Use when false positives cost more than misses. */
+    readonly lenient: {
+        minLength: number;
+        maxRepetition: number;
+        maxTailLoop: number;
+        maxCompressibility: number;
+    };
+};
+
+interface RepetitionOptions {
+    /** N-gram size. 3 suits prose; 2 is noisy, 4 misses short loops. */
+    n?: number;
+    /** Only analyse the first N characters. Keeps cost bounded on long outputs. */
+    maxSample?: number;
+}
+/**
+ * Fraction of n-grams that are duplicates. 0 = every n-gram unique, 1 = total collapse.
+ *
+ * Healthy prose sits near 0.00-0.10. A model stuck in a loop passes 0.5 quickly.
+ * Returns 0 for text too short to judge rather than guessing.
+ */
+declare function repetitionScore(text: string, options?: RepetitionOptions): number;
+interface TailLoopOptions {
+    /** How many trailing words to inspect. */
+    tailWords?: number;
+    /** Longest loop period to look for, in words. */
+    maxPeriod?: number;
+    /** A block must repeat at least this many times to count as a loop. */
+    minRepeats?: number;
+}
+/**
+ * Detects the specific failure where a model terminates in a repeating tail --
+ * the same clause emitted over and over until max_tokens runs out.
+ *
+ * Whole-output repetition misses this when the first half of the response was fine.
+ * Returns the fraction of the inspected tail covered by the loop.
+ */
+declare function tailLoopScore(text: string, options?: TailLoopOptions): number;
+
+interface CompressibilityOptions {
+    /** Sliding window for back-references, in characters. */
+    window?: number;
+    /** Only analyse the first N characters. */
+    maxSample?: number;
+    /** Shortest back-reference worth emitting. */
+    minMatch?: number;
+}
+/**
+ * Greedy LZ77 pass returning emitted-tokens / input-characters.
+ *
+ * Deliberately hand-rolled instead of node:zlib so the package stays
+ * runtime-agnostic (browser, edge, Deno, Bun) and dependency-free.
+ * This is not a real compressor; it only needs to move monotonically
+ * with redundancy, which is all the score requires.
+ *
+ * Healthy prose lands around 0.30-0.55. Degenerate output collapses toward 0.
+ */
+declare function compressionRatio(text: string, options?: CompressibilityOptions): number;
+/**
+ * Suspicion score derived from {@link compressionRatio}.
+ * `pivot` is the ratio treated as fully healthy; lower ratios scale up toward 1.
+ */
+declare function compressibilityScore(text: string, options?: CompressibilityOptions & {
+    pivot?: number;
+}): number;
+
+/**
+ * 1 when the response carries no usable content at all.
+ *
+ * Covers the cases a plain `!text` check misses: whitespace-only, a lone
+ * punctuation mark, an empty code fence, or an empty JSON envelope.
+ */
+declare function emptinessScore(text: string): number;
+/** 1 when the response is shorter than `minChars`, scaling down to 0 at the threshold. */
+declare function shortnessScore(text: string, minChars: number): number;
+
+interface TruncationOptions {
+    /**
+     * The provider's own stop reason, if you have it. When this says the output
+     * hit the token ceiling, that is authoritative and the heuristics are skipped.
+     */
+    finishReason?: string;
+}
+/**
+ * Detects output that stopped mid-thought.
+ *
+ * Prefers the provider's finish_reason when supplied, because that is ground
+ * truth. Falls back to structural signals: unbalanced fences or brackets, or a
+ * final sentence with no terminal punctuation.
+ *
+ * Returns a graded score, not a boolean -- a missing full stop alone is weak
+ * evidence and should not sink a response on its own.
+ */
+declare function truncationScore(text: string, options?: TruncationOptions): number;
+
+interface JsonOptions {
+    /** Allow the payload to sit inside a ```json fence rather than being bare. */
+    allowFence?: boolean;
+    /** Top-level keys that must be present for the payload to count as valid. */
+    requiredKeys?: string[];
+}
+interface JsonResult {
+    /** 0 when the payload parses and satisfies requiredKeys, 1 otherwise. */
+    score: number;
+    /** The parsed value, when parsing succeeded. */
+    value?: unknown;
+    reason?: 'unparseable' | 'missing-keys';
+    missingKeys?: string[];
+}
+/** Pull a JSON payload out of a ```json fence, or return the text unchanged. */
+declare function stripFence(text: string): string;
+/**
+ * Structured-output check. Models that "succeed" while emitting prose around
+ * the JSON, or an object missing half its keys, fail here.
+ */
+declare function jsonScore(text: string, options?: JsonOptions): JsonResult;
+
+interface LanguageOptions {
+    /** Below this word count the signal is unreliable and the score is 0. */
+    minWords?: number;
+}
+/** Share of tokens matching each known profile. Not a full language detector. */
+declare function languageProfile(text: string): Record<string, number>;
+/**
+ * Suspicion that the response is not in `expected`.
+ * Returns 0 for unknown languages or samples too short to judge --
+ * silence is better than a confident wrong answer here.
+ */
+declare function languageMismatchScore(text: string, expected: string, options?: LanguageOptions): number;
+declare const supportedLanguages: string[];
+
+export { type CheckOptions, type CompressibilityOptions, DegenerateOutputError, type JsonOptions, type JsonResult, type LanguageOptions, type Reason, type ReasonCode, type RepetitionOptions, type TailLoopOptions, type TruncationOptions, type Verdict, assertOutput, checkOutput, compressibilityScore, compressionRatio, emptinessScore, jsonScore, languageMismatchScore, languageProfile, presets, repetitionScore, shortnessScore, stripFence, supportedLanguages, tailLoopScore, truncationScore };