@devilsdev/rag-core 3.2.0 → 3.4.0

package/dist/all.d.ts

@@ -410,6 +410,7 @@ export { scoreAnswer } from "./types/evaluate/scoring";
 export { computeBLEU } from "./types/evaluate/scoring";
 export { computeROUGE } from "./types/evaluate/scoring";
 export { checkNumericConsistency } from "./types/evaluate/scoring";
+export { checkPolarityConsistency } from "./types/evaluate/scoring";
 export { extractNumbers } from "./types/evaluate/scoring";
 export { computeFaithfulness } from "./types/evaluate/faithfulness";
 export { computeSemanticSimilarity } from "./types/evaluate/semantic-similarity";
@@ -429,6 +430,10 @@ export { CitationTracker } from "./types/citation/citation-tracker";
 export { checkAnswer } from "./types/grounding/check-answer";
 export { normalizeSources } from "./types/grounding/check-answer";
 export { createAiSdkAdapter } from "./types/adapters/ai-sdk";
+export { createMastraScorer } from "./types/adapters/mastra";
+export { createMastraGuardrails } from "./types/adapters/mastra";
+export { groundingMiddleware } from "./types/adapters/ai-sdk-middleware";
+export { GroundingBlockedError } from "./types/adapters/ai-sdk-middleware";
 export { MemoryCache } from "./types/cache/memory-cache";
 export { RedisCache } from "./types/cache/redis-cache";
 export { mapSentenceToSources } from "./types/citation/source-mapper";

package/dist/index.cjs

@@ -1296,7 +1296,7 @@ var require_package = __commonJS({
   "package.json"(exports2, module2) {
     module2.exports = {
       name: "@devilsdev/rag-pipeline-utils",
-      version: "3.2.0",
+      version: "3.4.0",
       description: "Composable retrieval-augmented generation (RAG) for Node.js: grounded citations, evaluation, guardrails, hybrid retrieval and a CLI.",
       main: "dist/index.cjs",
       type: "commonjs",
@@ -1542,7 +1542,7 @@ var require_package = __commonJS({
         prettier: "^3.5.3",
         "prop-types": "^15.8.1",
         rimraf: "^6.0.1",
-        sharp: "^0.34.1",
+        sharp: "^0.35.5",
         "ts-jest": "^29.3.2",
         typedoc: "0.28.20",
         "typedoc-plugin-markdown": "4.14.1",
@@ -1550,10 +1550,10 @@ var require_package = __commonJS({
         "why-is-node-running": "^3.2.2"
       },
       dependencies: {
-        "@devilsdev/rag-connectors": "3.2.0",
-        "@devilsdev/rag-core": "3.2.0",
-        "@devilsdev/rag-eval": "3.2.0",
-        "@devilsdev/rag-guardrails": "3.2.0",
+        "@devilsdev/rag-connectors": "3.4.0",
+        "@devilsdev/rag-core": "3.4.0",
+        "@devilsdev/rag-eval": "3.4.0",
+        "@devilsdev/rag-guardrails": "3.4.0",
         ajv: "^8.17.1",
         axios: "^1.21.0",
         chokidar: "^4.0.3",
@@ -1590,14 +1590,16 @@ var require_package = __commonJS({
       },
       overrides: {
         "docusaurus-plugin-llms": {
-          minimatch: "^9.0.9",
+          minimatch: "^11.0.1",
           yaml: "^2.8.3"
         },
         "fast-uri": "^3.1.9",
         "follow-redirects": "^1.16.2",
         "form-data": "^4.0.6",
         qs: "^6.16.1",
+        "serialize-javascript": "^7.1.3",
         tinypool: "^2.2.0",
+        uuid: "^11.1.1",
         "ws@^8.0.0": "^8.21.0",
         zod: "^4.6.7"
       }
@@ -2443,6 +2445,56 @@ var require_scoring = __commonJS({
       const missing = claimNumbers.filter((n) => !docNumbers.has(n));
       return { consistent: missing.length === 0, claimNumbers, missing };
     }
+    var NEGATION_WORDS = new Set(
+      "not no never none nobody nothing nowhere neither nor cannot without dont doesnt didnt isnt arent wasnt werent wont wouldnt shouldnt couldnt cant hasnt havent hadnt mustnt neednt".split(" ")
+    );
+    var NON_NEGATING_NEXT = {
+      not: /* @__PURE__ */ new Set(["only", "just", "merely"]),
+      no: /* @__PURE__ */ new Set(["doubt", "matter"]),
+      nothing: /* @__PURE__ */ new Set(["but"])
+    };
+    function isNegated(tokens) {
+      let count = 0;
+      for (let i = 0; i < tokens.length; i++) {
+        const t = tokens[i];
+        if (!NEGATION_WORDS.has(t)) continue;
+        if (NON_NEGATING_NEXT[t] && NON_NEGATING_NEXT[t].has(tokens[i + 1])) {
+          continue;
+        }
+        count += 1;
+      }
+      return count % 2 === 1;
+    }
+    function checkPolarityConsistency(claimText, docText) {
+      const claimTokens = tokenize(claimText);
+      const claimNegated = isNegated(claimTokens);
+      const claimWords = [
+        ...new Set(
+          contentTokens(claimTokens.filter((t) => !NEGATION_WORDS.has(t)))
+        )
+      ];
+      if (claimWords.length === 0) {
+        return { consistent: true, claimNegated, sourceSentence: null };
+      }
+      const candidates = splitSentences(docText).map((sentence) => {
+        const tokens = tokenize(sentence);
+        const set = new Set(tokens);
+        const support = claimWords.filter((t) => set.has(t)).length / claimWords.length;
+        return { sentence, tokens, support };
+      }).filter((c) => c.support > 0).sort((a, b) => b.support - a.support);
+      if (candidates.length === 0) {
+        return { consistent: true, claimNegated, sourceSentence: null };
+      }
+      const best = candidates[0].support;
+      const compared = candidates.filter((c) => c.support === best);
+      const match = compared.find((c) => isNegated(c.tokens) === claimNegated);
+      return {
+        consistent: Boolean(match),
+        claimNegated,
+        sourceSentence: (match || compared[0]).sentence
+      };
+    }
+    var POLARITY_MISMATCH_FACTOR = 0.4;
     var NUMERIC_MISMATCH_FACTOR = 0.4;
     function computeSupportScore(claimTokens, docTokens, idfWeights = null, options = {}) {
       const claim = [...new Set(contentTokens(claimTokens))];
@@ -2463,6 +2515,9 @@ var require_scoring = __commonJS({
           const share = (numeric.claimNumbers.length - numeric.missing.length) / numeric.claimNumbers.length;
           score *= NUMERIC_MISMATCH_FACTOR * share;
         }
+        if (!checkPolarityConsistency(claimText, docText).consistent) {
+          score *= POLARITY_MISMATCH_FACTOR;
+        }
       }
       return score;
     }
@@ -2471,8 +2526,11 @@ var require_scoring = __commonJS({
       contentTokens,
       computeSupportScore,
       checkNumericConsistency,
+      checkPolarityConsistency,
+      isNegated,
       extractNumbers,
       NUMERIC_MISMATCH_FACTOR,
+      POLARITY_MISMATCH_FACTOR,
       scoreAnswer,
       tokenize,
       computeBLEU,

package/dist/types/adapters/ai-sdk-middleware.d.ts

@@ -0,0 +1,173 @@
+export type GroundingMiddlewareOptions = {
+    /**
+     * - Any retriever plugin (`retrieve()`); needed unless `pipeline` is given
+     */
+    retriever?: any;
+    /**
+     * - Embeds the query for vector retrievers
+     */
+    embedder?: any;
+    /**
+     * - Optional reranker plugin
+     */
+    reranker?: any;
+    /**
+     * - A createRagPipeline() result to retrieve with, instead of retriever/embedder/reranker; its llm is not called
+     */
+    pipeline?: any;
+    /**
+     * - Passages per call
+     */
+    topK?: number;
+    /**
+     * - Token budget for the passages (see packContext)
+     */
+    maxContextTokens?: number;
+    /**
+     * - Token counter for maxContextTokens
+     */
+    countTokens?: Function;
+    /**
+     * - Replaces the default RAG system prompt
+     */
+    systemPrompt?: string;
+    /**
+     * - Run checkAnswer on the answer
+     */
+    check?: boolean;
+    /**
+     * - An LLM with `generate()` (this library's LLM contract, e.g. `createAiSdkAdapter({ ai, languageModel }).llm`) that rules on the sentences the lexical check failed for lack of word overlap; see checkAnswer. Give it the base model, not the wrapped one, or each judge call is grounded again
+     */
+    judge?: any;
+    /**
+     * - Return the passages as source parts
+     */
+    sources?: boolean;
+    /**
+     * - PreRetrievalGuard options (true for defaults): injection detection on the query
+     */
+    preRetrieval?: any | boolean;
+    /**
+     * - RetrievalGuard options (true for defaults): ACL, minimum score, injection in documents
+     */
+    retrieval?: any | boolean;
+    /**
+     * - PostGenerationGuard options (true for defaults): PII redaction. Streams are buffered when it is on
+     */
+    postGeneration?: any | boolean;
+    /**
+     * - Throw when a guard flags the query or the answer, instead of continuing
+     */
+    strict?: boolean;
+    /**
+     * - `(params) => string`: the retrieval query; defaults to the text of the last user message
+     */
+    getQuery?: Function;
+    /**
+     * - Called with each grounding report
+     */
+    onResult?: Function;
+    /**
+     * - Middleware version for TypeScript: 'v2' for AI SDK 5, 'v3' for 6, 'v4' for 7 (the AI SDK ignores it at runtime)
+     */
+    specificationVersion?: string;
+};
+export type GroundingReport = {
+    /**
+     * - The query passages were retrieved for
+     */
+    query: string;
+    /**
+     * - The passages given to the model
+     */
+    sources: Array<{
+        id: string;
+        score?: number;
+    }>;
+    /**
+     * - Every sentence of the answer is supported by a passage
+     */
+    grounded?: boolean;
+    /**
+     * - Share of supported sentences, 0..1
+     */
+    groundedness?: number;
+    /**
+     * - Sentences no passage supports
+     */
+    unsupported?: string[];
+    /**
+     * - Per-sentence citations
+     */
+    citations?: Array<any>;
+    /**
+     * - With the `judge` option: the verdicts, the number of calls, the token usage when known, and `error` when the judge failed (its sentences then stay unsupported)
+     */
+    judge?: any;
+    /**
+     * - What each enabled guard found
+     */
+    guardrails: any;
+};
+/**
+ * Create AI SDK language model middleware that retrieves context for each
+ * call and checks the answer against it.
+ *
+ * @example
+ * import { generateText, wrapLanguageModel } from "ai";
+ * import { openai } from "@ai-sdk/openai";
+ * import { groundingMiddleware } from "@devilsdev/rag-pipeline-utils";
+ *
+ * const model = wrapLanguageModel({
+ *   model: openai("gpt-4o-mini"),
+ *   middleware: groundingMiddleware({ retriever, embedder }),
+ * });
+ * const { text, sources, providerMetadata } = await generateText({
+ *   model,
+ *   prompt: "How many vacation days do I get?",
+ * });
+ * providerMetadata.ragPipeline.grounded; // true when every sentence is supported
+ *
+ * @param {GroundingMiddlewareOptions} options
+ * @returns {Object} An AI SDK LanguageModelMiddleware
+ */
+export function groundingMiddleware(options?: GroundingMiddlewareOptions): any;
+/**
+ * @typedef {Object} GroundingMiddlewareOptions
+ * @property {Object} [retriever] - Any retriever plugin (`retrieve()`); needed unless `pipeline` is given
+ * @property {Object} [embedder] - Embeds the query for vector retrievers
+ * @property {Object} [reranker] - Optional reranker plugin
+ * @property {Object} [pipeline] - A createRagPipeline() result to retrieve with, instead of retriever/embedder/reranker; its llm is not called
+ * @property {number} [topK=3] - Passages per call
+ * @property {number} [maxContextTokens] - Token budget for the passages (see packContext)
+ * @property {Function} [countTokens] - Token counter for maxContextTokens
+ * @property {string} [systemPrompt] - Replaces the default RAG system prompt
+ * @property {boolean} [check=true] - Run checkAnswer on the answer
+ * @property {Object} [judge] - An LLM with `generate()` (this library's LLM contract, e.g. `createAiSdkAdapter({ ai, languageModel }).llm`) that rules on the sentences the lexical check failed for lack of word overlap; see checkAnswer. Give it the base model, not the wrapped one, or each judge call is grounded again
+ * @property {boolean} [sources=true] - Return the passages as source parts
+ * @property {Object|boolean} [preRetrieval] - PreRetrievalGuard options (true for defaults): injection detection on the query
+ * @property {Object|boolean} [retrieval] - RetrievalGuard options (true for defaults): ACL, minimum score, injection in documents
+ * @property {Object|boolean} [postGeneration] - PostGenerationGuard options (true for defaults): PII redaction. Streams are buffered when it is on
+ * @property {boolean} [strict=false] - Throw when a guard flags the query or the answer, instead of continuing
+ * @property {Function} [getQuery] - `(params) => string`: the retrieval query; defaults to the text of the last user message
+ * @property {Function} [onResult] - Called with each grounding report
+ * @property {string} [specificationVersion='v3'] - Middleware version for TypeScript: 'v2' for AI SDK 5, 'v3' for 6, 'v4' for 7 (the AI SDK ignores it at runtime)
+ */
+/**
+ * @typedef {Object} GroundingReport
+ * @property {string} query - The query passages were retrieved for
+ * @property {Array<{id: string, score?: number}>} sources - The passages given to the model
+ * @property {boolean} [grounded] - Every sentence of the answer is supported by a passage
+ * @property {number} [groundedness] - Share of supported sentences, 0..1
+ * @property {string[]} [unsupported] - Sentences no passage supports
+ * @property {Array<Object>} [citations] - Per-sentence citations
+ * @property {Object} [judge] - With the `judge` option: the verdicts, the number of calls, the token usage when known, and `error` when the judge failed (its sentences then stay unsupported)
+ * @property {Object} guardrails - What each enabled guard found
+ */
+export class GroundingBlockedError extends Error {
+    constructor(message: any, report: any);
+    code: string;
+    report: any;
+}
+/** Key of this middleware in providerOptions (input) and providerMetadata (output). */
+export const METADATA_KEY: "ragPipeline";

package/dist/types/adapters/mastra.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Create a Mastra scorer from one of this library's metrics.
+ *
+ * @example
+ * import { createScorer } from "@mastra/core/evals";
+ * import { createMastraScorer } from "@devilsdev/rag-pipeline-utils";
+ *
+ * const groundedness = createMastraScorer({ createScorer, metric: "groundedness" });
+ * const agent = new Agent({ ..., scorers: { groundedness: { scorer: groundedness } } });
+ *
+ * @param {Object} options
+ * @param {Function} options.createScorer - `createScorer` from "@mastra/core/evals"
+ * @param {'groundedness'|'faithfulness'|'contextPrecision'|'contextRecall'} [options.metric='groundedness']
+ * @param {Array} [options.context] - Fixed passages to score against (strings, { id, content }, LangChain Documents, LlamaIndex nodes)
+ * @param {Function} [options.getContext] - `(run) => passages`; defaults to the results of the agent's tool calls
+ * @param {string} [options.id] - Scorer id (default `rag-<metric>`)
+ * @returns {Object} A Mastra scorer
+ */
+export function createMastraScorer(options?: {
+    createScorer: Function;
+    metric?: "groundedness" | "faithfulness" | "contextPrecision" | "contextRecall";
+    context?: any[];
+    getContext?: Function;
+    id?: string;
+}): any;
+/**
+ * Create a Mastra processor with this library's guardrails: prompt-injection
+ * detection on the user's message (input) and PII redaction of the answer
+ * (output). A flagged message calls Mastra's `abort()`, which stops the run
+ * with a tripwire.
+ *
+ * @example
+ * const agent = new Agent({
+ *   ...,
+ *   inputProcessors: [guardrails],
+ *   outputProcessors: [guardrails],
+ * });
+ * const guardrails = createMastraGuardrails({ injection: true, pii: "redact" });
+ *
+ * @param {Object} [options]
+ * @param {boolean|Object} [options.injection=true] - Check the last user message; PreRetrievalGuard options
+ * @param {'redact'|'block'|false} [options.pii='redact'] - Redact PII in the answer, or abort the run
+ * @param {Object<string, RegExp>} [options.piiPatterns] - Extra PII patterns by name
+ * @param {string} [options.id='rag-pipeline-guardrails']
+ * @returns {Object} A Mastra processor
+ */
+export function createMastraGuardrails(options?: {
+    injection?: boolean | any;
+    pii?: "redact" | "block" | false;
+    piiPatterns?: {
+        [x: string]: RegExp;
+    };
+    id?: string;
+}): any;
+export declare let MASTRA_METRICS: string[];

package/dist/types/evaluate/scoring.d.ts

@@ -24,12 +24,15 @@ export function contentTokens(tokens: string[]): string[];
  * checkNumericConsistency), the score becomes
  *   lexicalScore * NUMERIC_MISMATCH_FACTOR (0.4) * matchedNumbers / claimNumbers
  * which is always below 0.4, so the claim fails the default 0.5 threshold.
+ * Polarity consistency: with the same texts, a claim that is negated where
+ * the document is not, or the reverse (see checkPolarityConsistency), has its
+ * score multiplied by POLARITY_MISMATCH_FACTOR (0.4).
  *
  * @param {string[]} claimTokens - Tokens of the sentence/claim/query
  * @param {string[]|Set<string>} docTokens - Tokens of the document
  * @param {Map<string, number>|null} [idfWeights] - Optional token weights
  * @param {{ claimText?: string, docText?: string }} [options] - Raw texts
- *   enabling the numeric consistency check
+ *   enabling the numeric and polarity consistency checks
  * @returns {number} Support score between 0 and 1
  */
 export function computeSupportScore(claimTokens: string[], docTokens: string[] | Set<string>, idfWeights?: Map<string, number> | null, options?: {
@@ -51,6 +54,37 @@ export function checkNumericConsistency(claimText: string, docText: string): {
     claimNumbers: string[];
     missing: string[];
 };
+/**
+ * Polarity consistency between a claim and a document. Word overlap alone
+ * grounds "employees never receive paid vacation" against a source that says
+ * "employees receive twenty days of paid vacation", because the negation is
+ * one word among many. The claim is compared with the document sentence that
+ * shares most of its content words (any of them, on a tie): it is consistent
+ * when that sentence is negated exactly when the claim is.
+ *
+ * Errs on the side of not grounding: a source that negates in other words
+ * ("contractors are excluded from vacation") does not match a negated claim,
+ * and a sentence joining a positive and a negative statement is judged by
+ * the source sentence closest to it as a whole.
+ *
+ * @param {string} claimText
+ * @param {string} docText
+ * @returns {{ consistent: boolean, claimNegated: boolean, sourceSentence: string|null }}
+ *   sourceSentence: the closest document sentence the claim was compared with
+ */
+export function checkPolarityConsistency(claimText: string, docText: string): {
+    consistent: boolean;
+    claimNegated: boolean;
+    sourceSentence: string | null;
+};
+/**
+ * Whether a token list reads as negated: an odd number of negations, so
+ * "not without approval" counts as positive.
+ *
+ * @param {string[]} tokens
+ * @returns {boolean}
+ */
+export function isNegated(tokens: string[]): boolean;
 /**
  * Extract the numbers stated in a text, as normalised value strings, so
  * "20", "20%", "twenty" and "Twenty" all yield "20", "1,000" yields "1000"
@@ -72,6 +106,12 @@ export function extractNumbers(text: string): {
  * keeps the score below 0.4, i.e. under the default 0.5 support threshold.
  */
 export const NUMERIC_MISMATCH_FACTOR: 0.4;
+/**
+ * Multiplier applied to the support score when a claim's polarity
+ * contradicts the document (see checkPolarityConsistency). It keeps the
+ * score below 0.4, under the default 0.5 support threshold.
+ */
+export const POLARITY_MISMATCH_FACTOR: 0.4;
 /**
  * Score a prompt-answer pair using multiple metrics
  * @param {string} actual - LLM response

package/dist/types/grounding/check-answer.d.ts

@@ -37,6 +37,74 @@ export type AnswerCheck = {
         content: string;
         metadata: any;
     }>;
+    /**
+     * - Only when the `judge` option is given
+     */
+    judge?: JudgeReport;
+};
+/**
+ * Any LLM for the `judge` option: this library's LLM plugin contract.
+ * `generate` may return the text, or an object with `text` or `content`
+ * (and optionally `usage`).
+ */
+export type JudgeLLM = {
+    generate: (prompt: string, context: Array<any>, options?: any) => (string | {
+        text?: string;
+        content?: string;
+        usage?: any;
+    } | Promise<string | {
+        text?: string;
+        content?: string;
+        usage?: any;
+    }>);
+};
+/**
+ * The verdict on one sentence when a judge is used.
+ */
+export type JudgeVerdict = {
+    sentence: string;
+    /**
+     * - The combined verdict
+     */
+    verdict: "supported" | "unsupported";
+    /**
+     * - Id of the supporting source, when supported
+     */
+    sourceId: string | null;
+    /**
+     * - Why, in words (the judge's reason when it decided)
+     */
+    reason: string;
+    /**
+     *   - What the lexical check found. Only `low_overlap` sentences go to the judge
+     */
+    lexicalReason: "supported" | "low_overlap" | "number_mismatch" | "polarity_mismatch" | "no_sources";
+    decidedBy: "lexical" | "judge";
+};
+/**
+ * What the judge did.
+ */
+export type JudgeReport = {
+    /**
+     * - One per sentence, in order
+     */
+    verdicts: JudgeVerdict[];
+    /**
+     * - Calls made to `judge.generate` (0 or 1)
+     */
+    calls: number;
+    /**
+     * - Token usage summed over the calls, when the LLM result carries `usage`
+     */
+    usage?: {
+        inputTokens: number;
+        outputTokens: number;
+        totalTokens: number;
+    };
+    /**
+     * - Why the judge's reply could not be parsed
+     */
+    error?: string;
 };
 /**
  * @typedef {Object} AnswerCheck
@@ -51,6 +119,37 @@ export type AnswerCheck = {
  *   referenceAnswer is given); only when `evaluate` is not false
  * @property {Array<{id: string, content: string, metadata: Object}>} sources
  *   - The sources as they were checked
+ * @property {JudgeReport} [judge] - Only when the `judge` option is given
+ */
+/**
+ * Any LLM for the `judge` option: this library's LLM plugin contract.
+ * `generate` may return the text, or an object with `text` or `content`
+ * (and optionally `usage`).
+ *
+ * @typedef {Object} JudgeLLM
+ * @property {(prompt: string, context: Array<any>, options?: Object) => (string|{text?: string, content?: string, usage?: Object}|Promise<string|{text?: string, content?: string, usage?: Object}>)} generate
+ */
+/**
+ * The verdict on one sentence when a judge is used.
+ *
+ * @typedef {Object} JudgeVerdict
+ * @property {string} sentence
+ * @property {'supported'|'unsupported'} verdict - The combined verdict
+ * @property {string|null} sourceId - Id of the supporting source, when supported
+ * @property {string} reason - Why, in words (the judge's reason when it decided)
+ * @property {'supported'|'low_overlap'|'number_mismatch'|'polarity_mismatch'|'no_sources'} lexicalReason
+ *   - What the lexical check found. Only `low_overlap` sentences go to the judge
+ * @property {'lexical'|'judge'} decidedBy
+ */
+/**
+ * What the judge did.
+ *
+ * @typedef {Object} JudgeReport
+ * @property {JudgeVerdict[]} verdicts - One per sentence, in order
+ * @property {number} calls - Calls made to `judge.generate` (0 or 1)
+ * @property {{inputTokens: number, outputTokens: number, totalTokens: number}} [usage]
+ *   - Token usage summed over the calls, when the LLM result carries `usage`
+ * @property {string} [error] - Why the judge's reply could not be parsed
  */
 /**
  * Check an answer against the sources it was generated from.
@@ -74,15 +173,24 @@ export type AnswerCheck = {
  * @param {Object} [params.embedder] - With a referenceAnswer, adds
  *   `evaluation.scores.semanticSimilarity` (computeSemanticSimilarity)
  * @param {boolean} [params.evaluate=true] - Also compute evaluation scores
+ * @param {JudgeLLM} [params.judge] - Opt-in LLM judge for sentences the
+ *   lexical check failed for lack of word overlap (correct paraphrases).
+ *   It never overturns a sentence that failed the number or negation check,
+ *   and a "supported" verdict must name a source whose numbers and negation
+ *   match the sentence. Unparseable output keeps a sentence unsupported.
+ *   With a judge, `grounded`, `groundedness` and `unsupported` combine both
+ *   checks and `judge` reports the verdicts; `citations` and `evaluation`
+ *   stay lexical. Errors thrown by `judge.generate` are not caught.
  * @returns {Promise<AnswerCheck>}
  */
-export function checkAnswer({ answer, sources, question, referenceAnswer, embedder, evaluate, }?: {
+export function checkAnswer({ answer, sources, question, referenceAnswer, embedder, evaluate, judge, }?: {
     answer: string;
     sources: Array<string | any>;
     question?: string;
     referenceAnswer?: string;
     embedder?: any;
     evaluate?: boolean;
+    judge?: JudgeLLM;
 }): Promise<AnswerCheck>;
 /**
  * Normalize sources to `{ id, content, metadata }`.

package/dist/types/grounding/llm-judge.d.ts

@@ -0,0 +1,86 @@
+export namespace REASONS {
+    let supported: string;
+    let lowOverlap: string;
+    let numberMismatch: string;
+    let polarityMismatch: string;
+    let noSources: string;
+}
+/** Instructions at the top of every judge prompt. */
+export const JUDGE_INSTRUCTIONS: string;
+/**
+ * Build the judge prompt for the given claims.
+ *
+ * @param {string[]} claims - Sentences to rule on
+ * @param {Array<{id: string, content: string}>} docs
+ * @returns {string}
+ */
+export function buildJudgePrompt(claims: string[], docs: Array<{
+    id: string;
+    content: string;
+}>): string;
+/**
+ * Parse the judge's reply into one verdict per claim. Tolerates a code fence
+ * around the JSON; anything else that is not the expected array leaves every
+ * claim without a verdict.
+ *
+ * @param {*} raw - What llm.generate() returned
+ * @param {number} claimCount
+ * @returns {{ verdicts: Array<{verdict: string, source: string|null, reason: string}|null>, error: string|null }}
+ *   verdicts[i] is null when claim i+1 got no well-formed verdict
+ */
+export function parseJudgeVerdicts(raw: any, claimCount: number): {
+    verdicts: Array<{
+        verdict: string;
+        source: string | null;
+        reason: string;
+    } | null>;
+    error: string | null;
+};
+/**
+ * The lexical reason for every sentence, in order.
+ *
+ * @param {Array<{text: string, classification: string}>} sentences - From
+ *   CitationTracker's hallucination report
+ * @param {Array<{content: string}>} docs
+ * @returns {string[]}
+ */
+export function lexicalReasons(sentences: Array<{
+    text: string;
+    classification: string;
+}>, docs: Array<{
+    content: string;
+}>): string[];
+/**
+ * Run the judge on the sentences that failed for lack of overlap and
+ * combine its verdicts with the lexical result.
+ *
+ * Overturn rule: a judge "supported" counts only when the sentence failed
+ * for lack of overlap, the verdict is well formed, it names a source that
+ * exists, and that source passes the numeric and polarity checks against
+ * the sentence. Everything else keeps the lexical verdict.
+ *
+ * @param {Object} params
+ * @param {{generate: Function}} params.judge
+ * @param {Array<{text: string, classification: string, bestSource: Object|null}>} params.sentences
+ * @param {Array<{id: string, content: string}>} params.docs
+ * @returns {Promise<{verdicts: Array<Object>, calls: number, usage?: Object, error?: string}>}
+ */
+export function runJudge({ judge, sentences, docs }: {
+    judge: {
+        generate: Function;
+    };
+    sentences: Array<{
+        text: string;
+        classification: string;
+        bestSource: any | null;
+    }>;
+    docs: Array<{
+        id: string;
+        content: string;
+    }>;
+}): Promise<{
+    verdicts: Array<any>;
+    calls: number;
+    usage?: any;
+    error?: string;
+}>;

package/dist/types/testing/matchers.d.ts

@@ -0,0 +1,53 @@
+export namespace ragMatchers {
+    export { toBeGroundedIn };
+    export { toCite };
+    export { toContainNoPII };
+}
+/**
+ * Passes when every sentence of the answer is supported by the sources: its
+ * content words, numbers and negation match a source (see checkAnswer).
+ *
+ * @param {string|{answer: string, results?: Array}} received - An answer, or a run() result
+ * @param {Array} [sources] - Defaults to the run() result's `results`
+ * @param {{threshold?: number}} [options] - Support threshold per sentence (default 0.5)
+ */
+export function toBeGroundedIn(received: string | {
+    answer: string;
+    results?: any[];
+}, sources?: any[], options?: {
+    threshold?: number;
+}): {
+    pass: boolean;
+    message: () => string;
+};
+/**
+ * Passes when the answer cites each of the given sources: at least one of its
+ * sentences is supported by that source.
+ *
+ * @param {string|{answer: string, results?: Array}} received - An answer, or a run() result
+ * @param {string|string[]} ids - Source id(s) that must be cited
+ * @param {Array} [sources] - Defaults to the run() result's `results`
+ */
+export function toCite(received: string | {
+    answer: string;
+    results?: any[];
+}, ids: string | string[], sources?: any[]): {
+    pass: boolean;
+    message: () => string;
+};
+/**
+ * Passes when no email address, phone number, US social security number or
+ * card number (the post-generation guard's patterns) appears in the value: a
+ * string, or every string nested in an object such as a run() result.
+ *
+ * @param {*} received
+ * @param {{patterns?: Object<string, RegExp>}} [options] - Extra patterns by name
+ */
+export function toContainNoPII(received: any, options?: {
+    patterns?: {
+        [x: string]: RegExp;
+    };
+}): {
+    pass: boolean;
+    message: () => string;
+};

package/dist/types/testing.d.ts

@@ -169,5 +169,9 @@ export function createTestPipeline({ documents, embedder, llm, pipeline: pipelin
 }>;
 import { runContractTests } from "./testing/contract-tests";
 import { checkContract } from "./testing/contract-tests";
+import { ragMatchers } from "./testing/matchers";
+import { toBeGroundedIn } from "./testing/matchers";
+import { toCite } from "./testing/matchers";
+import { toContainNoPII } from "./testing/matchers";
 import { MemoryRetriever } from "./connectors/memory-retriever";
-export { runContractTests, checkContract };
+export { runContractTests, checkContract, ragMatchers, toBeGroundedIn, toCite, toContainNoPII };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devilsdev/rag-core",
-  "version": "3.2.0",
+  "version": "3.4.0",
   "description": "createRagPipeline and its building blocks: retrieval, reranking, caching, cost tracking and model routing.",
   "keywords": [
     "rag",