@devilsdev/rag-core 3.3.0 → 3.4.0

package/dist/index.cjs

@@ -1296,7 +1296,7 @@ var require_package = __commonJS({
   "package.json"(exports2, module2) {
     module2.exports = {
       name: "@devilsdev/rag-pipeline-utils",
-      version: "3.3.0",
+      version: "3.4.0",
       description: "Composable retrieval-augmented generation (RAG) for Node.js: grounded citations, evaluation, guardrails, hybrid retrieval and a CLI.",
       main: "dist/index.cjs",
       type: "commonjs",
@@ -1550,10 +1550,10 @@ var require_package = __commonJS({
         "why-is-node-running": "^3.2.2"
       },
       dependencies: {
-        "@devilsdev/rag-connectors": "3.3.0",
-        "@devilsdev/rag-core": "3.3.0",
-        "@devilsdev/rag-eval": "3.3.0",
-        "@devilsdev/rag-guardrails": "3.3.0",
+        "@devilsdev/rag-connectors": "3.4.0",
+        "@devilsdev/rag-core": "3.4.0",
+        "@devilsdev/rag-eval": "3.4.0",
+        "@devilsdev/rag-guardrails": "3.4.0",
         ajv: "^8.17.1",
         axios: "^1.21.0",
         chokidar: "^4.0.3",

package/dist/types/adapters/ai-sdk-middleware.d.ts

@@ -35,6 +35,10 @@ export type GroundingMiddlewareOptions = {
      * - Run checkAnswer on the answer
      */
     check?: boolean;
+    /**
+     * - An LLM with `generate()` (this library's LLM contract, e.g. `createAiSdkAdapter({ ai, languageModel }).llm`) that rules on the sentences the lexical check failed for lack of word overlap; see checkAnswer. Give it the base model, not the wrapped one, or each judge call is grounded again
+     */
+    judge?: any;
     /**
      * - Return the passages as source parts
      */
@@ -96,6 +100,10 @@ export type GroundingReport = {
      * - Per-sentence citations
      */
     citations?: Array<any>;
+    /**
+     * - With the `judge` option: the verdicts, the number of calls, the token usage when known, and `error` when the judge failed (its sentences then stay unsupported)
+     */
+    judge?: any;
     /**
      * - What each enabled guard found
      */
@@ -135,6 +143,7 @@ export function groundingMiddleware(options?: GroundingMiddlewareOptions): any;
  * @property {Function} [countTokens] - Token counter for maxContextTokens
  * @property {string} [systemPrompt] - Replaces the default RAG system prompt
  * @property {boolean} [check=true] - Run checkAnswer on the answer
+ * @property {Object} [judge] - An LLM with `generate()` (this library's LLM contract, e.g. `createAiSdkAdapter({ ai, languageModel }).llm`) that rules on the sentences the lexical check failed for lack of word overlap; see checkAnswer. Give it the base model, not the wrapped one, or each judge call is grounded again
  * @property {boolean} [sources=true] - Return the passages as source parts
  * @property {Object|boolean} [preRetrieval] - PreRetrievalGuard options (true for defaults): injection detection on the query
  * @property {Object|boolean} [retrieval] - RetrievalGuard options (true for defaults): ACL, minimum score, injection in documents
@@ -152,6 +161,7 @@ export function groundingMiddleware(options?: GroundingMiddlewareOptions): any;
  * @property {number} [groundedness] - Share of supported sentences, 0..1
  * @property {string[]} [unsupported] - Sentences no passage supports
  * @property {Array<Object>} [citations] - Per-sentence citations
+ * @property {Object} [judge] - With the `judge` option: the verdicts, the number of calls, the token usage when known, and `error` when the judge failed (its sentences then stay unsupported)
  * @property {Object} guardrails - What each enabled guard found
  */
 export class GroundingBlockedError extends Error {

package/dist/types/grounding/check-answer.d.ts

@@ -37,6 +37,74 @@ export type AnswerCheck = {
         content: string;
         metadata: any;
     }>;
+    /**
+     * - Only when the `judge` option is given
+     */
+    judge?: JudgeReport;
+};
+/**
+ * Any LLM for the `judge` option: this library's LLM plugin contract.
+ * `generate` may return the text, or an object with `text` or `content`
+ * (and optionally `usage`).
+ */
+export type JudgeLLM = {
+    generate: (prompt: string, context: Array<any>, options?: any) => (string | {
+        text?: string;
+        content?: string;
+        usage?: any;
+    } | Promise<string | {
+        text?: string;
+        content?: string;
+        usage?: any;
+    }>);
+};
+/**
+ * The verdict on one sentence when a judge is used.
+ */
+export type JudgeVerdict = {
+    sentence: string;
+    /**
+     * - The combined verdict
+     */
+    verdict: "supported" | "unsupported";
+    /**
+     * - Id of the supporting source, when supported
+     */
+    sourceId: string | null;
+    /**
+     * - Why, in words (the judge's reason when it decided)
+     */
+    reason: string;
+    /**
+     *   - What the lexical check found. Only `low_overlap` sentences go to the judge
+     */
+    lexicalReason: "supported" | "low_overlap" | "number_mismatch" | "polarity_mismatch" | "no_sources";
+    decidedBy: "lexical" | "judge";
+};
+/**
+ * What the judge did.
+ */
+export type JudgeReport = {
+    /**
+     * - One per sentence, in order
+     */
+    verdicts: JudgeVerdict[];
+    /**
+     * - Calls made to `judge.generate` (0 or 1)
+     */
+    calls: number;
+    /**
+     * - Token usage summed over the calls, when the LLM result carries `usage`
+     */
+    usage?: {
+        inputTokens: number;
+        outputTokens: number;
+        totalTokens: number;
+    };
+    /**
+     * - Why the judge's reply could not be parsed
+     */
+    error?: string;
 };
 /**
  * @typedef {Object} AnswerCheck
@@ -51,6 +119,37 @@ export type AnswerCheck = {
  *   referenceAnswer is given); only when `evaluate` is not false
  * @property {Array<{id: string, content: string, metadata: Object}>} sources
  *   - The sources as they were checked
+ * @property {JudgeReport} [judge] - Only when the `judge` option is given
+ */
+/**
+ * Any LLM for the `judge` option: this library's LLM plugin contract.
+ * `generate` may return the text, or an object with `text` or `content`
+ * (and optionally `usage`).
+ *
+ * @typedef {Object} JudgeLLM
+ * @property {(prompt: string, context: Array<any>, options?: Object) => (string|{text?: string, content?: string, usage?: Object}|Promise<string|{text?: string, content?: string, usage?: Object}>)} generate
+ */
+/**
+ * The verdict on one sentence when a judge is used.
+ *
+ * @typedef {Object} JudgeVerdict
+ * @property {string} sentence
+ * @property {'supported'|'unsupported'} verdict - The combined verdict
+ * @property {string|null} sourceId - Id of the supporting source, when supported
+ * @property {string} reason - Why, in words (the judge's reason when it decided)
+ * @property {'supported'|'low_overlap'|'number_mismatch'|'polarity_mismatch'|'no_sources'} lexicalReason
+ *   - What the lexical check found. Only `low_overlap` sentences go to the judge
+ * @property {'lexical'|'judge'} decidedBy
+ */
+/**
+ * What the judge did.
+ *
+ * @typedef {Object} JudgeReport
+ * @property {JudgeVerdict[]} verdicts - One per sentence, in order
+ * @property {number} calls - Calls made to `judge.generate` (0 or 1)
+ * @property {{inputTokens: number, outputTokens: number, totalTokens: number}} [usage]
+ *   - Token usage summed over the calls, when the LLM result carries `usage`
+ * @property {string} [error] - Why the judge's reply could not be parsed
  */
 /**
  * Check an answer against the sources it was generated from.
@@ -74,15 +173,24 @@ export type AnswerCheck = {
  * @param {Object} [params.embedder] - With a referenceAnswer, adds
  *   `evaluation.scores.semanticSimilarity` (computeSemanticSimilarity)
  * @param {boolean} [params.evaluate=true] - Also compute evaluation scores
+ * @param {JudgeLLM} [params.judge] - Opt-in LLM judge for sentences the
+ *   lexical check failed for lack of word overlap (correct paraphrases).
+ *   It never overturns a sentence that failed the number or negation check,
+ *   and a "supported" verdict must name a source whose numbers and negation
+ *   match the sentence. Unparseable output keeps a sentence unsupported.
+ *   With a judge, `grounded`, `groundedness` and `unsupported` combine both
+ *   checks and `judge` reports the verdicts; `citations` and `evaluation`
+ *   stay lexical. Errors thrown by `judge.generate` are not caught.
  * @returns {Promise<AnswerCheck>}
  */
-export function checkAnswer({ answer, sources, question, referenceAnswer, embedder, evaluate, }?: {
+export function checkAnswer({ answer, sources, question, referenceAnswer, embedder, evaluate, judge, }?: {
     answer: string;
     sources: Array<string | any>;
     question?: string;
     referenceAnswer?: string;
     embedder?: any;
     evaluate?: boolean;
+    judge?: JudgeLLM;
 }): Promise<AnswerCheck>;
 /**
  * Normalize sources to `{ id, content, metadata }`.

package/dist/types/grounding/llm-judge.d.ts

@@ -0,0 +1,86 @@
+export namespace REASONS {
+    let supported: string;
+    let lowOverlap: string;
+    let numberMismatch: string;
+    let polarityMismatch: string;
+    let noSources: string;
+}
+/** Instructions at the top of every judge prompt. */
+export const JUDGE_INSTRUCTIONS: string;
+/**
+ * Build the judge prompt for the given claims.
+ *
+ * @param {string[]} claims - Sentences to rule on
+ * @param {Array<{id: string, content: string}>} docs
+ * @returns {string}
+ */
+export function buildJudgePrompt(claims: string[], docs: Array<{
+    id: string;
+    content: string;
+}>): string;
+/**
+ * Parse the judge's reply into one verdict per claim. Tolerates a code fence
+ * around the JSON; anything else that is not the expected array leaves every
+ * claim without a verdict.
+ *
+ * @param {*} raw - What llm.generate() returned
+ * @param {number} claimCount
+ * @returns {{ verdicts: Array<{verdict: string, source: string|null, reason: string}|null>, error: string|null }}
+ *   verdicts[i] is null when claim i+1 got no well-formed verdict
+ */
+export function parseJudgeVerdicts(raw: any, claimCount: number): {
+    verdicts: Array<{
+        verdict: string;
+        source: string | null;
+        reason: string;
+    } | null>;
+    error: string | null;
+};
+/**
+ * The lexical reason for every sentence, in order.
+ *
+ * @param {Array<{text: string, classification: string}>} sentences - From
+ *   CitationTracker's hallucination report
+ * @param {Array<{content: string}>} docs
+ * @returns {string[]}
+ */
+export function lexicalReasons(sentences: Array<{
+    text: string;
+    classification: string;
+}>, docs: Array<{
+    content: string;
+}>): string[];
+/**
+ * Run the judge on the sentences that failed for lack of overlap and
+ * combine its verdicts with the lexical result.
+ *
+ * Overturn rule: a judge "supported" counts only when the sentence failed
+ * for lack of overlap, the verdict is well formed, it names a source that
+ * exists, and that source passes the numeric and polarity checks against
+ * the sentence. Everything else keeps the lexical verdict.
+ *
+ * @param {Object} params
+ * @param {{generate: Function}} params.judge
+ * @param {Array<{text: string, classification: string, bestSource: Object|null}>} params.sentences
+ * @param {Array<{id: string, content: string}>} params.docs
+ * @returns {Promise<{verdicts: Array<Object>, calls: number, usage?: Object, error?: string}>}
+ */
+export function runJudge({ judge, sentences, docs }: {
+    judge: {
+        generate: Function;
+    };
+    sentences: Array<{
+        text: string;
+        classification: string;
+        bestSource: any | null;
+    }>;
+    docs: Array<{
+        id: string;
+        content: string;
+    }>;
+}): Promise<{
+    verdicts: Array<any>;
+    calls: number;
+    usage?: any;
+    error?: string;
+}>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devilsdev/rag-core",
-  "version": "3.3.0",
+  "version": "3.4.0",
   "description": "createRagPipeline and its building blocks: retrieval, reranking, caching, cost tracking and model routing.",
   "keywords": [
     "rag",