@devilsdev/rag-pipeline-utils 2.4.5 → 2.6.0

package/src/evaluate/semantic-similarity.js

@@ -0,0 +1,79 @@
+'use strict';
+
+/**
+ * Semantic similarity between an answer and a reference answer, from the
+ * cosine similarity of their embeddings. Unlike BLEU and ROUGE it does not
+ * penalise a correct answer that is worded differently.
+ *
+ * Embeddings capture topic and meaning, not facts: "twenty days" and
+ * "thirty days" score as very similar. Use it next to the grounding checks
+ * (computeFaithfulness, checkAnswer), not instead of them.
+ */
+
+function cosine(a, b) {
+  if (
+    !Array.isArray(a) ||
+    !Array.isArray(b) ||
+    a.length !== b.length ||
+    a.length === 0
+  ) {
+    throw new Error(
+      `Embeddings must be non-empty arrays of the same length (got ${a && a.length} and ${b && b.length})`,
+    );
+  }
+  let dot = 0;
+  let na = 0;
+  let nb = 0;
+  for (let i = 0; i < a.length; i += 1) {
+    dot += a[i] * b[i];
+    na += a[i] * a[i];
+    nb += b[i] * b[i];
+  }
+  if (na === 0 || nb === 0) return 0;
+  return dot / (Math.sqrt(na) * Math.sqrt(nb));
+}
+
+async function embedText(embedder, text, signal) {
+  const options = signal ? { signal } : {};
+  if (typeof embedder.embedQuery === 'function') {
+    return embedder.embedQuery(text, options);
+  }
+  return embedder.embed(text, options);
+}
+
+/**
+ * @param {string} answer - The generated answer
+ * @param {string} reference - The reference (ground-truth) answer
+ * @param {Object} options
+ * @param {Object} options.embedder - Any embedder with embed() or embedQuery()
+ *   (a built-in connector, LocalEmbedder fitted to your corpus, an AI SDK
+ *   adapter, HashEmbedder from /testing)
+ * @param {AbortSignal} [options.signal]
+ * @returns {Promise<number>} Cosine similarity clamped to 0..1
+ */
+async function computeSemanticSimilarity(
+  answer,
+  reference,
+  { embedder, signal } = {},
+) {
+  if (typeof answer !== 'string' || typeof reference !== 'string') {
+    throw new TypeError('answer and reference must be strings');
+  }
+  if (
+    !embedder ||
+    (typeof embedder.embed !== 'function' &&
+      typeof embedder.embedQuery !== 'function')
+  ) {
+    throw new TypeError(
+      'computeSemanticSimilarity needs an embedder with embed() or embedQuery()',
+    );
+  }
+  if (answer.trim() === '' || reference.trim() === '') return 0;
+  const [a, b] = await Promise.all([
+    embedText(embedder, answer, signal),
+    embedText(embedder, reference, signal),
+  ]);
+  return Math.min(1, Math.max(0, cosine(a, b)));
+}
+
+module.exports = { computeSemanticSimilarity };

package/src/experimental.js

@@ -0,0 +1,71 @@
+'use strict';
+
+/**
+ * @devilsdev/rag-pipeline-utils/experimental
+ *
+ * Modules that are simulated, incomplete or not safe for production. They
+ * are kept for experimentation and are NOT covered by the semver guarantees
+ * in SEMVER.md. See docs-site/docs/Experimental.md for what each one really
+ * does. Do not use them for authentication, tenant isolation, compliance or
+ * plugin isolation.
+ *
+ * - Model training / federated learning: metrics and weights are random.
+ * - MultiModalProcessor: returns random embeddings; only stores content.
+ * - AdaptiveRetrievalEngine: in-memory placeholder strategy.
+ * - SSO providers: OAuth2/OIDC use PKCE and OIDC validates id_tokens, but
+ *   the flows are lightly tested; SAML refuses to authenticate (no XML
+ *   signature verification); Active Directory is a development stub. A
+ *   development bypass (any code accepted, fixed identity) exists only
+ *   with `devBypass: true` outside production.
+ * - TenantManager / ResourceMonitor: usage figures are random.
+ * - DataGovernanceManager: random classification; erasure deletes nothing.
+ * - PluginSandbox: execute() is not functional; plugins are not isolated.
+ * - PluginCertification: automated checks return fixed scores (use
+ *   `rag-pipeline hub certify check` for computed scores).
+ */
+
+const { MultiModalProcessor } = require('./ai/multimodal');
+const { AdaptiveRetrievalEngine } = require('./ai/retrieval-engine');
+const {
+  FederatedLearningCoordinator,
+} = require('./ai/federation/federated-learning-coordinator');
+const {
+  ModelTrainingOrchestrator,
+} = require('./ai/training/model-training-orchestrator');
+const { DataGovernanceManager } = require('./enterprise/data-governance');
+const {
+  TenantManager,
+  ResourceMonitor,
+} = require('./enterprise/multi-tenancy');
+const {
+  SSOManager,
+  SAMLProvider,
+  OAuth2Provider,
+  ActiveDirectoryProvider,
+  OIDCProvider,
+} = require('./enterprise/sso-integration');
+const { PluginSandbox } = require('./security/plugin-sandbox');
+const { PluginCertification } = require('./ecosystem/plugin-certification');
+
+module.exports = {
+  // AI/ML
+  MultiModalProcessor,
+  AdaptiveRetrievalEngine,
+  FederatedLearningCoordinator,
+  ModelTrainingOrchestrator,
+
+  // Enterprise
+  DataGovernanceManager,
+  DataGovernance: DataGovernanceManager,
+  TenantManager,
+  ResourceMonitor,
+  SSOManager,
+  SAMLProvider,
+  OAuth2Provider,
+  ActiveDirectoryProvider,
+  OIDCProvider,
+
+  // Security / ecosystem
+  PluginSandbox,
+  PluginCertification,
+};

package/src/grounding/check-answer.js

@@ -0,0 +1,173 @@
+'use strict';
+
+/**
+ * Check an answer that any framework produced (LangChain, LlamaIndex, Mastra,
+ * the AI SDK, your own code) against the sources it was given: citations
+ * per sentence, groundedness, unsupported sentences and evaluation scores,
+ * without using createRagPipeline().
+ */
+
+const { CitationTracker } = require('../citation/citation-tracker');
+const { PipelineEvaluator } = require('../evaluate/pipeline-evaluator');
+const {
+  computeSemanticSimilarity,
+} = require('../evaluate/semantic-similarity');
+
+/**
+ * Text of one source in the shapes common frameworks use.
+ * @private
+ */
+function sourceText(source) {
+  if (source == null) return '';
+  if (typeof source === 'string') return source;
+  if (typeof source.content === 'string') return source.content; // this library
+  if (typeof source.pageContent === 'string') return source.pageContent; // LangChain Document
+  if (typeof source.text === 'string') return source.text; // LlamaIndex node, plain
+  if (source.node) return sourceText(source.node); // LlamaIndex NodeWithScore
+  if (typeof source.getContent === 'function')
+    return String(source.getContent());
+  return '';
+}
+
+/**
+ * An id for a source: its own id when it has one, otherwise its position.
+ * @private
+ */
+function sourceId(source, index) {
+  const candidates = [
+    source && source.id,
+    source && source.id_,
+    source && source.node && (source.node.id_ || source.node.id),
+    source && source.metadata && (source.metadata.id || source.metadata.source),
+  ];
+  const id = candidates.find(
+    (c) => typeof c === 'string' || typeof c === 'number',
+  );
+  return id !== undefined ? String(id) : String(index + 1);
+}
+
+/**
+ * Normalize sources to `{ id, content, metadata }`.
+ *
+ * Accepts strings, `{ id, content }` (this library), LangChain Documents
+ * (`{ pageContent, metadata }`), LlamaIndex nodes (`{ text }`) and
+ * `NodeWithScore` (`{ node, score }`).
+ *
+ * @param {Array<string|Object>} sources
+ * @returns {Array<{id: string, content: string, metadata: Object}>}
+ */
+function normalizeSources(sources) {
+  if (!Array.isArray(sources)) {
+    throw new TypeError('sources must be an array');
+  }
+  return sources
+    .map((source, index) => ({
+      id: sourceId(source, index),
+      content: sourceText(source),
+      metadata:
+        (source &&
+          (source.metadata || (source.node && source.node.metadata))) ||
+        {},
+    }))
+    .filter((s) => s.content.trim() !== '');
+}
+
+/**
+ * @typedef {Object} AnswerCheck
+ * @property {boolean} grounded - Every sentence is supported by a source
+ * @property {number} groundedness - Share of supported sentences, 0..1
+ * @property {string[]} unsupported - Sentences no source supports (this
+ *   includes a sentence whose numbers do not match its source)
+ * @property {Array<{sentence: string, sources: Array}>} citations - Sources
+ *   supporting each sentence
+ * @property {{scores: Object, details: Object, metadata: Object}|undefined} evaluation
+ *   - Faithfulness, relevance, context precision (and context recall when a
+ *   referenceAnswer is given); only when `evaluate` is not false
+ * @property {Array<{id: string, content: string, metadata: Object}>} sources
+ *   - The sources as they were checked
+ */
+
+/**
+ * Check an answer against the sources it was generated from.
+ *
+ * @example
+ * // A LangChain chain's output
+ * const check = await checkAnswer({
+ *   question: input,
+ *   answer: result.answer,
+ *   sources: result.context, // LangChain Documents
+ * });
+ * if (!check.grounded) console.warn(check.unsupported);
+ *
+ * @param {Object} params
+ * @param {string} params.answer - The generated answer
+ * @param {Array<string|Object>} params.sources - Documents the answer was
+ *   generated from (see normalizeSources for accepted shapes)
+ * @param {string} [params.question] - The question; needed for relevance
+ * @param {string} [params.referenceAnswer] - Ground truth, for context recall
+ *   (and semantic similarity when an embedder is given)
+ * @param {Object} [params.embedder] - With a referenceAnswer, adds
+ *   `evaluation.scores.semanticSimilarity` (computeSemanticSimilarity)
+ * @param {boolean} [params.evaluate=true] - Also compute evaluation scores
+ * @returns {Promise<AnswerCheck>}
+ */
+async function checkAnswer({
+  answer,
+  sources,
+  question,
+  referenceAnswer,
+  embedder,
+  evaluate = true,
+} = {}) {
+  if (typeof answer !== 'string') {
+    throw new TypeError('answer must be a string');
+  }
+  const docs = normalizeSources(sources);
+
+  // Models sometimes return nothing; that is an ungrounded answer, not an
+  // error. (CitationTracker rejects empty text.)
+  if (answer.trim() === '') {
+    return {
+      grounded: false,
+      groundedness: 0,
+      unsupported: [],
+      citations: [],
+      evaluation: undefined,
+      sources: docs,
+    };
+  }
+
+  const tracked = new CitationTracker().track(answer, docs);
+  const sentences =
+    (tracked.hallucinationReport && tracked.hallucinationReport.sentences) ||
+    [];
+  const unsupported = sentences
+    .filter((s) => s.classification !== 'grounded')
+    .map((s) => s.text);
+
+  let evaluation;
+  if (evaluate) {
+    evaluation = await new PipelineEvaluator().evaluate(
+      { query: question || '', answer, results: docs },
+      referenceAnswer ? { referenceAnswer } : {},
+    );
+    if (referenceAnswer && embedder) {
+      evaluation.scores.semanticSimilarity = await computeSemanticSimilarity(
+        answer,
+        referenceAnswer,
+        { embedder },
+      );
+    }
+  }
+
+  return {
+    grounded: sentences.length > 0 && unsupported.length === 0,
+    groundedness: tracked.groundednessScore,
+    unsupported,
+    citations: tracked.citations,
+    evaluation,
+    sources: docs,
+  };
+}
+
+module.exports = { checkAnswer, normalizeSources };

package/src/guardrails/guardrails-pipeline.js

@@ -16,6 +16,20 @@ const DEFAULT_CONFIG = {
   strict: false, // If true, block on any guard failure; if false, warn and continue
 };
 
+/**
+ * Reduce removed documents to their ids so filtered-out (possibly
+ * access-controlled) content is never returned to the caller.
+ *
+ * @param {{results: Array, removed: Array, metadata: object}} filterOutput
+ * @returns {{results: Array, removed: Array<{id: *}>, metadata: object}}
+ */
+function summarizeFilter(filterOutput) {
+  return {
+    ...filterOutput,
+    removed: filterOutput.removed.map((doc) => ({ id: doc.id })),
+  };
+}
+
 /**
  * Pipeline wrapper that applies all three guardrail layers (pre-retrieval,
  * retrieval, and post-generation) around an existing RAG pipeline. In strict
@@ -92,21 +106,49 @@ class GuardrailsPipeline extends EventEmitter {
       query = guardrails.preRetrieval.sanitizedQuery;
     }
 
-    // 2. Run the underlying pipeline with (possibly sanitized) query
+    // 2. Run the underlying pipeline with (possibly sanitized) query. The
+    // retrieval guard is passed in as options.filterResults so it runs
+    // between retrieval and generation (supported by createRagPipeline).
+    let preGenerationFilter = null;
+    const filterOptions = this.retrievalGuard
+      ? {
+          filterResults: (docs) => {
+            preGenerationFilter = this.retrievalGuard.filter(
+              docs,
+              input.context,
+            );
+            return preGenerationFilter.results;
+          },
+        }
+      : {};
     const result = await this.pipeline.run({
       ...input,
       query,
-      options: { ...input.options, citations: true },
+      options: { ...input.options, citations: true, ...filterOptions },
     });
 
     // 3. Retrieval guard on results
-    if (this.retrievalGuard && result.results) {
-      guardrails.retrieval = this.retrievalGuard.filter(
-        result.results,
-        input.context,
+    if (this.retrievalGuard && preGenerationFilter) {
+      guardrails.retrieval = summarizeFilter(preGenerationFilter);
+      this.emit('retrieval', guardrails.retrieval);
+    } else if (this.retrievalGuard && result.results) {
+      // The wrapped pipeline ignored options.filterResults, so it generated
+      // from unfiltered documents. Filtering now only fixes result.results;
+      // if ACL removed anything the answer may contain restricted content.
+      guardrails.retrieval = summarizeFilter(
+        this.retrievalGuard.filter(result.results, input.context),
       );
       this.emit('retrieval', guardrails.retrieval);
       result.results = guardrails.retrieval.results;
+
+      if (guardrails.retrieval.metadata.reasons.acl) {
+        return {
+          success: false,
+          error:
+            'Retrieval guard removed access-controlled documents after generation; answer withheld',
+          guardrails,
+        };
+      }
     }
 
     // 4. Post-generation guard on answer
@@ -118,19 +160,39 @@ class GuardrailsPipeline extends EventEmitter {
       this.emit('postGeneration', guardrails.postGeneration);
 
       if (!guardrails.postGeneration.safe && this.config.strict) {
+        // Do not spread result: it carries the raw answer and citations.
         return {
-          ...result,
           success: false,
           error: 'Post-generation guard flagged output',
           guardrails,
         };
       }
       result.answer = guardrails.postGeneration.sanitizedOutput;
+      this._redactDeep(result.citations);
+      this._redactDeep(result.evaluation);
     }
 
     result.guardrails = guardrails;
     return result;
   }
+
+  /**
+   * Citations and evaluation details copy sentences from the raw answer, so
+   * scrub every string in them with the same PII patterns as the answer.
+   *
+   * @param {*} value - Object, array or string to redact in place
+   * @returns {*} The redacted value
+   * @private
+   */
+  _redactDeep(value) {
+    if (typeof value === 'string') return this.postGuard.redact(value);
+    if (value && typeof value === 'object') {
+      for (const key of Object.keys(value)) {
+        value[key] = this._redactDeep(value[key]);
+      }
+    }
+    return value;
+  }
 }
 
 module.exports = { GuardrailsPipeline, DEFAULT_CONFIG };