@devilsdev/rag-pipeline-utils 2.5.0 → 2.6.0

package/dist/types/core/validate-plugin.d.ts

@@ -0,0 +1,45 @@
+export type PluginValidationResult = {
+    /**
+     * - True when there are no errors
+     */
+    valid: boolean;
+    /**
+     * - Problems that stop the plugin from working
+     */
+    errors: string[];
+    /**
+     * - Methods some commands need, e.g. `store`
+     * for `rag-pipeline ingest`
+     */
+    warnings: string[];
+};
+/**
+ * @typedef {Object} PluginValidationResult
+ * @property {boolean} valid - True when there are no errors
+ * @property {string[]} errors - Problems that stop the plugin from working
+ * @property {string[]} warnings - Methods some commands need, e.g. `store`
+ *   for `rag-pipeline ingest`
+ */
+/**
+ * Checks that an object can be used as a pipeline stage, without throwing,
+ * so plugin authors can assert on it in their own tests.
+ *
+ * By default it checks what createRagPipeline() calls. With
+ * `{ registry: true }` it also applies the rules pluginRegistry.register()
+ * enforces: `metadata` with a string `name` and `version` and a `type`
+ * equal to the stage, plus the methods in the stage's contract.
+ *
+ * @example
+ * const { validatePlugin } = require('@devilsdev/rag-pipeline-utils');
+ * const result = validatePlugin('llm', myLlm, { registry: true });
+ * expect(result.errors).toEqual([]);
+ *
+ * @param {'loader'|'embedder'|'retriever'|'reranker'|'llm'} stage
+ * @param {object} plugin - Plugin instance or object
+ * @param {Object} [options]
+ * @param {boolean} [options.registry=false] - Also check registry rules
+ * @returns {PluginValidationResult}
+ */
+export function validatePlugin(stage: "loader" | "embedder" | "retriever" | "reranker" | "llm", plugin: object, options?: {
+    registry?: boolean;
+}): PluginValidationResult;

package/dist/types/evaluate/semantic-similarity.d.ts

@@ -0,0 +1,14 @@
+/**
+ * @param {string} answer - The generated answer
+ * @param {string} reference - The reference (ground-truth) answer
+ * @param {Object} options
+ * @param {Object} options.embedder - Any embedder with embed() or embedQuery()
+ *   (a built-in connector, LocalEmbedder fitted to your corpus, an AI SDK
+ *   adapter, HashEmbedder from /testing)
+ * @param {AbortSignal} [options.signal]
+ * @returns {Promise<number>} Cosine similarity clamped to 0..1
+ */
+export function computeSemanticSimilarity(answer: string, reference: string, { embedder, signal }?: {
+    embedder: any;
+    signal?: AbortSignal;
+}): Promise<number>;

package/dist/types/grounding/check-answer.d.ts

@@ -0,0 +1,101 @@
+export type AnswerCheck = {
+    /**
+     * - Every sentence is supported by a source
+     */
+    grounded: boolean;
+    /**
+     * - Share of supported sentences, 0..1
+     */
+    groundedness: number;
+    /**
+     * - Sentences no source supports (this
+     * includes a sentence whose numbers do not match its source)
+     */
+    unsupported: string[];
+    /**
+     * - Sources
+     * supporting each sentence
+     */
+    citations: Array<{
+        sentence: string;
+        sources: any[];
+    }>;
+    /**
+     *   - Faithfulness, relevance, context precision (and context recall when a
+     *   referenceAnswer is given); only when `evaluate` is not false
+     */
+    evaluation: {
+        scores: any;
+        details: any;
+        metadata: any;
+    } | undefined;
+    /**
+     *   - The sources as they were checked
+     */
+    sources: Array<{
+        id: string;
+        content: string;
+        metadata: any;
+    }>;
+};
+/**
+ * @typedef {Object} AnswerCheck
+ * @property {boolean} grounded - Every sentence is supported by a source
+ * @property {number} groundedness - Share of supported sentences, 0..1
+ * @property {string[]} unsupported - Sentences no source supports (this
+ *   includes a sentence whose numbers do not match its source)
+ * @property {Array<{sentence: string, sources: Array}>} citations - Sources
+ *   supporting each sentence
+ * @property {{scores: Object, details: Object, metadata: Object}|undefined} evaluation
+ *   - Faithfulness, relevance, context precision (and context recall when a
+ *   referenceAnswer is given); only when `evaluate` is not false
+ * @property {Array<{id: string, content: string, metadata: Object}>} sources
+ *   - The sources as they were checked
+ */
+/**
+ * Check an answer against the sources it was generated from.
+ *
+ * @example
+ * // A LangChain chain's output
+ * const check = await checkAnswer({
+ *   question: input,
+ *   answer: result.answer,
+ *   sources: result.context, // LangChain Documents
+ * });
+ * if (!check.grounded) console.warn(check.unsupported);
+ *
+ * @param {Object} params
+ * @param {string} params.answer - The generated answer
+ * @param {Array<string|Object>} params.sources - Documents the answer was
+ *   generated from (see normalizeSources for accepted shapes)
+ * @param {string} [params.question] - The question; needed for relevance
+ * @param {string} [params.referenceAnswer] - Ground truth, for context recall
+ *   (and semantic similarity when an embedder is given)
+ * @param {Object} [params.embedder] - With a referenceAnswer, adds
+ *   `evaluation.scores.semanticSimilarity` (computeSemanticSimilarity)
+ * @param {boolean} [params.evaluate=true] - Also compute evaluation scores
+ * @returns {Promise<AnswerCheck>}
+ */
+export function checkAnswer({ answer, sources, question, referenceAnswer, embedder, evaluate, }?: {
+    answer: string;
+    sources: Array<string | any>;
+    question?: string;
+    referenceAnswer?: string;
+    embedder?: any;
+    evaluate?: boolean;
+}): Promise<AnswerCheck>;
+/**
+ * Normalize sources to `{ id, content, metadata }`.
+ *
+ * Accepts strings, `{ id, content }` (this library), LangChain Documents
+ * (`{ pageContent, metadata }`), LlamaIndex nodes (`{ text }`) and
+ * `NodeWithScore` (`{ node, score }`).
+ *
+ * @param {Array<string|Object>} sources
+ * @returns {Array<{id: string, content: string, metadata: Object}>}
+ */
+export function normalizeSources(sources: Array<string | any>): Array<{
+    id: string;
+    content: string;
+    metadata: any;
+}>;

package/dist/types/retrieval/file-vector-store.d.ts

@@ -17,6 +17,18 @@ export class FileVectorStore {
     });
     path: string;
     _index: any;
+    _loadedDigest: string;
+    _cacheId: string;
+    _revision: number;
+    /**
+     * Cache identity for createRagPipeline({ cache }): a digest of the index
+     * as loaded, so processes reading the same unchanged index share cached
+     * results; after this instance changes the index the key also carries the
+     * instance and a revision.
+     * @param {'retriever'} role
+     * @returns {string|null}
+     */
+    getCacheKey(role: "retriever"): string | null;
     _emptyIndex(): {
         version: number;
         embedder: any;

package/dist/types/testing.d.ts

@@ -0,0 +1,168 @@
+/**
+ * Deterministic embedder: hashed bag of words, L2-normalised. Texts that
+ * share words get similar vectors, so retrieval ranks sensibly in tests.
+ */
+export class HashEmbedder {
+    /**
+     * @param {Object} [options]
+     * @param {number} [options.dimensions=256] - Vector length
+     */
+    constructor({ dimensions }?: {
+        dimensions?: number;
+    });
+    dimensions: number;
+    metadata: {
+        name: string;
+        version: string;
+        type: string;
+    };
+    /**
+     * Cache identity for createRagPipeline({ cache }).
+     * @param {string} role
+     * @returns {string|null}
+     */
+    getCacheKey(role: string): string | null;
+    /** @private */
+    private _vector;
+    /**
+     * @param {string} text
+     * @param {{ signal?: AbortSignal }} [options]
+     * @returns {Promise<number[]>}
+     */
+    embed(text: string, { signal }?: {
+        signal?: AbortSignal;
+    }): Promise<number[]>;
+    /**
+     * Same as embed().
+     * @param {string} text
+     * @param {{ signal?: AbortSignal }} [options]
+     * @returns {Promise<number[]>}
+     */
+    embedQuery(text: string, options?: {
+        signal?: AbortSignal;
+    }): Promise<number[]>;
+    /**
+     * @param {string[]} texts
+     * @param {{ signal?: AbortSignal }} [options]
+     * @returns {Promise<number[][]>}
+     */
+    embedBatch(texts: string[], { signal }?: {
+        signal?: AbortSignal;
+    }): Promise<number[][]>;
+}
+/**
+ * LLM stand-in. Answers with `answer` (a string, or a function of the query
+ * and documents), or else with the top document's content. Every call is
+ * recorded in `calls`, so tests can assert on what the pipeline sent.
+ */
+export class EchoLLM {
+    /**
+     * @param {Object} [options]
+     * @param {string|function(string, Array<Object>): string} [options.answer]
+     * @param {string} [options.fallback='No information found.'] - Answer when
+     *   there are no documents and no `answer`
+     */
+    constructor({ answer, fallback }?: {
+        answer?: string | ((arg0: string, arg1: Array<any>) => string);
+        fallback?: string;
+    });
+    answer: string | ((arg0: string, arg1: Array<any>) => string);
+    fallback: string;
+    /** @type {Array<{query: string, docs: Array<Object>, options: Object}>} */
+    calls: Array<{
+        query: string;
+        docs: Array<any>;
+        options: any;
+    }>;
+    metadata: {
+        name: string;
+        version: string;
+        type: string;
+    };
+    /**
+     * Cache identity for createRagPipeline({ cache }); none when `answer` is a
+     * function, since its output cannot be identified.
+     * @param {string} role
+     * @returns {string|null}
+     */
+    getCacheKey(role: string): string | null;
+    /** @private */
+    private _answer;
+    /**
+     * @param {string} query
+     * @param {Array<Object>} [docs]
+     * @param {{ signal?: AbortSignal }} [options]
+     * @returns {Promise<string>}
+     */
+    generate(query: string, docs?: Array<any>, options?: {
+        signal?: AbortSignal;
+    }): Promise<string>;
+    /**
+     * Streams the same answer word by word, then a done chunk.
+     * @param {string} query
+     * @param {Array<Object>} [docs]
+     * @param {{ signal?: AbortSignal }} [options]
+     * @yields {{ token: string, done: boolean }}
+     */
+    generateStream(query: string, docs?: Array<any>, options?: {
+        signal?: AbortSignal;
+    }): AsyncGenerator<{
+        token: string;
+        done: boolean;
+    }, void, unknown>;
+}
+/** Short documents for tests and examples. */
+export const fixtures: Readonly<{
+    documents: readonly (Readonly<{
+        id: "vacation";
+        content: "Full-time employees receive twenty days of paid vacation per year. Unused days carry over to March.";
+        metadata: Readonly<{
+            source: "vacation.md";
+        }>;
+    }> | Readonly<{
+        id: "remote-work";
+        content: "Employees may work remotely up to three days per week with manager approval.";
+        metadata: Readonly<{
+            source: "remote-work.md";
+        }>;
+    }> | Readonly<{
+        id: "expenses";
+        content: "Submit receipts within thirty days. Travel above 500 EUR needs pre-approval from finance.";
+        metadata: Readonly<{
+            source: "expenses.md";
+        }>;
+    }>)[];
+}>;
+/**
+ * A pipeline with HashEmbedder, a MemoryRetriever already holding
+ * `documents`, and EchoLLM (or the llm you pass).
+ *
+ * @example
+ * const { createTestPipeline } = require('@devilsdev/rag-pipeline-utils/testing');
+ * const { pipeline, llm } = await createTestPipeline();
+ * const result = await pipeline.run({ query: 'How many vacation days?' });
+ * // result.results[0].id === 'vacation'; llm.calls.length === 1
+ *
+ * @param {Object} [options]
+ * @param {Array<{id: string, content: string, metadata?: Object}>} [options.documents=fixtures.documents]
+ * @param {Object} [options.embedder] - Defaults to a new HashEmbedder
+ * @param {Object|null} [options.llm] - Defaults to a new EchoLLM; null for retrieval only
+ * @param {Object} [options.pipeline] - Extra createRagPipeline options (e.g. reranker)
+ * @returns {Promise<{ pipeline: Object, embedder: Object, retriever: MemoryRetriever, llm: Object|null }>}
+ */
+export function createTestPipeline({ documents, embedder, llm, pipeline: pipelineOptions, }?: {
+    documents?: Array<{
+        id: string;
+        content: string;
+        metadata?: any;
+    }>;
+    embedder?: any;
+    llm?: any | null;
+    pipeline?: any;
+}): Promise<{
+    pipeline: any;
+    embedder: any;
+    retriever: MemoryRetriever;
+    llm: any | null;
+}>;
+import { MemoryRetriever } from "./connectors/memory-retriever";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devilsdev/rag-pipeline-utils",
-  "version": "2.5.0",
+  "version": "2.6.0",
   "description": "Composable retrieval-augmented generation (RAG) for Node.js: grounded citations, evaluation, guardrails, hybrid retrieval and a CLI.",
   "main": "dist/index.cjs",
   "type": "commonjs",
@@ -9,6 +9,9 @@
     "*": {
       "experimental": [
         "dist/experimental.d.ts"
+      ],
+      "testing": [
+        "dist/testing.d.ts"
       ]
     }
   },
@@ -23,6 +26,11 @@
       "require": "./dist/experimental.cjs",
       "import": "./dist/experimental.mjs"
     },
+    "./testing": {
+      "types": "./dist/testing.d.ts",
+      "require": "./dist/testing.cjs",
+      "import": "./dist/testing.mjs"
+    },
     "./cli": "./bin/cli.js"
   },
   "bin": {
@@ -59,16 +67,16 @@
     "lint-staged": "lint-staged",
     "postinstall": "husky install",
     "verify:mocks": "node scripts/verify-fixture-mocks.js",
-    "build:docs": "npx docusaurus build docs-site",
+    "build:docs": "npm run docs:build",
     "ci": "node scripts/ci-runner.js",
-    "docs:dev": "npx docusaurus start docs-site",
+    "docs:dev": "cd docs-site && npm run start",
     "docs:version": "npx docusaurus docs:version",
     "watch:lqip": "node docs-site/scripts/watch-images.js",
     "docs:clear": "docusaurus clear --site-dir docs-site",
-    "docs:start": "docusaurus start docs-site",
+    "docs:start": "cd docs-site && npm run start",
     "docs:build": "cd docs-site && npm run build",
     "docs:serve": "docusaurus serve docs-site",
-    "docs:deploy": "docusaurus deploy docs-site",
+    "docs:deploy": "node docs-site/scripts/sync-changelog.js && docusaurus deploy docs-site",
     "roadmap:close": "node scripts/close-done-roadmap-issues.js",
     "labels:sync": "node scripts/ensure-roadmap-labels.js",
     "release:tag": "node scripts/generate-release-note.js v$(node -p \"require('./package.json').version\")",
@@ -176,8 +184,12 @@
   "devDependencies": {
     "@babel/core": "^7.26.10",
     "@babel/preset-env": "^7.26.9",
-    "@docusaurus/core": "3.10.0",
-    "@docusaurus/preset-classic": "3.10.0",
+    "@docusaurus/core": "3.10.2",
+    "@docusaurus/preset-classic": "3.10.2",
+    "@docusaurus/theme-mermaid": "3.10.2",
+    "@easyops-cn/docusaurus-search-local": "0.55.3",
+    "@fontsource/inter": "5.3.0",
+    "@fontsource/jetbrains-mono": "5.3.0",
     "@octokit/rest": "^21.1.1",
     "@opentelemetry/api": "^1.9.0",
     "@opentelemetry/auto-instrumentations-node": "^0.52.1",
@@ -192,8 +204,11 @@
     "@types/jest": "^29.5.14",
     "@types/node": "22.14.1",
     "acorn": "8.15.0",
+    "ai": "6.0.317",
+    "ai-v7": "npm:ai@7.0.156",
     "babel-jest": "^29.7.0",
     "cross-env": "7.0.3",
+    "docusaurus-plugin-llms": "0.6.1",
     "esbuild": "^0.25.2",
     "eslint": "^8.57.1",
     "eslint-plugin-storybook": "^0.12.0",

package/src/adapters/ai-sdk.js

@@ -0,0 +1,215 @@
+'use strict';
+
+/**
+ * Adapter for the Vercel AI SDK (`ai` package): use an AI SDK embedding
+ * model and language model as this library's `embedder` and `llm`.
+ *
+ * The AI SDK is not a dependency. You pass in the functions you already
+ * import from it, so any AI SDK version from 5 on works, from ESM or
+ * CommonJS, and this package keeps supporting Node 18 (AI SDK 7 needs Node
+ * 22 and is ESM-only).
+ */
+
+const { buildRagPrompt } = require('../connectors/rag-prompt');
+const { componentCacheKey } = require('../cache/component-keys');
+
+/** Identity of an AI SDK model object (provider and model id). */
+const modelId = (model) =>
+  `${model.provider || 'unknown'}:${model.modelId || model.modelName || 'unknown'}`;
+
+/**
+ * @typedef {Object} AiSdkFunctions
+ * @property {Function} [embed] - `embed` from 'ai' (needed for an embedder)
+ * @property {Function} [embedMany] - `embedMany` from 'ai' (batch embedding)
+ * @property {Function} [generateText] - `generateText` from 'ai' (needed for an llm)
+ * @property {Function} [streamText] - `streamText` from 'ai' (streaming)
+ */
+
+function requireFunction(ai, name, purpose) {
+  if (typeof ai[name] !== 'function') {
+    throw new Error(
+      `createAiSdkAdapter: ${purpose} needs ai.${name}. Pass the AI SDK functions, e.g. import { ${name} } from "ai".`,
+    );
+  }
+}
+
+/**
+ * Options for every generateText / streamText call. The RAG system prompt
+ * (retrieved text is untrusted reference data, not instructions) is passed
+ * as both `instructions` (AI SDK 7) and `system` (AI SDK 5 and 6; 7 reads
+ * `instructions ?? system`), so it reaches the model on every version.
+ */
+function textCallOptions(model, query, docs, settings, signal, systemPrompt) {
+  const { system, user } = buildRagPrompt(query, docs, { systemPrompt });
+  return {
+    ...settings,
+    model,
+    ...(system && { system, instructions: system }),
+    prompt: user,
+    ...(signal && { abortSignal: signal }),
+  };
+}
+
+/** Per-call settings the pipeline may pass through to the model. */
+const CALL_SETTINGS = [
+  'temperature',
+  'maxOutputTokens',
+  'topP',
+  'topK',
+  'presencePenalty',
+  'frequencyPenalty',
+  'seed',
+  'stopSequences',
+  'maxRetries',
+];
+
+function pickSettings(options) {
+  const out = {};
+  for (const key of CALL_SETTINGS) {
+    if (options[key] !== undefined) out[key] = options[key];
+  }
+  return out;
+}
+
+/**
+ * Wrap AI SDK models as an embedder and an llm for createRagPipeline().
+ *
+ * @example
+ * import { embed, embedMany, generateText, streamText } from "ai";
+ * import { openai } from "@ai-sdk/openai";
+ * import { createAiSdkAdapter, createRagPipeline } from "@devilsdev/rag-pipeline-utils";
+ *
+ * const { embedder, llm } = createAiSdkAdapter({
+ *   ai: { embed, embedMany, generateText, streamText },
+ *   embeddingModel: openai.embedding("text-embedding-3-small"),
+ *   languageModel: openai("gpt-4o-mini"),
+ * });
+ * const pipeline = createRagPipeline({ embedder, retriever, llm });
+ *
+ * @param {Object} options
+ * @param {AiSdkFunctions} options.ai - Functions imported from 'ai'
+ * @param {Object} [options.embeddingModel] - AI SDK embedding model
+ * @param {Object} [options.languageModel] - AI SDK language model
+ * @param {Object} [options.settings] - Defaults for every text call, e.g.
+ *   `{ temperature: 0, maxOutputTokens: 512 }`
+ * @param {string} [options.systemPrompt] - Replaces the built-in RAG system
+ *   prompt
+ * @returns {{ embedder: Object|null, llm: Object|null }}
+ */
+function createAiSdkAdapter(options = {}) {
+  const {
+    ai,
+    embeddingModel,
+    languageModel,
+    settings = {},
+    systemPrompt,
+  } = options;
+  if (!ai || typeof ai !== 'object') {
+    throw new Error(
+      'createAiSdkAdapter: pass `ai`, the AI SDK functions, e.g. { embed, embedMany, generateText, streamText } from "ai".',
+    );
+  }
+  if (!embeddingModel && !languageModel) {
+    throw new Error(
+      'createAiSdkAdapter: pass an embeddingModel, a languageModel, or both.',
+    );
+  }
+
+  let embedder = null;
+  if (embeddingModel) {
+    requireFunction(ai, 'embed', 'an embeddingModel');
+    const embedOne = async (text, { signal } = {}) => {
+      const { embedding } = await ai.embed({
+        model: embeddingModel,
+        value: String(text),
+        ...(signal && { abortSignal: signal }),
+      });
+      return embedding;
+    };
+    embedder = {
+      metadata: { name: 'ai-sdk-embedder', version: '1.0.0', type: 'embedder' },
+      getCacheKey: (role) =>
+        role === 'embedder'
+          ? componentCacheKey('ai-sdk-embed', {
+              model: modelId(embeddingModel),
+            })
+          : null,
+      embed: embedOne,
+      embedQuery: embedOne,
+      /** One vector per text; uses embedMany when it was passed in. */
+      async embedBatch(texts, { signal } = {}) {
+        if (typeof ai.embedMany !== 'function') {
+          const vectors = [];
+          for (const text of texts)
+            vectors.push(await embedOne(text, { signal }));
+          return vectors;
+        }
+        const { embeddings } = await ai.embedMany({
+          model: embeddingModel,
+          values: texts.map(String),
+          ...(signal && { abortSignal: signal }),
+        });
+        return embeddings;
+      },
+    };
+  }
+
+  let llm = null;
+  if (languageModel) {
+    requireFunction(ai, 'generateText', 'a languageModel');
+    llm = {
+      metadata: { name: 'ai-sdk-llm', version: '1.0.0', type: 'llm' },
+      // The AI SDK leaves an unset temperature to the provider, whose default
+      // is usually 1, so unset counts as non-deterministic: generation
+      // caching needs `settings.temperature: 0`.
+      config: { temperature: settings.temperature ?? 1 },
+      getCacheKey: (role) =>
+        role === 'llm'
+          ? componentCacheKey('ai-sdk', {
+              model: modelId(languageModel),
+              settings: JSON.stringify(settings),
+              systemPrompt,
+            })
+          : null,
+      async generate(query, docs = [], callOptions = {}) {
+        const { text } = await ai.generateText(
+          textCallOptions(
+            languageModel,
+            query,
+            docs,
+            { ...settings, ...pickSettings(callOptions) },
+            callOptions.signal,
+            systemPrompt,
+          ),
+        );
+        return text;
+      },
+    };
+    if (typeof ai.streamText === 'function') {
+      llm.generateStream = async function* generateStream(
+        query,
+        docs = [],
+        callOptions = {},
+      ) {
+        const result = ai.streamText(
+          textCallOptions(
+            languageModel,
+            query,
+            docs,
+            { ...settings, ...pickSettings(callOptions) },
+            callOptions.signal,
+            systemPrompt,
+          ),
+        );
+        for await (const token of result.textStream) {
+          yield { token, done: false };
+        }
+        yield { token: '', done: true };
+      };
+    }
+  }
+
+  return { embedder, llm };
+}
+
+module.exports = { createAiSdkAdapter };

package/src/cache/component-keys.js

@@ -0,0 +1,38 @@
+'use strict';
+
+/**
+ * Builds the cache identity a built-in component reports through
+ * getCacheKey(role): every setting that changes its output, so two
+ * configurations never share cached results.
+ */
+
+const crypto = require('crypto');
+
+const digest = (value) =>
+  crypto
+    .createHash('sha256')
+    .update(String(value ?? ''))
+    .digest('hex')
+    .slice(0, 16);
+
+/**
+ * @param {string} kind - e.g. 'openai'
+ * @param {Object} fields - Settings that change the output
+ * @returns {string}
+ */
+function componentCacheKey(kind, fields) {
+  const parts = Object.keys(fields)
+    .sort()
+    .map((name) => {
+      const value = fields[name];
+      // Long free text (system prompts) is hashed to keep keys short.
+      const text =
+        typeof value === 'string' && value.length > 64
+          ? `#${digest(value)}`
+          : value;
+      return `${name}=${text === undefined ? '' : text}`;
+    });
+  return `${kind}|${parts.join('|')}`;
+}
+
+module.exports = { componentCacheKey, digest };