@devilsdev/rag-pipeline-utils 1.0.0

package/jest.config.js

@@ -0,0 +1,14 @@
+/**
+ * Version: 0.1.0
+ * Path: /jest.config.js
+ * Description: Jest configuration for native ESM test support
+ * Author: Ali Kahwaji
+ */
+
+export default {
+    testEnvironment: 'node',
+    transform: {}, // Native ESM — no Babel used
+    roots: ['<rootDir>/__tests__'],
+    verbose: true
+  };
+  

package/package.json

@@ -0,0 +1,65 @@
+{
+  "name": "@devilsdev/rag-pipeline-utils",
+  "version": "1.0.0",
+  "description": "A modular toolkit for building RAG (Retrieval-Augmented Generation) pipelines in Node.js",
+  "main": "src/core/create-pipeline.js",
+  "bin": {
+    "rag-pipeline": "./bin/cli.js"
+  },
+  "type": "module",
+  "scripts": {
+    "test": "cross-env NODE_OPTIONS=--experimental-vm-modules jest --coverage",
+    "test:watch": "jest --watch",
+    "lint": "eslint .",
+    "prepare": "husky install"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/DevilsDev/rag-pipeline-utils.git"
+  },
+  "keywords": [
+    "rag",
+    "retrieval-augmented-generation",
+    "vectorstore",
+    "llm",
+    "pipeline",
+    "cli",
+    "openai"
+  ],
+  "author": "Ali Kahwaji",
+  "license": "Apache-2.0",
+  "bugs": {
+    "url": "https://github.com/DevilsDev/rag-pipeline-utils/issues"
+  },
+  "homepage": "https://github.com/DevilsDev/rag-pipeline-utils#readme",
+  "publishConfig": {
+    "access": "public"
+  },
+  "devDependencies": {
+    "@semantic-release/changelog": "^6.0.3",
+    "@semantic-release/git": "^10.0.1",
+    "@semantic-release/github": "^9.2.6",
+    "@semantic-release/npm": "^11.0.3",
+    "@types/jest": "^29.5.14",
+    "cross-env": "^7.0.3",
+    "esbuild": "^0.25.2",
+    "eslint": "^8.57.1",
+    "husky": "^8.0.3",
+    "jest": "^29.7.0",
+    "prettier": "^3.5.3",
+    "rimraf": "^6.0.1",
+    "semantic-release": "^21.1.2",
+    "ts-jest": "^29.3.2"
+  },
+  "dependencies": {
+    "axios": "^1.8.4",
+    "chalk": "^5.4.1",
+    "commander": "^13.1.0",
+    "csv-parse": "^5.6.0",
+    "dotenv": "^16.5.0",
+    "fast-glob": "^3.3.3",
+    "jsdom": "^26.1.0",
+    "openai": "^4.93.0",
+    "pino": "^9.6.0"
+  }
+}

package/public/src/App.jsx

@@ -0,0 +1,57 @@
+/**
+ * Version: 0.1.0
+ * Path: /public/src/App.jsx
+ * Description: Evaluation results dashboard UI
+ * Author: Ali Kahwaji
+ */
+
+import { useEffect, useState } from 'react';
+
+export default function App() {
+  const [results, setResults] = useState([]);
+  const [summary, setSummary] = useState({ passRate: 0, avgBLEU: 0, avgROUGE: 0 });
+
+  useEffect(() => {
+    fetch('/results/sample-eval.json')
+      .then(res => res.json())
+      .then(data => {
+        setResults(data);
+
+        const total = data.length;
+        const passCount = data.filter(r => r.success).length;
+        const avgBLEU = data.reduce((s, r) => s + r.scores.bleu, 0) / total;
+        const avgROUGE = data.reduce((s, r) => s + r.scores.rouge, 0) / total;
+
+        setSummary({
+          passRate: (passCount / total * 100).toFixed(1),
+          avgBLEU: avgBLEU.toFixed(2),
+          avgROUGE: avgROUGE.toFixed(2)
+        });
+      });
+  }, []);
+
+  return (
+    <main className="max-w-6xl mx-auto p-6">
+      <h1 className="text-3xl font-bold mb-4">RAG Evaluation Dashboard</h1>
+
+      <div className="mb-6 p-4 border rounded shadow bg-white">
+        <h2 className="text-xl font-semibold mb-2">Summary</h2>
+        <p> Pass Rate: <strong>{summary.passRate}%</strong></p>
+        <p> Avg BLEU: <strong>{summary.avgBLEU}</strong></p>
+        <p> Avg ROUGE: <strong>{summary.avgROUGE}</strong></p>
+      </div>
+
+      {results.map((res, i) => (
+        <div key={i} className="mb-4 p-4 border-l-4 rounded bg-white shadow border-l-gray-300">
+          <h3 className="font-bold text-lg">Prompt {i + 1}</h3>
+          <p className="mt-2"><strong>Prompt:</strong> {res.prompt}</p>
+          <p><strong>Expected:</strong> {res.expected}</p>
+          <p><strong>Actual:</strong> {res.actual}</p>
+          <p> BLEU: <strong>{res.scores.bleu.toFixed(2)}</strong></p>
+          <p> ROUGE: <strong>{res.scores.rouge.toFixed(2)}</strong></p>
+          <p>Pass: <strong className={res.success ? 'text-green-600' : 'text-red-600'}>{res.success ? 'Yes' : 'No'}</strong></p>
+        </div>
+      ))}
+    </main>
+  );
+}

package/release.config.js

@@ -0,0 +1,23 @@
+/**
+ * Version: 0.1.0
+ * Path: /release.config.js
+ * Description: Semantic release configuration for automated changelog, versioning, GitHub and npm publishing
+ * Author: Ali Kahwaji
+ */
+
+export default {
+  branches: ['main'],
+  plugins: [
+    '@semantic-release/commit-analyzer',
+    '@semantic-release/release-notes-generator',
+    ['@semantic-release/changelog', {
+      changelogFile: 'CHANGELOG.md'
+    }],
+    '@semantic-release/npm',
+    ['@semantic-release/git', {
+      assets: ['CHANGELOG.md', 'package.json'],
+      message: 'chore(release): ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}'
+    }],
+    '@semantic-release/github'
+  ]
+};

package/server.js

@@ -0,0 +1,31 @@
+/**
+ * Version: 0.1.0
+ * Path: /server.js
+ * Description: Local static server for serving evaluation dashboard
+ * Author: Ali Kahwaji
+ */
+
+import express from 'express';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const app = express();
+const port = process.env.PORT || 3000;
+
+// Serve evaluation results statically
+app.use('/results', express.static(path.join(__dirname, '__tests__/fixtures')));
+
+// Serve public frontend assets
+app.use(express.static(path.join(__dirname, 'public')));
+
+// Fallback to index.html
+app.get('*', (req, res) => {
+  res.sendFile(path.join(__dirname, 'public/index.html'));
+});
+
+app.listen(port, () => {
+  console.log(` Dashboard running at http://localhost:${port}`);
+});

package/src/config/load-config.js

@@ -0,0 +1,35 @@
+/**
+ * Version: 0.1.0
+ * Path: /src/config/load-config.js
+ * Description: Loader and validator for `.ragrc.json` configuration files
+ * Author: Ali Kahwaji
+ */
+
+import fs from 'fs';
+import path from 'path';
+import { validate } from './validate-schema.js';
+
+const DEFAULT_CONFIG_FILE = '.ragrc.json';
+
+/**
+ * Load and validate the RAG configuration from a file.
+ *
+ * @param {string} [configPath] - Optional custom config path
+ * @returns {object} - Validated config object
+ * @throws {Error} - If file not found or validation fails
+ */
+export function loadRagConfig(configPath = DEFAULT_CONFIG_FILE) {
+  const fullPath = path.resolve(process.cwd(), configPath);
+
+  if (!fs.existsSync(fullPath)) {
+    throw new Error(`Config file not found: ${fullPath}`);
+  }
+
+  const raw = fs.readFileSync(fullPath, 'utf-8');
+  const config = JSON.parse(raw);
+
+  validate(config); // Throws if invalid
+
+  return config;
+}
+

package/src/config/validate-schema.js

@@ -0,0 +1,49 @@
+/**
+ * Version: 0.1.0
+ * Path: /src/config/validate-schema.js
+ * Description: JSON schema validation for `.ragrc.json` config using AJV
+ * Author: Ali Kahwaji
+ */
+
+import Ajv from 'ajv';
+
+// Define allowed config structure
+const schema = {
+  type: 'object',
+  properties: {
+    loader: { type: 'string' },
+    embedder: { type: 'string' },
+    retriever: { type: 'string' },
+    llm: { type: 'string' },
+    chunk: {
+      type: 'object',
+      properties: {
+        strategy: { type: 'string' },
+        size: { type: 'number' }
+      },
+      required: ['strategy', 'size'],
+      additionalProperties: false
+    }
+  },
+  required: ['loader', 'embedder', 'retriever', 'llm'],
+  additionalProperties: false
+};
+
+const ajv = new Ajv();
+const validateSchema = ajv.compile(schema);
+
+/**
+ * Validates a given config object against the `.ragrc.json` schema.
+ * Throws if the schema is invalid.
+ *
+ * @param {object} config - Configuration object
+ * @throws {Error} - Validation errors, if any
+ */
+export function validate(config) {
+  const valid = validateSchema(config);
+  if (!valid) {
+    const messages = validateSchema.errors.map(e => `${e.instancePath} ${e.message}`).join(', ');
+    throw new Error(`Invalid config: ${messages}`);
+  }
+}
+

package/src/core/create-pipeline.js

@@ -0,0 +1,82 @@
+/**
+ * Version: 0.1.3
+ * Path: /src/core/create-pipeline.js
+ * Description: RAG pipeline factory with optional LLM reranker integration
+ * Author: Ali Kahwaji
+ */
+
+import { PluginRegistry } from './plugin-registry.js';
+import { logger } from '../utils/logger.js';
+import { withRetry } from '../utils/retry.js';
+import { LLMReranker } from '../reranker/llm-reranker.js';
+
+const registry = new PluginRegistry();
+
+/**
+ * Create a RAG pipeline instance using registered plugin components
+ * @param {object} config
+ * @param {string} config.loader
+ * @param {string} config.embedder
+ * @param {string} config.retriever
+ * @param {string} config.llm
+ * @param {boolean} [config.useReranker=false]
+ * @returns {object} RAG pipeline instance
+ */
+export function createRagPipeline({ loader, embedder, retriever, llm, useReranker = false }) {
+  const rerankerInstance = useReranker ? new LLMReranker({ llm: registry.get('llm', llm) }) : null;
+
+  return {
+    async ingest(path) {
+      logger.info({ loader, embedder, retriever }, 'Pipeline ingest start');
+      const loaderInstance = registry.get('loader', loader);
+      const embedderInstance = registry.get('embedder', embedder);
+      const retrieverInstance = registry.get('retriever', retriever);
+
+      const documents = await loaderInstance.load(path);
+      const chunks = documents.flatMap(doc => doc.chunk());
+      const vectors = await embedderInstance.embed(chunks);
+
+      await withRetry(() => retrieverInstance.store(vectors), {
+        label: 'vector-store',
+        retries: 3,
+        initialDelay: 500
+      });
+
+      logger.info('Ingestion pipeline completed');
+    },
+
+    async query(prompt) {
+      logger.info({ prompt, embedder, retriever, llm, useReranker }, 'Pipeline query start');
+      const embedderInstance = registry.get('embedder', embedder);
+      const retrieverInstance = registry.get('retriever', retriever);
+      const llmInstance = registry.get('llm', llm);
+
+      const queryVector = await embedderInstance.embedQuery(prompt);
+      let retrieved = await withRetry(() => retrieverInstance.retrieve(queryVector), {
+        label: 'vector-retrieve',
+        retries: 3,
+        initialDelay: 500
+      });
+
+      if (rerankerInstance) {
+        logger.info('Applying LLM reranker to retrieved chunks');
+        retrieved = await withRetry(() => rerankerInstance.rerank(prompt, retrieved), {
+          label: 'rerank',
+          retries: 2,
+          initialDelay: 400
+        });
+      }
+
+      const result = await withRetry(() => llmInstance.generate(prompt, retrieved), {
+        label: 'llm-generate',
+        retries: 3,
+        initialDelay: 500
+      });
+
+      return result;
+    }
+  };
+}
+
+export { registry };
+

package/src/core/plugin-registry.js

@@ -0,0 +1,61 @@
+/**
+ * Version: 1.0.0
+ * Path: /src/core/plugin-registry.js
+ * Description: Registry for managing pluggable components (loader, embedder, retriever, llm)
+ * Author: Ali Kahwaji
+ */
+
+export class PluginRegistry {
+  #registry = {
+    loader: new Map(),
+    embedder: new Map(),
+    retriever: new Map(),
+    llm: new Map(),
+    reranker: new Map(),
+  };
+
+  /**
+   * Registers a plugin instance under a specific type and name.
+   * @param {'loader' | 'embedder' | 'retriever' | 'llm' | 'reranker'} type - Plugin type category.
+   * @param {string} name - Unique name/key for this plugin.
+   * @param {object} plugin - Plugin implementation (e.g., instance or factory).
+   */
+  register(type, name, plugin) {
+    const group = this.#registry[type];
+    if (!group) {
+      throw new Error(`Unknown plugin type: '${type}'`);
+    }
+    group.set(name, plugin);
+  }
+
+  /**
+   * Retrieves a plugin by type and name.
+   * @param {'loader' | 'embedder' | 'retriever' | 'llm' | 'reranker'} type - Plugin type.
+   * @param {string} name - Plugin name/key.
+   * @returns {object} - The plugin instance.
+   */
+  get(type, name) {
+    const group = this.#registry[type];
+    if (!group) {
+      throw new Error(`Unknown plugin type: '${type}'`);
+    }
+    const plugin = group.get(name);
+    if (!plugin) {
+      throw new Error(`Plugin not found: [${type}:${name}]`);
+    }
+    return plugin;
+  }
+
+  /**
+   * Lists all registered plugin names for a type.
+   * @param {'loader' | 'embedder' | 'retriever' | 'llm' | 'reranker'} type - Plugin category.
+   * @returns {string[]} - Array of registered plugin names.
+   */
+  list(type) {
+    const group = this.#registry[type];
+    if (!group) {
+      throw new Error(`Unknown plugin type: '${type}'`);
+    }
+    return [...group.keys()];
+  }
+}

package/src/dag/dag-engine.js

@@ -0,0 +1,70 @@
+/**
+ * Version: 0.1.0
+ * Path: /src/dag/dag-engine.js
+ * Description: DAG executor for chained RAG pipelines
+ * Author: Ali Kahwaji
+ */
+
+export class DAGNode {
+    constructor(id, run) {
+      this.id = id;
+      this.run = run; // async (input) => output
+      this.inputs = [];
+      this.outputs = [];
+    }
+  }
+  
+  export class DAG {
+    constructor() {
+      this.nodes = new Map();
+    }
+  
+    addNode(id, fn) {
+      const node = new DAGNode(id, fn);
+      this.nodes.set(id, node);
+      return node;
+    }
+  
+    connect(fromId, toId) {
+      const from = this.nodes.get(fromId);
+      const to = this.nodes.get(toId);
+      if (!from || !to) throw new Error(`Invalid edge: ${fromId} → ${toId}`);
+      from.outputs.push(to);
+      to.inputs.push(from);
+    }
+  
+    /**
+     * Executes DAG from source nodes to sinks using topological order
+     * @param {any} seed - Initial input to the graph
+     * @returns {Promise<any>} - Final output from sink(s)
+     */
+    async execute(seed) {
+      const visited = new Set();
+      const order = this.topoSort();
+      const results = new Map();
+  
+      for (const node of order) {
+        const inputValues = node.inputs.map(n => results.get(n.id));
+        const input = inputValues.length === 1 ? inputValues[0] : inputValues;
+        const output = await node.run(input ?? seed);
+        results.set(node.id, output);
+      }
+  
+      const last = order.at(-1);
+      return results.get(last.id);
+    }
+  
+    topoSort() {
+      const order = [];
+      const visited = new Set();
+      const visit = (node) => {
+        if (visited.has(node.id)) return;
+        visited.add(node.id);
+        node.inputs.forEach(visit);
+        order.push(node);
+      };
+      for (const node of this.nodes.values()) visit(node);
+      return order;
+    }
+  }
+  

package/src/evaluate/evaluator.js

@@ -0,0 +1,44 @@
+/**
+ * Version: 0.1.1
+ * Path: /src/evaluate/evaluator.js
+ * Description: Evaluation runner with scoring metrics for batch RAG QA
+ * Author: Ali Kahwaji
+ */
+
+import { createRagPipeline } from '../core/create-pipeline.js';
+import { logger } from '../utils/logger.js';
+import fs from 'fs/promises';
+import path from 'path';
+import { scoreAnswer } from './scoring.js';
+
+/**
+ * Evaluate a list of prompt/answer pairs using the RAG pipeline and compute scores
+ * @param {string} datasetPath - Path to JSON file with { prompt, expected }[]
+ * @param {object} config - RAG plugin config (loader optional)
+ * @returns {Promise<Array<{ prompt: string, expected: string, actual: string, success: boolean, scores: object }>>}
+ */
+export async function evaluateRagDataset(datasetPath, config) {
+  const file = await fs.readFile(path.resolve(datasetPath), 'utf-8');
+  const cases = JSON.parse(file);
+  const pipeline = createRagPipeline(config);
+
+  const results = [];
+  for (const { prompt, expected } of cases) {
+    try {
+      logger.info({ prompt }, 'Evaluating case');
+      const actual = await pipeline.query(prompt);
+      const success = normalizeText(actual) === normalizeText(expected);
+      const scores = scoreAnswer(actual, expected);
+      results.push({ prompt, expected, actual, success, scores });
+    } catch (err) {
+      logger.error({ prompt, error: err.message }, 'Evaluation failed');
+      results.push({ prompt, expected, actual: '', success: false, scores: { bleu: 0, rouge: 0 } });
+    }
+  }
+  return results;
+}
+
+function normalizeText(txt) {
+  return txt.trim().toLowerCase().replace(/[^a-z0-9 ]/g, '').replace(/\s+/g, ' ');
+}
+

package/src/evaluate/scoring.js

@@ -0,0 +1,74 @@
+/**
+ * Version: 0.1.0
+ * Path: /src/evaluate/scoring.js
+ * Description: Basic scoring metrics (BLEU, ROUGE-L) for RAG output evaluation
+ * Author: Ali Kahwaji
+ */
+
+/**
+ * Tokenize a string by words
+ * @param {string} text
+ * @returns {string[]}
+ */
+function tokenize(text) {
+    return text.toLowerCase().replace(/[^a-z0-9 ]/g, '').split(/\s+/).filter(Boolean);
+  }
+  
+  /**
+   * Compute BLEU-1 score: unigram precision
+   * @param {string[]} candidate - Generated tokens
+   * @param {string[]} reference - Reference tokens
+   * @returns {number}
+   */
+  export function computeBLEU(candidate, reference) {
+    const candCounts = candidate.reduce((acc, t) => (acc[t] = (acc[t] || 0) + 1, acc), {});
+    const refCounts = reference.reduce((acc, t) => (acc[t] = (acc[t] || 0) + 1, acc), {});
+  
+    let match = 0;
+    for (const token of Object.keys(candCounts)) {
+      match += Math.min(candCounts[token], refCounts[token] || 0);
+    }
+    return candidate.length ? match / candidate.length : 0;
+  }
+  
+  /**
+   * Compute ROUGE-L (Longest Common Subsequence) recall
+   * @param {string[]} candidate
+   * @param {string[]} reference
+   * @returns {number}
+   */
+  export function computeROUGE(candidate, reference) {
+    const m = candidate.length;
+    const n = reference.length;
+    const dp = Array(m + 1).fill(null).map(() => Array(n + 1).fill(0));
+  
+    for (let i = 1; i <= m; i++) {
+      for (let j = 1; j <= n; j++) {
+        if (candidate[i - 1] === reference[j - 1]) {
+          dp[i][j] = dp[i - 1][j - 1] + 1;
+        } else {
+          dp[i][j] = Math.max(dp[i - 1][j], dp[i][j - 1]);
+        }
+      }
+    }
+  
+    const lcs = dp[m][n];
+    return reference.length ? lcs / reference.length : 0;
+  }
+  
+  /**
+   * Score a prompt-answer pair using multiple metrics
+   * @param {string} actual - LLM response
+   * @param {string} expected - Reference answer
+   * @returns {{ bleu: number, rouge: number }}
+   */
+  export function scoreAnswer(actual, expected) {
+    const candTokens = tokenize(actual);
+    const refTokens = tokenize(expected);
+    return {
+      bleu: computeBLEU(candTokens, refTokens),
+      rouge: computeROUGE(candTokens, refTokens)
+    };
+  }
+  
+  

package/src/loader/csv-loader.js

@@ -0,0 +1,30 @@
+/**
+ * Version: 0.1.0
+ * Path: /src/loader/csv-loader.js
+ * Description: CSV document loader with row-wise chunking
+ * Author: Ali Kahwaji
+ */
+
+import fs from 'fs/promises';
+import path from 'path';
+import { parse } from 'csv-parse/sync';
+
+export class CSVLoader {
+  /**
+   * Load and parse a CSV file into chunks
+   * @param {string} filePath
+   * @returns {Promise<Array<{ chunk(): string[] }>>}
+   */
+  async load(filePath) {
+    const absPath = path.resolve(filePath);
+    const raw = await fs.readFile(absPath, 'utf-8');
+    const records = parse(raw, { columns: true });
+
+    return [
+      {
+        chunk: () => records.map(r => Object.values(r).join(' '))
+      }
+    ];
+  }
+}
+

package/src/loader/directory-loader.js

@@ -0,0 +1,38 @@
+/**
+ * Version: 0.1.0
+ * Path: /src/loader/directory-loader.js
+ * Description: Recursive directory loader with file type routing
+ * Author: Ali Kahwaji
+ */
+
+import fg from 'fast-glob';
+import { MarkdownLoader } from './markdown-loader.js';
+import { HTMLLoader } from './html-loader.js';
+import { CSVLoader } from './csv-loader.js';
+
+const loaders = {
+  '.md': new MarkdownLoader(),
+  '.html': new HTMLLoader(),
+  '.csv': new CSVLoader()
+};
+
+export class DirectoryLoader {
+  /**
+   * Load and route supported files from a directory
+   * @param {string} dirPath
+   * @returns {Promise<Array<{ chunk(): string[] }>>}
+   */
+  async load(dirPath) {
+    const matches = await fg([`${dirPath}/**/*.{md,html,csv}`]);
+    const all = [];
+    for (const file of matches) {
+      const ext = file.slice(file.lastIndexOf('.'));
+      const loader = loaders[ext];
+      if (loader) {
+        const docs = await loader.load(file);
+        all.push(...docs);
+      }
+    }
+    return all;
+  }
+}