@devilsdev/rag-pipeline-utils 2.0.0 → 2.1.0

package/.eslintrc.json

@@ -1,21 +1,27 @@
 {
-    "env": {
-      "es2022": true,
-      "node": true,
-      "jest": true
-    },
-    "extends": [
-      "eslint:recommended"
+  "env": {
+    "es2022": true,
+    "node": true,
+    "jest": true
+  },
+  "extends": [
+    "eslint:recommended",
+    "plugin:storybook/recommended"
+  ],
+  "parserOptions": {
+    "ecmaVersion": "latest",
+    "sourceType": "module"
+  },
+  "rules": {
+    "no-unused-vars": "warn",
+    "no-console": "off",
+    "semi": [
+      "error",
+      "always"
     ],
-    "parserOptions": {
-      "ecmaVersion": "latest",
-      "sourceType": "module"
-    },
-    "rules": {
-      "no-unused-vars": "warn",
-      "no-console": "off",
-      "semi": ["error", "always"],
-      "quotes": ["error", "single"]
-    }
+    "quotes": [
+      "error",
+      "single"
+    ]
   }
-  
+}

package/.github/workflows/docs-deploy.yml

@@ -0,0 +1,46 @@
+# .github/workflows/docs-deploy.yml
+# Version: 1.1.0
+# Description: Deploys Docusaurus to GitHub Pages on changes to documentation files
+# Author: Ali Kahwaji
+
+name: 📚 Deploy Docusaurus Docs
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'docs-site/**'
+      - 'docs/**'
+      - 'blog/**'
+      - 'sidebars.js'
+      - 'docusaurus.config.js'
+      - 'package.json'
+
+jobs:
+  deploy:
+    name: Deploy to GitHub Pages
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: 📥 Checkout repository
+        uses: actions/checkout@v4
+
+      - name: ⚙️ Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - name: 📦 Install dependencies
+        run: npm ci
+
+      - name: 🛠️ Build Docusaurus site
+        run: npm run docs:build
+
+      - name: 🚀 Deploy to GitHub Pages
+        uses: peaceiris/actions-gh-pages@v4
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./docs-site/build
+          publish_branch: gh-pages
+          user_name: github-actions[bot]
+          user_email: 41898282+github-actions[bot]@users.noreply.github.com

package/.github/workflows/publish-release-blog.yml

@@ -0,0 +1,53 @@
+# .github/workflows/publish-release-blog.yml
+# Version: 1.1.0
+# Description: Auto-publish blog post from CHANGELOG or prepared .md after NPM release
+# Author: Ali Kahwaji
+
+name: ✨ Publish Release Blog Post
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  publish-blog-post:
+    name: Generate & Commit Blog Post
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: ⬇️ Checkout main branch
+        uses: actions/checkout@v4
+        with:
+          ref: main         # ✅ Ensures blog is committed to 'main'
+          fetch-depth: 0
+
+      - name: 🔧 Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - name: ✏️ Generate blog post
+        run: |
+          VERSION=${GITHUB_REF#refs/tags/}
+          DATE=$(date +'%Y-%m-%d')
+          FILE="docs-site/blog/${DATE}-release-${VERSION}.md"
+
+          echo "---" > $FILE
+          echo "slug: release-${VERSION}" >> $FILE
+          echo "title: ✨ Version ${VERSION} Released" >> $FILE
+          echo "authors: [ali]" >> $FILE
+          echo "tags: [release, changelog]" >> $FILE
+          echo "---" >> $FILE
+          echo >> $FILE
+          echo "RAG Pipeline Utils v${VERSION} is now available on NPM!" >> $FILE
+          echo >> $FILE
+          echo "#### Highlights" >> $FILE
+          echo "- 🚀 Auto-published blog from GitHub release" >> $FILE
+          echo "- 📦 Version bump: ${VERSION}" >> $FILE
+          echo "- 📘 See CHANGELOG.md for full details" >> $FILE
+
+      - name: 📋 Commit blog post
+        uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+          commit_message: "docs(blog): add release post for ${{ github.ref_name }}"
+          file_pattern: "docs-site/blog/*.md"

package/.github/workflows/storybook-deploy.yml

@@ -0,0 +1,33 @@
+# .github/workflows/storybook-deploy.yml
+name: 📘 Deploy Storybook
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'docs-site/src/components/**'
+      - '.storybook/**'
+
+jobs:
+  deploy-storybook:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Build Storybook
+        run: npm run build-storybook
+
+      - name: Deploy to GitHub Pages
+        uses: peaceiris/actions-gh-pages@v4
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./storybook-static

package/.husky/pre-commit

@@ -3,3 +3,4 @@
 
 npm run validate-fixtures && npm run lint
 npx lint-staged
+node scripts/validate-changelog-version.js

package/.storybook/main.js

@@ -0,0 +1,20 @@
+
+
+/** @type { import('@storybook/react-webpack5').StorybookConfig } */
+const config = {
+  "stories": [
+    "../src/**/*.mdx",
+    "../src/**/*.stories.@(js|jsx|mjs|ts|tsx)"
+  ],
+  "addons": [
+    "@storybook/addon-webpack5-compiler-swc",
+    "@storybook/addon-essentials",
+    "@storybook/addon-onboarding",
+    "@storybook/addon-interactions"
+  ],
+  "framework": {
+    "name": "@storybook/react-webpack5",
+    "options": {}
+  }
+};
+export default config;

package/.storybook/preview.js

@@ -0,0 +1,13 @@
+/** @type { import('@storybook/react').Preview } */
+const preview = {
+  parameters: {
+    controls: {
+      matchers: {
+       color: /(background|color)$/i,
+       date: /Date$/i,
+      },
+    },
+  },
+};
+
+export default preview;

package/CHANGELOG.md

@@ -1,6 +1,20 @@
 # Changelog
 
-# [2.0.0](https://github.com/DevilsDev/rag-pipeline-utils/compare/v1.0.1...v2.0.0) (2025-04-17)
+# [2.1.0](https://github.com/DevilsDev/rag-pipeline-utils/compare/v2.0.2...v2.1.0) (2025-04-18)
+
+
+### Features
+
+* **docs-site:** add Storybook integration with BlogCard + LQIP image automation ([80965b8](https://github.com/DevilsDev/rag-pipeline-utils/commit/80965b8ef1d26372852ee0f39b56a73f337e0cea))
+
+## [2.0.1](https://github.com/DevilsDev/rag-pipeline-utils/compare/v2.0.0...v2.0.1) (2025-04-17)
+
+
+### Bug Fixes
+
+* add @docusaurus/preset-classic to enable CI builds ([ed5e771](https://github.com/DevilsDev/rag-pipeline-utils/commit/ed5e771f919face4693a6a7abfed4cf0fe1325ca))
+
+## [2.0.0] - 2025-04-17 (2025-04-17)
 
 
 ### Bug Fixes

package/README.md

@@ -1,117 +1,146 @@
 # @DevilsDev/rag-pipeline-utils
 
+
+
+# RAG Pipeline Utils
+
+> Modular utilities for building Retrieval-Augmented Generation (RAG) pipelines, with CI-verified plugin contracts and mockable interfaces.
+
 [![CI](https://github.com/DevilsDev/rag-pipeline-utils/actions/workflows/ci.yml/badge.svg)](https://github.com/DevilsDev/rag-pipeline-utils/actions)
 [![codecov](https://codecov.io/gh/DevilsDev/rag-pipeline-utils/branch/main/graph/badge.svg)](https://codecov.io/gh/DevilsDev/rag-pipeline-utils)
 ![Release](https://img.shields.io/github/v/release/DevilsDev/rag-pipeline-utils?label=release)
+![License](https://img.shields.io/github/license/devilsdev/rag-pipeline-utils)
 
-**A modular Node.js toolkit for building RAG (Retrieval-Augmented Generation) pipelines with clean abstractions, pluggable components, and scalable architecture.**
+## Description
 
----
+`@devilsdev/rag-pipeline-utils` provides a clean, extensible, and test-friendly foundation for constructing robust RAG pipelines using a plugin-based architecture. It enables developers to plug in their own document loaders, embedders, retrievers, and LLMs via configuration, and includes a CLI, mock contract verification, schema validation, and test harnesses to ensure production-readiness.
 
-## Features
+## Key Features
 
-- ✅ Plugin-based architecture for RAG pipelines
-- ✅ Embedding + retrieval abstraction
-- ✅ Vector DB integrations (e.g., Pinecone)
-- ✅ Modular document loaders (PDF, HTML, Markdown, CSV, Directory)
-- ✅ CLI with `.ragrc.json` config fallback
-- ✅ Retry, backoff, and logging built-in
-- ✅ Reranker support and evaluation tooling
-- ✅ React dashboard for evaluation results
+- **Plugin Contracts**: Runtime and CI verification of plugin interfaces (loader, embedder, retriever, llm).
+- **CLI Support**: Easily run `ingest` and `query` commands using `.ragrc.json` configuration.
+- **Test Fixtures**: Snapshot, fallback, and structured reranker validation.
+- **Schema Enforcement**: `.ragrc.json` schema validation to catch config issues early.
+- **Mock Repair Utilities**: Auto-fix for missing or broken mock implementations.
+- **Modular Design**: Drop-in plugin support using custom class-based implementations.
 
----
+## Benefits of This Modular Design
+
+- **Extensibility**: Developers can inject their own loader, retriever, or LLM logic without modifying the core.
+- **Testability**: Mocks and CI tools validate contract adherence for reliable test coverage.
+- **Decoupling**: Each plugin can be independently implemented and swapped in via `.ragrc.json`.
+- **CI Resilience**: The CI pipeline includes linting, contract verification, and test coverage enforcement.
 
 ## Installation
 
 ```bash
-npm install @yourorg/rag-pipeline-utils
+npm install @devilsdev/rag-pipeline-utils
 ```
 
----
+## Basic Usage
 
-## Usage
+1. Create a `.ragrc.json` config:
 
-```js
-import { createRagPipeline } from '@yourorg/rag-pipeline-utils';
+```json
+{
+  "loader": {
+    "pdf": "./src/mocks/pdf-loader.js"
+  },
+  "embedder": {
+    "openai": "./src/mocks/openai-embedder.js"
+  },
+  "retriever": {
+    "pinecone": "./src/mocks/pinecone-retriever.js"
+  },
+  "llm": {
+    "openai": "./src/mocks/openai-llm.js"
+  },
+  "namespace": "demo-namespace",
+  "pipeline": ["loader", "embedder", "retriever"]
+}
+```
 
-const rag = createRagPipeline({
-  loader: 'markdown',
-  embedder: 'openai',
-  retriever: 'pinecone',
-  llm: 'openai-gpt-4',
-  useReranker: true
-});
+2. Run an ingestion:
 
-const answer = await rag.query('What is a vector database?');
-console.log(answer);
+```bash
+node bin/cli.js ingest ./docs/sample.pdf
 ```
 
----
-
-## Testing
+3. Run a query:
 
 ```bash
-npm test
+node bin/cli.js query "What is in the document?"
 ```
 
----
+## Plugin Contracts
+
+Each plugin must implement the following interface:
+
+| Type      | Required Methods                            |
+| --------- | ------------------------------------------- |
+| loader    | `load(filePath)`                          |
+| embedder  | `embed(documents)`, `embedQuery(query)` |
+| retriever | `store(vectors)`, `search(queryVector)` |
+| llm       | `ask(prompt)`                             |
+
+## Folder Structure
+
+```text
+/project-root
+├── /src
+│   ├── /routes
+│   ├── /services
+│   ├── /utils
+│   └── app.js
+├── /scripts
+│   ├── setup.js
+│   ├── ci-runner.js
+│   └── repair-fixtures.js
+├── /__tests__
+│   ├── /fixtures
+│   ├── /unit
+│   ├── /integration
+├── /public
+├── README.md
+├── USE_CASES.md
+├── CHANGELOG.md
+└── .env.example
+```
 
-## CLI
 
-```bash
-rag-pipeline ingest ./docs/my.pdf --loader pdf
-rag-pipeline query "Explain RAG pipelines" --llm openai-gpt-4
-rag-pipeline evaluate ./fixtures/sample-eval-dataset.json
-rag-pipeline rerank "Explain embeddings" --retriever pinecone
-```
 
-Supports `.ragrc.json` fallback configuration:
+## Why Use This?
 
-```json
-{
-  "loader": "directory",
-  "embedder": "openai",
-  "retriever": "pinecone",
-  "llm": "openai-gpt-4",
-  "useReranker": true
-}
-```
+- **Developer Experience**: Fast iteration, full contract CI
+- **Pluggable Design**: Swap in your stack (OpenAI, Pinecone, Langchain, etc.)
+- **Minimal & Modular**: Clean separation of concerns
+- **Prod-ready**: 100% test coverage, schema validation, CLI control
 
----
 
-## Project Structure
 
-```
-/src
-  ├── core/           # Pipeline + registry
-  ├── loader/         # Loaders for md, html, csv, dir
-  ├── reranker/       # LLM-based context reranker
-  ├── evaluate/       # Scoring, batch QA evaluation
-  ├── utils/          # Logger, retry, schema
-/bin/cli.js           # CLI entrypoint
-/__tests__/           # Unit, integration, snapshot tests
-/public/              # Evaluation dashboard (React)
-.github/workflows/ci.yml # CI config
-```
+## Use Cases
 
----
+Explore real-world DevX-focused examples in [USE_CASES.md](./Use-Cases.md), including:
 
-## Evaluation Dashboard
+- Mock-driven plugin TDD
+- CI-based interface validation
+- Multivendor ingestion and search
+- Runtime swap of LLM, retrievers, or embedders
+- Bootstrap pipelines with auto-repairable defaults
 
-```bash
-node server.js
-```
 
-Visit: [http://localhost:3000](http://localhost:3000) → Visualizes metrics, BLEU/ROUGE, pass rate.
 
----
+## Contributing
+
+We welcome contributions via PRs, especially new mock providers, validation helpers, and runtime extensions. Please follow the test + CI conventions before submitting.
+
+
 
 ## License
 
-Apache-2.0
+This project is licensed under the **GNU GPL v3.0** License - see the [LICENSE](./LICENSE) file for details.
 
 ---
 
-## 👤 Author
+Built with ❤️ by [Ali Kahwaji].
 
-Ali Kahwaji — [@DevilsDev](https://github.com/DevilsDev)

package/Use-Cases.md

@@ -0,0 +1,134 @@
+# RAG Pipeline Utils — Developer-Focused Use Cases
+
+> Modular, composable, and CI-verified RAG orchestration for modern AI applications.
+
+---
+
+##  Use Case 1: Pluggable Local RAG Stack for Prototyping
+
+**Problem:** Traditional RAG tools force tight vendor dependencies.
+
+**Solution:** Use mockable local plugin implementations to simulate loaders, embedders, retrievers, and LLMs.
+
+```bash
+npm install @devilsdev/rag-pipeline-utils
+```
+
+```js
+// cli.js or app.js
+import { PluginRegistry } from './src/core/plugin-registry.js';
+import { loadRagConfig } from './src/config/load-config.js';
+
+const config = loadRagConfig('.ragrc.json');
+const loader = PluginRegistry.getPlugin('loader', 'pdf');
+const docs = loader.load('./test.pdf');
+```
+
+**Benefits**
+- Local plugin mocks with expected method contracts
+- Reusable plugin templates (`pdf-loader.js`, `openai-embedder.js`)
+- Dev-first DX: debug, snapshot, iterate without cloud costs
+
+---
+
+##  Use Case 2: Declarative CLI RAG Pipelines (CI-Ready)
+
+**Problem:** Hardcoded pipeline steps can't be validated or changed by non-engineers.
+
+**Solution:** Define `.ragrc.json` once and reuse across environments.
+
+```json
+{
+  "loader": { "pdf": "./src/mocks/pdf-loader.js" },
+  "embedder": { "openai": "./src/mocks/openai-embedder.js" },
+  "retriever": { "pinecone": "./src/mocks/pinecone-retriever.js" },
+  "llm": { "openai": "./src/mocks/openai-llm.js" },
+  "namespace": "docs-poc",
+  "pipeline": ["loader", "embedder", "retriever"]
+}
+```
+
+```bash
+node ./bin/cli.js ingest ./docs/whitepaper.pdf
+node ./bin/cli.js query "What’s the goal of this architecture?"
+```
+
+**Benefits**
+- Teams can standardize pipelines in source control
+- Tests fail fast if structure or mocks drift
+- Non-devs can operate pipelines via CLI
+
+---
+
+## Use Case 3: Mock-First TDD for Plugin Development
+
+**Problem:** LLM, vector, and loader logic are often developed without tests.
+
+**Solution:** Define contract-compliant mocks and CI-validate their structure before connecting to APIs.
+
+```js
+export default class CustomEmbedder {
+  embed(docs) { return docs.map(...); }
+  embedQuery(q) { return [...]; }
+}
+```
+
+Run full contract verification:
+
+```bash
+npm run ci
+```
+
+**Benefits**
+- Every plugin is CI-verified for method presence
+- Shared contracts enforce required interfaces
+- Onboarding new devs? Just follow `pluginContracts.js`
+
+---
+
+##  Use Case 4: Vendor-Neutral AI Data Pipelines
+
+**Problem:** Every RAG stack is locked to OpenAI + Pinecone + Langchain.
+
+**Solution:** Use this project to mix, match, or mock vector/LLM pairs:
+
+```js
+"retriever": { "localstore": "./src/plugins/local-retriever.js" },
+"llm": { "cohere": "./src/plugins/cohere-llm.js" }
+```
+
+**Benefits**
+- Run multiple RAG pipelines in parallel (A/B testing, evals)
+- Lower infra cost with offline/local plugin support
+- Integrate with tools outside Langchain ecosystem
+
+---
+
+## 🛠 Use Case 5: Plugin Engineering with Guardrails
+
+**Problem:** Complex plugin systems break silently.
+
+**Solution:** This framework has:
+- Plugin method contract enforcement
+- CLI test runner (`scripts/ci-runner.js`)
+- Auto repair tool (`scripts/repair-fixtures.js`)
+
+**DX Boost**
+- One-line plugin repairs
+- Validated schema structure
+- Aligned to real-world developer workflows
+
+---
+
+## Cross References
+
+- [README](./README.md) — Project overview, architecture, CLI and configuration details
+- [pluginContracts.js](./src/core/plugin-contracts.js) — Interface contracts for loader, embedder, retriever, LLM
+- [diagnostic-reporter.js](./src/utils/ci/diagnostic-reporter.js) — CI error helper
+- [repair-fixtures.js](./scripts/repair-fixtures.js) — Auto-regeneration of mocks
+
+---
+
+## License
+
+Licensed under the **GPL-3.0 License** — see [LICENSE](./LICENSE) for full terms.

package/docs-site/.plaiceholder-cache/img_hero.jpg.json

@@ -0,0 +1 @@
+{"base64":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAQAAAACCAIAAADwyuo0AAAACXBIWXMAAAsTAAALEwEAmpwYAAAAGklEQVR4nGPILapfuW5vblF9blE9A4QD4QMApeANEwX0WNkAAAAASUVORK5CYII="}

package/docs-site/.plaiceholder-cache/img_logo.png.json

@@ -0,0 +1 @@
+{"base64":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAQAAAAECAYAAACp8Z5+AAAACXBIWXMAAAsTAAALEwEAmpwYAAAAT0lEQVR4nAFEALv/AIXNrxKW5b44TKykITGQkA0Atv/dK37FpoIVaGhbSqqbIQAPMjMPAAcXJAAHGTkXJjUQABw3QRQOJT0bFCA3HBUiNRZrJhOUbaRuqQAAAABJRU5ErkJggg=="}

package/docs-site/.storybook/preview.js

@@ -0,0 +1,8 @@
+// .storybook/preview.js
+export const parameters = {
+    layout: 'centered',
+    docs: {
+      title: '🧩 RAG Pipeline UI Components',
+    },
+  };
+  

package/docs-site/blog/2021-08-26-welcome/index.md

@@ -1,7 +1,6 @@
 ---
 slug: welcome
 title: Welcome
-authors: [slorber, yangshun]
 tags: [facebook, hello, docusaurus]
 ---