@devilsdev/rag-pipeline-utils 2.0.1 → 2.1.0

package/.eslintrc.json

@@ -1,21 +1,27 @@
 {
-    "env": {
-      "es2022": true,
-      "node": true,
-      "jest": true
-    },
-    "extends": [
-      "eslint:recommended"
+  "env": {
+    "es2022": true,
+    "node": true,
+    "jest": true
+  },
+  "extends": [
+    "eslint:recommended",
+    "plugin:storybook/recommended"
+  ],
+  "parserOptions": {
+    "ecmaVersion": "latest",
+    "sourceType": "module"
+  },
+  "rules": {
+    "no-unused-vars": "warn",
+    "no-console": "off",
+    "semi": [
+      "error",
+      "always"
     ],
-    "parserOptions": {
-      "ecmaVersion": "latest",
-      "sourceType": "module"
-    },
-    "rules": {
-      "no-unused-vars": "warn",
-      "no-console": "off",
-      "semi": ["error", "always"],
-      "quotes": ["error", "single"]
-    }
+    "quotes": [
+      "error",
+      "single"
+    ]
   }
-  
+}

package/.github/workflows/docs-deploy.yml

@@ -1,19 +1,28 @@
-name: Deploy Docusaurus to GitHub Pages
+# .github/workflows/docs-deploy.yml
+# Version: 1.1.0
+# Description: Deploys Docusaurus to GitHub Pages on changes to documentation files
+# Author: Ali Kahwaji
+
+name: 📚 Deploy Docusaurus Docs
 
 on:
   push:
     branches: [main]
     paths:
-                - 'docs-site/**'
-                - 'blog/**'
-                - 'sidebars.js'
-                - 'docusaurus.config.js'
+      - 'docs-site/**'
+      - 'docs/**'
+      - 'blog/**'
+      - 'sidebars.js'
+      - 'docusaurus.config.js'
+      - 'package.json'
 
 jobs:
   deploy:
+    name: Deploy to GitHub Pages
     runs-on: ubuntu-latest
+
     steps:
-      - name: 📥 Checkout
+      - name: 📥 Checkout repository
         uses: actions/checkout@v4
 
       - name: ⚙️ Setup Node.js
@@ -24,13 +33,14 @@ jobs:
       - name: 📦 Install dependencies
         run: npm ci
 
-      - name: 🛠️ Build static site
-        run: npm run build:docs
+      - name: 🛠️ Build Docusaurus site
+        run: npm run docs:build
 
       - name: 🚀 Deploy to GitHub Pages
         uses: peaceiris/actions-gh-pages@v4
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: ./docs-site/build
-              
-          
+          publish_branch: gh-pages
+          user_name: github-actions[bot]
+          user_email: 41898282+github-actions[bot]@users.noreply.github.com

package/.github/workflows/publish-release-blog.yml

@@ -1,5 +1,5 @@
 # .github/workflows/publish-release-blog.yml
-# Version: 1.0.0
+# Version: 1.1.0
 # Description: Auto-publish blog post from CHANGELOG or prepared .md after NPM release
 # Author: Ali Kahwaji
 
@@ -15,8 +15,11 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - name: ⬇️ Checkout code
+      - name: ⬇️ Checkout main branch
         uses: actions/checkout@v4
+        with:
+          ref: main         # ✅ Ensures blog is committed to 'main'
+          fetch-depth: 0
 
       - name: 🔧 Set up Node.js
         uses: actions/setup-node@v4
@@ -36,13 +39,12 @@ jobs:
           echo "tags: [release, changelog]" >> $FILE
           echo "---" >> $FILE
           echo >> $FILE
-
           echo "RAG Pipeline Utils v${VERSION} is now available on NPM!" >> $FILE
           echo >> $FILE
           echo "#### Highlights" >> $FILE
-          echo "- Automated blog generation on GitHub releases" >> $FILE
-          echo "- Version bump: ${VERSION}" >> $FILE
-          echo "- Ensure CHANGELOG.md is synced" >> $FILE
+          echo "- 🚀 Auto-published blog from GitHub release" >> $FILE
+          echo "- 📦 Version bump: ${VERSION}" >> $FILE
+          echo "- 📘 See CHANGELOG.md for full details" >> $FILE
 
       - name: 📋 Commit blog post
         uses: stefanzweifel/git-auto-commit-action@v5

package/.github/workflows/storybook-deploy.yml

@@ -0,0 +1,33 @@
+# .github/workflows/storybook-deploy.yml
+name: 📘 Deploy Storybook
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'docs-site/src/components/**'
+      - '.storybook/**'
+
+jobs:
+  deploy-storybook:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Build Storybook
+        run: npm run build-storybook
+
+      - name: Deploy to GitHub Pages
+        uses: peaceiris/actions-gh-pages@v4
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./storybook-static

package/.storybook/main.js

@@ -0,0 +1,20 @@
+
+
+/** @type { import('@storybook/react-webpack5').StorybookConfig } */
+const config = {
+  "stories": [
+    "../src/**/*.mdx",
+    "../src/**/*.stories.@(js|jsx|mjs|ts|tsx)"
+  ],
+  "addons": [
+    "@storybook/addon-webpack5-compiler-swc",
+    "@storybook/addon-essentials",
+    "@storybook/addon-onboarding",
+    "@storybook/addon-interactions"
+  ],
+  "framework": {
+    "name": "@storybook/react-webpack5",
+    "options": {}
+  }
+};
+export default config;

package/.storybook/preview.js

@@ -0,0 +1,13 @@
+/** @type { import('@storybook/react').Preview } */
+const preview = {
+  parameters: {
+    controls: {
+      matchers: {
+       color: /(background|color)$/i,
+       date: /Date$/i,
+      },
+    },
+  },
+};
+
+export default preview;

package/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+# [2.1.0](https://github.com/DevilsDev/rag-pipeline-utils/compare/v2.0.2...v2.1.0) (2025-04-18)
+
+
+### Features
+
+* **docs-site:** add Storybook integration with BlogCard + LQIP image automation ([80965b8](https://github.com/DevilsDev/rag-pipeline-utils/commit/80965b8ef1d26372852ee0f39b56a73f337e0cea))
+
 ## [2.0.1](https://github.com/DevilsDev/rag-pipeline-utils/compare/v2.0.0...v2.0.1) (2025-04-17)
 
 

package/README.md

@@ -120,7 +120,7 @@ Each plugin must implement the following interface:
 
 ## Use Cases
 
-Explore real-world DevX-focused examples in [USE_CASES.md](./USE_CASES.md), including:
+Explore real-world DevX-focused examples in [USE_CASES.md](./Use-Cases.md), including:
 
 - Mock-driven plugin TDD
 - CI-based interface validation

package/docs-site/.plaiceholder-cache/img_hero.jpg.json

@@ -0,0 +1 @@
+{"base64":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAQAAAACCAIAAADwyuo0AAAACXBIWXMAAAsTAAALEwEAmpwYAAAAGklEQVR4nGPILapfuW5vblF9blE9A4QD4QMApeANEwX0WNkAAAAASUVORK5CYII="}

package/docs-site/.plaiceholder-cache/img_logo.png.json

@@ -0,0 +1 @@
+{"base64":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAQAAAAECAYAAACp8Z5+AAAACXBIWXMAAAsTAAALEwEAmpwYAAAAT0lEQVR4nAFEALv/AIXNrxKW5b44TKykITGQkA0Atv/dK37FpoIVaGhbSqqbIQAPMjMPAAcXJAAHGTkXJjUQABw3QRQOJT0bFCA3HBUiNRZrJhOUbaRuqQAAAABJRU5ErkJggg=="}

package/docs-site/.storybook/preview.js

@@ -0,0 +1,8 @@
+// .storybook/preview.js
+export const parameters = {
+    layout: 'centered',
+    docs: {
+      title: '🧩 RAG Pipeline UI Components',
+    },
+  };
+  

package/docs-site/blog/2025-04-17-release-v2.0.2.md

@@ -0,0 +1,13 @@
+---
+slug: release-v2.0.2
+title: ✨ Version v2.0.2 Released
+authors: [ali]
+tags: [release, changelog]
+---
+
+RAG Pipeline Utils vv2.0.2 is now available on NPM!
+
+#### Highlights
+- 🚀 Auto-published blog from GitHub release
+- 📦 Version bump: v2.0.2
+- 📘 See CHANGELOG.md for full details

package/docs-site/blog/2025-04-18-architecture-overview.md

@@ -0,0 +1,34 @@
+---
+slug: architecture-overview
+title: RAG Pipeline Architecture Overview
+authors: [ali]
+tags: [architecture, design, modular]
+description: A deep dive into the architecture powering RAG Pipeline Utils — from plugin interfaces to flexible pipeline composition.
+---
+
+Welcome to the first post in our deep dive blog series on **RAG Pipeline Utils** — a fully modular, testable, and production-grade framework for building retrieval-augmented generation pipelines.
+
+## Why Modular RAG?
+
+Most RAG tools are tightly coupled or opinionated. We built this utility to empower dev teams with:
+
+-  Swappable plugins for loaders, embedders, retrievers, LLMs
+-  CLI tools for evaluation & schema validation
+-  Docusaurus-based docs with versioned releases
+-  Full CI/CD and automated NPM releases
+
+## Architecture Highlights
+
+- `PluginRegistry`: Registers and resolves all components.
+- `validateSchema`: Enforces `.ragrc.json` format across all pipelines.
+- `mocks/`: Local test doubles for every plugin interface.
+- `__tests__/`: Dual-layer coverage — unit + integration.
+
+## Try It Now
+
+```bash
+npm install @devilsdev/rag-pipeline-utils
+npx rag-utils run --config .ragrc.json
+```
+
+Stay tuned for future posts including CI validation, RAG evaluation tooling, and how to build your own dashboard for prompt scoring.

package/docs-site/blog/2025-04-19-prompt-engineering-best-practices.md

@@ -0,0 +1,31 @@
+---
+title: "Prompt Engineering Best Practices for RAG"
+slug: prompt-engineering-best-practices
+authors: [ali]
+tags: [prompt-engineering, design]
+image: /img/social/prompt-engineering-card.png
+description: "A deep dive into prompt optimization strategies for RAG pipelines using modular design."
+---
+Prompt engineering is a critical lever in maximizing the effectiveness of Retrieval-Augmented Generation (RAG) systems. In this article, we explore:
+
+## Prompting in Modular RAG Pipelines
+
+Your `rag-pipeline-utils` system allows swappable LLM backends and encourages prompt templates as part of the reranker or query orchestrator.
+
+- Always define clear input/output expectations
+- Leverage JSON or YAML-based structured prompts where applicable
+- Evaluate with real examples using the CLI + evaluation framework
+
+## What to Include in Prompts
+
+- **Context injection**: retrieved chunks, metadata, source provenance
+- **Task framing**: instruction-oriented for LLM consistency
+- **Scoring heuristics**: if doing ranking, consider scoring guidelines
+
+## Prompt as Plugin (Future Feature)
+
+Eventually, we plan to support `promptTemplate` as a plugin type – enabling reusable prompt logic alongside `loader`, `retriever`, etc.
+
+---
+
+Stay tuned for our upcoming prompt debugger dashboard, and follow [releases](/blog/tags/release) for versioned updates.

package/docs-site/blog/2025-04-20-image-showcase.md

@@ -0,0 +1,23 @@
+---
+slug: image-showcase
+title: Blog Image Component Showcase
+authors: [ali]
+tags: [tutorials, design, image]
+description: Demonstrating LQIP-powered blog images using the BlogImage component.
+---
+
+import BlogImage from '@site/src/components/BlogImage';
+
+This post demonstrates the reusable **`<BlogImage />`** component, which displays optimized blurred placeholders (LQIPs) for faster rendering.
+
+## Example
+
+<BlogImage srcKey="img_hero" alt="Hero demonstration" />
+
+---
+
+## Benefits Recap
+
+- Blurred placeholder loading for faster perception
+- Fallback to default cache when missing
+- Ready for automation via `watch-images.js`

package/docs-site/blog/authors.yml

@@ -1,5 +1,5 @@
 ali:
   name: Ali Kahwaji
-  title: Creator of RAG Pipeline Utils
-  url: https://github.com/DevilsDev
+  title: Creator & Maintainer of RAG Pipeline Utils
+  url: https://github.com/alikahwaji
   image_url: /img/authors/ali.jpg

package/docs-site/blog/tags.yml

@@ -1,28 +1,30 @@
-facebook:
-  label: Facebook
-  permalink: /facebook
-  description: Facebook tag description
+# docs-site/blog/tags.yml
+# Description: Central tag definitions for Docusaurus blog filtering and grouping
 
-hello:
-  label: Hello
-  permalink: /hello
-  description: Hello tag description
+- label: Release
+  slug: release
+  permalink: /blog/tags/release
 
-docusaurus:
-  label: Docusaurus
-  permalink: /docusaurus
-  description: Docusaurus tag description
+- label: Architecture
+  slug: architecture
+  permalink: /blog/tags/architecture
 
-hola:
-  label: Hola
-  permalink: /hola
-  description: Hola tag description
+- label: Prompt Engineering
+  slug: prompt-engineering
+  permalink: /blog/tags/prompt-engineering
 
-release:
-  label: Release
-  permalink: /blog/tags/release
+- label: Evaluation
+  slug: evaluation
+  permalink: /blog/tags/evaluation
+
+- label: Tutorials
+  slug: tutorials
+  permalink: /blog/tags/tutorials
 
-changelog:
-  label: Changelog
+- label: Changelog
+  slug: changelog
   permalink: /blog/tags/changelog
 
+- label: Design
+  slug: design
+  permalink: /blog/tags/design

package/docs-site/docs/Architecture.md

@@ -4,7 +4,7 @@ This page outlines the internal architecture of the RAG pipeline utilities, emph
 
 ---
 
-## 🧩 Core Design Philosophy
+## Core Design Philosophy
 
 The architecture adheres to enterprise-grade practices:
 
@@ -16,7 +16,7 @@ The architecture adheres to enterprise-grade practices:
 
 ---
 
-## 🏗️ Layered Components
+## Layered Components
 
 ```
 +-------------------------+
@@ -36,7 +36,7 @@ The architecture adheres to enterprise-grade practices:
 
 ---
 
-## 📦 Core Interfaces
+## Core Interfaces
 
 Each plugin is registered via the `PluginRegistry` and looked up by key.
 
@@ -65,7 +65,7 @@ interface Retriever {
 
 ---
 
-## 🔁 DAG Support
+## DAG Support
 
 The `dag-engine.js` module supports chaining multiple components:
 - Example: Summarize → Retrieve → Rerank → LLM

package/docs-site/docs/CLI.md

@@ -14,7 +14,7 @@ npx rag-pipeline <command>
 
 ---
 
-## 🔧 Commands
+## Commands
 
 ### `ingest`
 ```bash
@@ -42,7 +42,7 @@ rag-pipeline rerank "What is RAG?" --retriever pinecone
 
 ---
 
-## ⚙️ `.ragrc.json` Config
+## `.ragrc.json` Config
 
 Defaults can be set in a file:
 ```json
@@ -57,7 +57,7 @@ Defaults can be set in a file:
 
 ---
 
-## 🐳 Docker Usage
+##  Docker Usage
 
 ```bash
 docker build -t rag-pipeline .

package/docs-site/docs/Evaluation.md

@@ -4,7 +4,7 @@ This module allows you to evaluate the quality of generated LLM responses in a R
 
 ---
 
-## 🎯 Objective
+## Objective
 
 Quantify how well the pipeline performs on question-answer pairs using:
 - **BLEU**: Precision on matching tokens
@@ -13,7 +13,7 @@ Quantify how well the pipeline performs on question-answer pairs using:
 
 ---
 
-## 📄 Dataset Format
+## Dataset Format
 
 Each entry in your JSON dataset should include:
 
@@ -28,20 +28,20 @@ Saved as: `sample-eval-dataset.json`
 
 ---
 
-## 🚀 CLI Evaluation
+## CLI Evaluation
 
 ```bash
 rag-pipeline evaluate ./fixtures/sample-eval-dataset.json
 ```
 
 Outputs:
-- ✅ Pass/fail per item
-- 📊 BLEU/ROUGE scores
-- 📈 Aggregated metrics
+- Pass/fail per item
+- BLEU/ROUGE scores
+- Aggregated metrics
 
 ---
 
-## 📊 Dashboard Visualization
+## Dashboard Visualization
 
 Use the React dashboard under `/public`:
 

package/docs-site/docs/FAQ.md

@@ -1,6 +1,6 @@
 # FAQ
 
-### ❓ Can I use this with my own vector database?
+### Can I use this with my own vector database?
 Yes! You can implement the `Retriever` interface and register it:
 ```ts
 registry.register('retriever', 'mydb', new MyRetriever());
@@ -9,17 +9,17 @@ Then use it via CLI or `.ragrc.json`.
 
 ---
 
-### ❓ Is streaming LLM response supported?
+### Is streaming LLM response supported?
 Yes. LLM runners can yield async output tokens. This is handled internally via `AsyncIterable` in compatible models.
 
 ---
 
-### ❓ How do I evaluate performance?
+### How do I evaluate performance?
 Use `rag-pipeline evaluate dataset.json` or open the dashboard at `http://localhost:3000` to visualize BLEU/ROUGE and pass rates.
 
 ---
 
-### ❓ What file types can I ingest?
+### What file types can I ingest?
 Currently supported:
 - `.pdf` (mocked)
 - `.md`
@@ -29,7 +29,7 @@ Currently supported:
 
 ---
 
-### ❓ Can I rerank results using a local model?
+### Can I rerank results using a local model?
 Yes, as long as your local LLM implements:
 ```ts
 llm.generate(prompt, context): Promise<string>
@@ -38,7 +38,7 @@ Use it to power `LLMReranker`.
 
 ---
 
-### ❓ How do I contribute?
+### How do I contribute?
 Fork the repo → add your plugin or CLI feature → test → submit PR.
 
 ---

package/docs-site/docs/Introduction.md

@@ -4,11 +4,11 @@ sidebar_position: 1
 
 # Introduction
 
-Welcome to **@yourorg/rag-pipeline-utils**, a modular Node.js toolkit for building scalable, pluggable Retrieval-Augmented Generation (RAG) systems.
+Welcome to **@DevilsDev/rag-pipeline-utils**, a modular Node.js toolkit for building scalable, pluggable Retrieval-Augmented Generation (RAG) systems.
 
 ---
 
-## 🚀 What is RAG?
+##  What is RAG?
 
 RAG combines information retrieval with large language models to enhance generation with factual grounding. It typically involves:
 
@@ -19,7 +19,7 @@ This project provides all the composable parts to build your own RAG pipeline.
 
 ---
 
-## 🔧 Why Use This Toolkit?
+##  Why Use This Toolkit?
 
 - **Plugin-based**: Swap in your own loaders, retrievers, LLMs
 - **Streaming-ready**: Async-friendly, token-by-token output
@@ -29,7 +29,7 @@ This project provides all the composable parts to build your own RAG pipeline.
 
 ---
 
-## 🧱 Architecture
+## Architecture
 
 Each RAG pipeline includes:
 
@@ -43,13 +43,13 @@ The pipeline can be used via code or via CLI commands.
 
 ---
 
-## ✅ Status
+## Status
 
 This project is:
-- ✅ Production-ready
-- ✅ Covered by unit/integration tests
-- ✅ Compatible with Node.js `>=18`
-- ✅ Published to npm under `@yourorg`
+- Production-ready
+- Covered by unit/integration tests
+- Compatible with Node.js `>=18`
+- Published to npm under `DevilsDev`
 
 ---
 

package/docs-site/docs/Plugins.md

@@ -4,7 +4,7 @@ The toolkit is fully modular. You can inject your own loaders, retrievers, embed
 
 ---
 
-## 🔌 Plugin Architecture
+## Plugin Architecture
 
 All plugins follow a strict interface pattern. Each is registered under a namespace using the plugin registry:
 
@@ -21,7 +21,7 @@ You can register plugins for:
 
 ---
 
-## 🧱 Loader Example
+## Loader Example
 
 ```ts
 class MyMarkdownLoader {
@@ -39,7 +39,7 @@ registry.register('loader', 'md', new MyMarkdownLoader());
 
 ---
 
-## ⚡ Runtime Override
+## Runtime Override
 
 CLI and API will resolve plugin names from `pluginRegistry`. Use: