@devilsdev/rag-pipeline-utils 2.1.4 → 2.1.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devilsdev/rag-pipeline-utils",
-  "version": "2.1.4",
+  "version": "2.1.5",
   "description": "A modular toolkit for building RAG (Retrieval-Augmented Generation) pipelines in Node.js",
   "main": "src/core/create-pipeline.js",
   "bin": {
@@ -27,7 +27,10 @@
     "docs:start": "docusaurus start docs-site",
     "docs:build": "docusaurus build docs-site",
     "docs:serve": "docusaurus serve docs-site",
-    "docs:deploy": "docusaurus deploy docs-site"
+    "docs:deploy": "docusaurus deploy docs-site",
+    "release:tag": "node scripts/tag-release.js",
+    "roadmap:close": "node scripts/close-done-roadmap-issues.js",
+    "labels:sync": "node scripts/ensure-roadmap-labels.js"
   },
   "lint-staged": {
     "**/*.{js,ts}": [
@@ -68,14 +71,6 @@
     "@semantic-release/github": "^9.2.6",
     "@semantic-release/npm": "^11.0.3",
     "@semantic-release/release-notes-generator": "^14.0.3",
-    "@storybook/addon-essentials": "^8.6.12",
-    "@storybook/addon-interactions": "^8.6.12",
-    "@storybook/addon-onboarding": "^8.6.12",
-    "@storybook/addon-webpack5-compiler-swc": "^3.0.0",
-    "@storybook/blocks": "^8.6.12",
-    "@storybook/react": "^8.6.12",
-    "@storybook/react-webpack5": "^8.6.12",
-    "@storybook/test": "^8.6.12",
     "@types/jest": "^29.5.14",
     "babel-jest": "^29.7.0",
     "codecov": "^3.8.2",
@@ -92,10 +87,10 @@
     "prop-types": "^15.8.1",
     "rimraf": "^6.0.1",
     "semantic-release": "^21.1.2",
-    "storybook": "^8.6.12",
     "ts-jest": "^29.3.2"
   },
   "dependencies": {
+    "@octokit/rest": "^21.1.1",
     "ajv": "^8.17.1",
     "axios": "^1.8.4",
     "chalk": "^5.4.1",
@@ -105,6 +100,7 @@
     "fast-glob": "^3.3.3",
     "framer-motion": "^12.7.4",
     "jsdom": "^26.1.0",
+    "octokit": "^4.1.3",
     "openai": "^4.93.0",
     "pino": "^9.6.0",
     "plaiceholder": "^3.0.0",

package/scripts/banner-injector.js

@@ -1,89 +1,102 @@
 /**
- * File: scripts/banner-injector.js
- * Description: Script to programmatically inject file-level banners into specified project files
- * Author: Ali Kahwaji
  * Version: 1.0.0
+ * Description: Syncs roadmap items from a structured Markdown file into GitHub Issues with labels and assignees.
+ * Author: Ali Kahwaji
  */
 
 import fs from 'fs';
 import path from 'path';
-import glob from 'glob';
+import { Octokit } from 'octokit';
+import dotenv from 'dotenv';
 
-const banners = {
-  '__tests__/unit/reranker/llm-reranker.test.js': `/**
- * File: __tests__/unit/reranker/llm-reranker.test.js
- * Description: Unit and integration tests with mocks for RAG pipeline
- * Author: Ali Kahwaji
- * Version: 1.0.4
- */\n\n`,
+dotenv.config();
 
-  '__tests__/unit/reranker/reranker.snapshot.test.js': `/**
- * File: __tests__/unit/reranker/reranker.snapshot.test.js
- * Description: Snapshot tests for LLM reranker
- * Author: Ali Kahwaji
- * Version: 1.0.4
- */\n\n`,
+/**
+ * GitHub Personal Access Token required via .env or CI secrets
+ * Ensure the token has 'repo' scope for private issue creation.
+ */
+const GITHUB_TOKEN = process.env.GITHUB_TOKEN;
+const GITHUB_REPO = process.env.GITHUB_REPO; // e.g. DevilsDev/rag-pipeline-utils
+const ROADMAP_PATH = path.resolve('.github/PROJECT_ROADMAP.md');
 
-  '__tests__/unit/core/plugin-registry.test.js': `/**
- * File: __tests__/unit/core/plugin-registry.test.js
- * Description: Unit and integration tests with mocks for RAG pipeline
- * Author: Ali Kahwaji
- * Version: 1.0.4
- */\n\n`,
+if (!GITHUB_TOKEN || !GITHUB_REPO) {
+  console.error('❌ GITHUB_TOKEN and GITHUB_REPO must be defined in environment.');
+  process.exit(1);
+}
 
-  '__tests__/integration/config/load-config.test.js': `/**
- * File: __tests__/integration/config/load-config.test.js
- * Description: Unit and integration tests with mocks for RAG pipeline
- * Author: Ali Kahwaji
- * Version: 1.0.4
- */\n\n`,
+const octokit = new Octokit({ auth: GITHUB_TOKEN });
 
-  '__tests__/integration/cli/config-flow.test.js': `/**
- * File: __tests__/integration/cli/config-flow.test.js
- * Description: Unit and integration tests with mocks for RAG pipeline
- * Author: Ali Kahwaji
- * Version: 1.0.4
- */\n\n`,
+/**
+ * Parse roadmap Markdown file into structured issue definitions
+ */
+function parseRoadmapMarkdown(markdown) {
+  const lines = markdown.split('\n');
+  const issues = [];
 
-  'src/mocks/pdf-loader.js': `/**
- * File: src/mocks/pdf-loader.js
- * Description: Unit and integration tests with mocks for RAG pipeline
- * Author: Ali Kahwaji
- * Version: 1.0.4
- */\n\n`,
+  for (const line of lines) {
+    if (line.startsWith('|')) {
+      const cells = line.split('|').map(c => c.trim());
 
-  'src/mocks/openai-embedder.js': `/**
- * File: src/mocks/openai-embedder.js
- * Description: Unit and integration tests with mocks for RAG pipeline
- * Author: Ali Kahwaji
- * Version: 1.0.4
- */\n\n`,
+      if (cells.length < 6 || cells[0] === 'Phase') continue;
 
-  'src/mocks/pinecone-retriever.js': `/**
- * File: src/mocks/pinecone-retriever.js
- * Description: Unit and integration tests with mocks for RAG pipeline
- * Author: Ali Kahwaji
- * Version: 1.0.4
- */\n\n`
-};
-
-Object.entries(banners).forEach(([filePath, banner]) => {
-  const resolvedPath = path.resolve(filePath);
+      const [, phase, priority, feature, group, tags] = cells;
 
-  if (!fs.existsSync(resolvedPath)) {
-    console.warn(`[SKIPPED] ${filePath} does not exist.`);
-    return;
+      issues.push({
+        title: `${phase.trim()}: ${feature.trim()}`,
+        body: `**Feature Group:** ${group}\n**Priority:** ${priority}`,
+        labels: [phase, priority, ...tags.split(',').map(tag => tag.trim())].filter(Boolean),
+      });
+    }
   }
 
-  const originalContent = fs.readFileSync(resolvedPath, 'utf-8');
+  return issues;
+}
+
+/**
+ * Create an issue in GitHub if it doesn't already exist
+ */
+async function createOrUpdateIssues(issues) {
+  const [owner, repo] = GITHUB_REPO.split('/');
+
+  const { data: existingIssues } = await octokit.rest.issues.listForRepo({
+    owner,
+    repo,
+    state: 'all',
+    per_page: 100,
+  });
 
-  // Skip if banner already present
-  if (originalContent.startsWith('/**')) {
-    console.log(`[SKIPPED] ${filePath} already contains a banner.`);
-    return;
+  for (const issue of issues) {
+    const exists = existingIssues.find(i => i.title === issue.title);
+    if (exists) {
+      console.log(`✅ Skipped (exists): ${issue.title}`);
+      continue;
+    }
+
+    await octokit.rest.issues.create({
+      owner,
+      repo,
+      title: issue.title,
+      body: issue.body,
+      labels: issue.labels,
+    });
+
+    console.log(`➕ Created: ${issue.title}`);
+  }
+}
+
+/**
+ * Entrypoint
+ */
+async function main() {
+  try {
+    const roadmap = fs.readFileSync(ROADMAP_PATH, 'utf-8');
+    const issues = parseRoadmapMarkdown(roadmap);
+    await createOrUpdateIssues(issues);
+    console.log(`🎯 Synced ${issues.length} roadmap issues.`);
+  } catch (err) {
+    console.error('🚨 Roadmap sync failed:', err.message);
+    process.exit(1);
   }
+}
 
-  const newContent = banner + originalContent;
-  fs.writeFileSync(resolvedPath, newContent, 'utf-8');
-  console.log(`[INJECTED] Banner added to ${filePath}`);
-});
+main();

package/scripts/close-done-roadmap-issues.js

@@ -0,0 +1,65 @@
+/**
+ * Version: 1.1.0
+ * Description: Closes GitHub issues marked as ✅ Done in the PROJECT_ROADMAP.md
+ * Author: Ali Kahwaji
+ */
+
+import fs from 'fs';
+import { Octokit } from 'octokit';
+
+const GITHUB_TOKEN = process.env.GITHUB_TOKEN;
+const OWNER = 'DevilsDev';
+const REPO = 'rag-pipeline-utils';
+const ROADMAP_PATH = '.github/PROJECT_ROADMAP.md';
+
+const octokit = new Octokit({ auth: GITHUB_TOKEN });
+
+async function closeDoneIssues() {
+  const content = fs.readFileSync(ROADMAP_PATH, 'utf-8');
+  const lines = content.split('\n').filter(line => line.startsWith('|'));
+  if (lines.length < 3) {
+    console.log('⚠️ No roadmap rows found to evaluate.');
+    return;
+  }
+
+  const rows = lines.slice(2); // Skip header and separator
+
+  const { data: issues } = await octokit.rest.issues.listForRepo({
+    owner: OWNER,
+    repo: REPO,
+    state: 'open',
+    per_page: 100
+  });
+
+  for (const row of rows) {
+    const cols = row.split('|').map(col => col.trim());
+    const [, , , title, , status] = cols;
+
+    const safeStatus = status?.toLowerCase?.();
+    if (!safeStatus || !safeStatus.includes('✅')) continue;
+
+    const match = issues.find(issue => issue.title.trim() === title.trim());
+    if (!match) continue;
+
+    await octokit.rest.issues.createComment({
+      owner: OWNER,
+      repo: REPO,
+      issue_number: match.number,
+      body: '✅ Automatically closed from roadmap sync.'
+    });
+
+    await octokit.rest.issues.update({
+      owner: OWNER,
+      repo: REPO,
+      issue_number: match.number,
+      state: 'closed'
+    });
+
+    console.log(`🔒 Closed: #${match.number} "${title}"`);
+  }
+}
+
+closeDoneIssues().catch(error => {
+  console.error('❌ Error closing roadmap issues:', error);
+  process.exit(1);
+});

package/scripts/create-roadmap-issues.js

@@ -0,0 +1,47 @@
+// scripts/create-roadmap-issues.js
+
+/**
+ * Version: 2.0.0
+ * Description: Parses PROJECT_ROADMAP.md and syncs GitHub issues
+ * Author: Ali Kahwaji
+ */
+
+import fs from 'fs/promises';
+import path from 'path';
+import { Octokit } from 'octokit';
+import yaml from 'js-yaml';
+
+const roadmapPath = new URL('../.github/PROJECT_ROADMAP.md', import.meta.url);
+
+export async function createRoadmapIssues({ token, owner, repo }) {
+  const octokit = new Octokit({ auth: token });
+  const content = await fs.readFile(roadmapPath, 'utf-8');
+
+  const blocks = content.split(/^##\s+/gm).slice(1);
+
+  for (const block of blocks) {
+    const [header, ...bodyLines] = block.trim().split('\n');
+    const title = header.trim();
+    const metadata = {};
+    const description = [];
+
+    for (const line of bodyLines) {
+      if (line.includes(':') && line.includes('**')) {
+        const [key, val] = line.split(':');
+        metadata[key.trim()] = val.replace(/\*\*/g, '').trim();
+      } else {
+        description.push(line);
+      }
+    }
+
+    const issue = await octokit.rest.issues.create({
+      owner,
+      repo,
+      title,
+      body: description.join('\n'),
+      labels: [metadata.Priority, metadata.Group].filter(Boolean),
+    });
+
+    console.log(`✅ Created: ${issue.data.title}`);
+  }
+}

package/scripts/ensure-roadmap-labels.js

@@ -0,0 +1,73 @@
+/**
+ * Version: 2.0.1
+ * Path: scripts/ensure-roadmap-labels.js
+ * Description: Ensures consistent GitHub labels for roadmap tracking and automation.
+ * Author: Ali Kahwaji
+ */
+
+import { Octokit } from 'octokit';
+
+
+/**
+ * Label definitions for roadmap tracking.
+ * Modify this array to add or update roadmap labels.
+ * Each object must include: name, color (hex), description.
+ */
+const roadmapLabels = [
+  { name: 'priority: high', color: 'e11d48', description: 'High priority roadmap item' },
+  { name: 'priority: medium', color: 'f59e0b', description: 'Medium priority roadmap item' },
+  { name: 'priority: low', color: '10b981', description: 'Low priority roadmap item' },
+  { name: 'group: docs', color: '6366f1', description: 'Documentation features' },
+  { name: 'group: devx', color: '06b6d4', description: 'Developer experience improvements' },
+  { name: 'group: community', color: 'ec4899', description: 'Community tools & engagement' },
+  { name: 'group: blog', color: 'f97316', description: 'Blog & SEO enhancements' },
+  { name: 'group: infra', color: '64748b', description: 'Infrastructure & deployment features' },
+  { name: 'group: hydra', color: '9333ea', description: 'Hydra runtime & configuration' },
+  { name: 'status: done', color: '10b981', description: 'Issue has been completed' }
+];
+
+/**
+ * Ensures all roadmap labels are present in the target GitHub repository.
+ * Skips label creation if already present.
+ *
+ * @param {Object} params
+ * @param {string} params.token - GitHub access token
+ * @param {string} params.owner - Repository owner
+ * @param {string} params.repo  - Repository name
+ */
+async function ensureRoadmapLabels({ token, owner, repo }) {
+  const octokit = new Octokit({ auth: token });
+
+  const { data: existingLabels } = await octokit.rest.issues.listLabelsForRepo({
+    owner,
+    repo
+  });
+
+  const existingNames = new Set(existingLabels.map((label) => label.name));
+
+  for (const label of roadmapLabels) {
+    if (!existingNames.has(label.name)) {
+      await octokit.rest.issues.createLabel({ owner, repo, ...label });
+    }
+  }
+}
+
+// CLI support
+if (process.argv[1] === new URL(import.meta.url).pathname) {
+  const [owner, repo] = process.env.GITHUB_REPOSITORY?.split('/') || [];
+  const token = process.env.GITHUB_TOKEN;
+
+  if (!token || !owner || !repo) {
+    console.error('Missing required GITHUB_TOKEN or GITHUB_REPOSITORY');
+    process.exit(1);
+  }
+
+  ensureRoadmapLabels({ token, owner, repo })
+    .then(() => console.log('✅ Roadmap labels ensured'))
+    .catch((err) => {
+      console.error('Label sync failed:', err);
+      process.exit(1);
+    });
+}
+
+export { ensureRoadmapLabels, roadmapLabels };

package/scripts/label-roadmap-issues.js

@@ -0,0 +1,125 @@
+/**
+ * Version: 2.0.0
+ * Description: Auto-label roadmap issues and clean up outdated items
+ * Author: Ali Kahwaji
+ */
+
+import { Octokit } from 'octokit';
+import fs from 'fs/promises';
+import 'dotenv/config';
+
+const repo = process.env.GITHUB_REPO;
+const token = process.env.GITHUB_TOKEN;
+const ROADMAP_FILE = '.github/PROJECT_ROADMAP.md';
+const CLOSE_STALE = true;
+
+if (!repo || !token) {
+  console.error('Missing GITHUB_REPO or GITHUB_TOKEN env vars');
+  process.exit(1);
+}
+
+const [owner, repoName] = repo.split('/');
+const octokit = new Octokit({ auth: token });
+
+const PRIORITY_LABELS = {
+  High: 'priority: high',
+  Medium: 'priority: medium',
+  Low: 'priority: low',
+};
+
+const GROUP_LABELS = {
+  'Documentation': 'group: docs',
+  'Developer Experience': 'group: devx',
+  'Community': 'group: community',
+  'Blog & SEO': 'group: blog',
+  'Infrastructure': 'group: infra',
+};
+
+function parseRoadmapMarkdown(content) {
+  const lines = content.split('\n').filter(l => l.includes('|'));
+  const titles = new Set();
+
+  for (const line of lines) {
+    const parts = line.split('|').map(p => p.trim());
+    if (parts.length < 4 || parts[0] === 'Phase') continue;
+    const title = parts[3];
+    titles.add(title);
+  }
+
+  return titles;
+}
+
+async function run() {
+  const roadmapRaw = await fs.readFile(ROADMAP_FILE, 'utf8');
+  const roadmapTitles = parseRoadmapMarkdown(roadmapRaw);
+
+  const issues = await octokit.paginate(octokit.rest.issues.listForRepo, {
+    owner,
+    repo: repoName,
+    state: 'open',
+    per_page: 100,
+  });
+
+  for (const issue of issues) {
+    if (issue.pull_request) continue;
+
+    const title = issue.title.trim();
+    const body = issue.body || '';
+    const labels = new Set(issue.labels.map(l => typeof l === 'string' ? l : l.name));
+
+    const priority = Object.keys(PRIORITY_LABELS).find(p => body.includes(`Priority: ${p}`));
+    const group = Object.keys(GROUP_LABELS).find(g => body.includes(`Group: ${g}`));
+
+    const expectedLabels = [
+      ...(priority ? [PRIORITY_LABELS[priority]] : []),
+      ...(group ? [GROUP_LABELS[group]] : [])
+    ];
+
+    const currentRoadmap = roadmapTitles.has(title);
+
+    // Update Labels
+    const labelsToAdd = expectedLabels.filter(label => !labels.has(label));
+    const labelsToRemove = [...labels].filter(label =>
+      Object.values(PRIORITY_LABELS).includes(label) ||
+      Object.values(GROUP_LABELS).includes(label)
+    ).filter(label => !expectedLabels.includes(label));
+
+    if (labelsToAdd.length > 0 || labelsToRemove.length > 0) {
+      console.log(`🔁 Updating labels for issue #${issue.number}: ${title}`);
+      if (labelsToAdd.length) {
+        await octokit.rest.issues.addLabels({
+          owner,
+          repo: repoName,
+          issue_number: issue.number,
+          labels: labelsToAdd,
+        });
+      }
+      if (labelsToRemove.length) {
+        for (const label of labelsToRemove) {
+          await octokit.rest.issues.removeLabel({
+            owner,
+            repo: repoName,
+            issue_number: issue.number,
+            name: label,
+          });
+        }
+      }
+    }
+
+    // Optionally Close stale issues
+    if (CLOSE_STALE && !currentRoadmap) {
+      console.log(`🧹 Closing stale issue #${issue.number} (${title}) – not in roadmap`);
+      await octokit.rest.issues.update({
+        owner,
+        repo: repoName,
+        issue_number: issue.number,
+        state: 'closed',
+      });
+    }
+  }
+}
+
+run().catch(err => {
+  console.error('❌ Label script failed:', err);
+  process.exit(1);
+});

package/scripts/manual-test-checklist.md

@@ -0,0 +1,90 @@
+# ✅ Manual Test Checklist for RAG Pipeline Utils
+
+This document outlines manual test points for validating roadmap automation, documentation builds, CI/CD, blog sync, and changelog generation.
+
+---
+
+## 1. 🔄 Roadmap Sync + Label Automation
+
+| ✅ Task | Command or Trigger | Notes |
+|--------|--------------------|-------|
+| Sync `PROJECT_ROADMAP.md` to GitHub Issues | `git push` or `workflow_dispatch` for `sync-roadmap.yml` | Issues, assignees, labels should match |
+| Auto-create missing roadmap labels | `sync-roadmap.yml` step: `ensure-roadmap-labels.js` | Should not error even if labels exist |
+| Label updates reflect correct color & description | Rerun roadmap sync | Existing labels should be updated |
+| Roadmap issue closed on ✅ Done status | Modify `PROJECT_ROADMAP.md` status and sync | Corresponding issue should be auto-closed |
+
+---
+
+## 2. 🧭 CLI + Manual Utilities
+
+| ✅ Task | Command | Notes |
+|--------|---------|-------|
+| Create roadmap issues manually | `node scripts/create-roadmap-issues.js` | Creates GitHub Issues |
+| Ensure labels via CLI | `node scripts/ensure-roadmap-labels.js` | Must succeed with idempotency |
+| Generate changelog and preview | `npm run release:tag` | Prompt for tag + generate CHANGELOG |
+| Preview auto-generated blog post | Preview `.md` file in `/docs-site/blog` before push |
+
+---
+
+## 3. 📚 Docs + Deployment
+
+| ✅ Task | Command | Notes |
+|--------|---------|-------|
+| Build Docusaurus | `npm run docs:build` | No broken links, light/dark theme OK |
+| Deploy to GitHub Pages | `npm run docs:deploy` | Check rendered site at `/rag-pipeline-utils` |
+| Verify CHANGELOG link works | Test `/blog` page post link to `CHANGELOG.md` |
+
+---
+
+## 4. 🧪 CI/CD Validation
+
+| ✅ Task | Check | Notes |
+|--------|-------|-------|
+| CI Lint + Test runs | ✅ `ci.yml` | Lint passes, all Jest tests green |
+| Docs CI runs correctly | ✅ `docs-ci.yml` | No script missing error, build valid |
+| Docs auto-deploys on push | ✅ `docs-deploy.yml` | Publishes to GitHub Pages |
+
+---
+
+## 5. 🏷 GitHub Releases
+
+| ✅ Task | Trigger | Notes |
+|--------|---------|-------|
+| Generate blog post on release | Publish GitHub tag | `publish-release-blog.yml` should commit new `.md` |
+| Auto-fill blog content | From `CHANGELOG.md` and compare link | Author + commit stats populated |
+
+---
+
+## 6. 🛡 Release Gatekeeping & Metadata
+
+| ✅ Task | Outcome | Notes |
+|--------|---------|-------|
+| Badge renders in README | `![Release Review Passed]` | Should point to last workflow run |
+| Protected branch config for PRs | Status check on `release:review` | Prevents push without passing review |
+
+---
+
+## 7. 🧾 Filesystem and Cleanup
+
+| ✅ Task | Notes |
+|--------|-------|
+| `src/stories/` removed | No Storybook assets in GH Pages |
+| `postcss.config.cjs` & `tailwind.config.js` removed | Confirm no broken styling or build errors |
+
+---
+
+## 🔁 Optional Manual Commands
+
+```bash
+# Sync roadmap manually
+node scripts/create-roadmap-issues.js
+
+# Ensure roadmap labels
+node scripts/ensure-roadmap-labels.js
+
+# Trigger changelog + blog post flow
+npm run release:tag
+
+# Build and deploy docs
+npm run docs:build && npm run docs:deploy
+```