@devilsdev/rag-pipeline-utils 1.0.1 → 2.0.0

package/__tests__/unit/reranker/reranker.structured-output.test.js

@@ -0,0 +1,17 @@
+/**
+ * File: __tests__/unit/reranker/reranker.structured-output.test.js
+ * Version: 1.0.0
+ * Description: Validates structured LLM output parsing
+ * Author: @devilsdev
+ */
+import { LLMReranker } from '../../../src/reranker/llm-reranker.js';
+
+test('parses structured LLM JSON result', async () => {
+    const mockLLM = { async generate() {
+      return JSON.stringify({ ranking: [0, 2, 1] });
+    }};
+    const docs = [{ text: 'Alpha' }, { text: 'Beta' }, { text: 'Gamma' }];
+    const reranker = new LLMReranker({ llm: mockLLM });
+    const result = await reranker.rerank('Which is alpha?', docs);
+    expect(result[0].text).toBe('Alpha');
+  });

package/bin/cli.js

@@ -1,168 +1,62 @@
-#!/usr/bin/env node
-
 /**
- * Version: 0.1.8
- * Path: /bin/cli.js
- * Description: CLI interface for rag-pipeline utilities with loaders, reranker, and evaluation
+ * Version: 1.1.0
+ * Description: Final CLI entrypoint with dynamic plugin loading and full config validation
  * Author: Ali Kahwaji
  */
 
-import { Command } from 'commander';
-import { createRagPipeline, registry } from '../src/core/create-pipeline.js';
+import { resolve } from 'path';
+import { fileURLToPath } from 'url';
 import { loadRagConfig } from '../src/config/load-config.js';
-import { logger } from '../src/utils/logger.js';
-import { evaluateRagDataset } from '../src/evaluate/evaluator.js';
-import { LLMReranker } from '../src/reranker/llm-reranker.js';
+import { loadPluginsFromJson } from '../src/config/load-plugin-config.js';
+import registry from '../src/core/plugin-registry.js';
 
-import { MarkdownLoader } from '../src/loader/markdown-loader.js';
-import { HTMLLoader } from '../src/loader/html-loader.js';
-import { CSVLoader } from '../src/loader/csv-loader.js';
-import { DirectoryLoader } from '../src/loader/directory-loader.js';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = resolve(__filename, '..');
+const CONFIG_PATH = resolve('.ragrc.json');
 
-// Minimal test plugin implementations
-class PDFLoader {
-  async load(filePath) {
-    return [{ chunk: () => [`Test loaded: ${filePath}`] }];
-  }
-}
-class OpenAIEmbedder {
-  async embed(chunks) {
-    return chunks.map((text, i) => ({ id: `v${i}`, values: [0.1, 0.2, 0.3], metadata: { text } }));
-  }
-  async embedQuery(prompt) {
-    return [0.1, 0.2, 0.3];
-  }
-}
-class PineconeRetriever {
-  async store() {}
-  async retrieve() {
-    return [
-      { id: 'a', text: 'Chunk about pine trees', metadata: {} },
-      { id: 'b', text: 'Chunk about vectors', metadata: {} },
-      { id: 'c', text: 'Chunk about databases', metadata: {} }
-    ];
-  }
-}
-class OpenAILLM {
-  async generate(prompt, context) {
-    return `Generated answer using: ${context.map(d => d.text).join(', ')}`;
-  }
-}
+const args = process.argv.slice(2);
+const command = args[0];
 
-// Register mock/default plugins
-registry.register('loader', 'pdf', new PDFLoader());
-registry.register('loader', 'markdown', new MarkdownLoader());
-registry.register('loader', 'html', new HTMLLoader());
-registry.register('loader', 'csv', new CSVLoader());
-registry.register('loader', 'directory', new DirectoryLoader());
-registry.register('embedder', 'openai', new OpenAIEmbedder());
-registry.register('retriever', 'pinecone', new PineconeRetriever());
-registry.register('llm', 'openai-gpt-4', new OpenAILLM());
-
-const program = new Command();
-program
-  .name('rag-pipeline')
-  .description('CLI for RAG pipeline utilities')
-  .version('0.1.8');
-
-function resolveConfig(cliOpts) {
+async function runCLI() {
   try {
-    const fileConfig = loadRagConfig();
-    return { ...fileConfig, ...cliOpts };
+    const config = loadRagConfig(CONFIG_PATH);
+    await loadPluginsFromJson(CONFIG_PATH);
+
+    switch (command) {
+      case 'ingest': {
+        const input = args[1];
+        const loader = registry.get('loader', Object.keys(config.loader)[0]);
+        const embedder = registry.get('embedder', Object.keys(config.embedder)[0]);
+        const retriever = registry.get('retriever', Object.keys(config.retriever)[0]);
+
+        const docs = await loader.load(input);
+        const vectors = await embedder.embed(docs);
+        await retriever.store(vectors, config.namespace);
+
+        console.log('Ingestion complete');
+        break;
+      }
+      case 'query': {
+        const question = args.slice(1).join(' ');
+        const embedder = registry.get('embedder', Object.keys(config.embedder)[0]);
+        const retriever = registry.get('retriever', Object.keys(config.retriever)[0]);
+        const llm = registry.get('llm', Object.keys(config.llm)[0]);
+
+        const queryVec = await embedder.embedQuery(question);
+        const docs = await retriever.search(queryVec, config.namespace);
+        const answer = await llm.ask(question, docs);
+
+        console.log('Answer:', answer);
+        break;
+      }
+      default:
+        console.error('Unknown command:', command);
+        process.exit(1);
+    }
   } catch (err) {
-    logger.warn({ error: err.message }, 'Config file fallback failed');
-    return cliOpts;
+    console.error('Error during execution:', err.message);
+    process.exit(1);
   }
 }
 
-program
-  .command('ingest')
-  .argument('<path>', 'Path to document(s)')
-  .option('--loader <type>', 'Document loader type')
-  .option('--embedder <type>', 'Embedder type')
-  .option('--retriever <type>', 'Vector store retriever type')
-  .action(async (docPath, opts) => {
-    try {
-      const config = resolveConfig(opts);
-      const pipeline = createRagPipeline(config);
-      await pipeline.ingest(docPath);
-      console.log('Ingestion complete'); // required by test assertion
-    } catch (err) {
-      logger.error({ error: err.message }, 'Ingestion failed');
-      console.error('Error during ingestion:', err.message);
-      process.exit(1);
-    }
-  });
-
-program
-  .command('query')
-  .argument('<prompt>', 'Prompt to submit')
-  .option('--embedder <type>', 'Embedder type')
-  .option('--retriever <type>', 'Retriever type')
-  .option('--llm <type>', 'LLM type')
-  .action(async (prompt, opts) => {
-    try {
-      const config = resolveConfig(opts);
-      const pipeline = createRagPipeline(config);
-      const answer = await pipeline.query(prompt);
-      console.log('\n Answer:\n', answer);
-    } catch (err) {
-      logger.error({ error: err.message }, 'Query failed');
-      process.exit(1);
-    }
-  });
-
-program
-  .command('evaluate')
-  .argument('<dataset>', 'Path to JSON file of evaluation cases')
-  .option('--embedder <type>', 'Embedder type')
-  .option('--retriever <type>', 'Retriever type')
-  .option('--llm <type>', 'LLM type')
-  .action(async (dataset, opts) => {
-    try {
-      const config = resolveConfig(opts);
-      const results = await evaluateRagDataset(dataset, config);
-      const passRate = results.filter(r => r.success).length / results.length;
-      const avgBLEU = results.reduce((sum, r) => sum + r.scores.bleu, 0) / results.length;
-      const avgROUGE = results.reduce((sum, r) => sum + r.scores.rouge, 0) / results.length;
-
-      results.forEach((r, i) => {
-        console.log(`\nCase ${i + 1}:\nPrompt: ${r.prompt}\nPass: ${r.success}\nBLEU: ${r.scores.bleu.toFixed(2)}\nROUGE: ${r.scores.rouge.toFixed(2)}`);
-      });
-
-      console.log(`\n Summary:\nPassed: ${Math.round(passRate * 100)}%\nAvg BLEU: ${avgBLEU.toFixed(2)}\nAvg ROUGE: ${avgROUGE.toFixed(2)}`);
-    } catch (err) {
-      logger.error({ error: err.message }, 'Evaluation failed');
-      process.exit(1);
-    }
-  });
-
-program
-  .command('rerank')
-  .argument('<prompt>', 'Prompt to rerank context for')
-  .option('--retriever <type>', 'Retriever type')
-  .option('--llm <type>', 'LLM type')
-  .action(async (prompt, opts) => {
-    try {
-      const config = resolveConfig(opts);
-      const retriever = registry.get('retriever', config.retriever);
-      const llm = registry.get('llm', config.llm);
-      const embedder = registry.get('embedder', config.embedder);
-
-      const queryVector = await embedder.embedQuery(prompt);
-      const docs = await retriever.retrieve(queryVector);
-
-      const reranker = new LLMReranker({ llm });
-      const reranked = await reranker.rerank(prompt, docs);
-
-      console.log('\n Reranked Results:');
-      reranked.forEach((doc, i) => {
-        console.log(`\n#${i + 1}: ${doc.text}`);
-      });
-    } catch (err) {
-      logger.error({ error: err.message }, 'Rerank failed');
-      process.exit(1);
-    }
-  });
-
-program.parse();
+await runCLI();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devilsdev/rag-pipeline-utils",
-  "version": "1.0.1",
+  "version": "2.0.0",
   "description": "A modular toolkit for building RAG (Retrieval-Augmented Generation) pipelines in Node.js",
   "main": "src/core/create-pipeline.js",
   "bin": {
@@ -8,10 +8,22 @@
   },
   "type": "module",
   "scripts": {
+    "validate-fixtures": "node ./scripts/verify-fixtures.js",
+    "setup": "node ./scripts/setup.js",
     "test": "node ./scripts/setup.js && cross-env NODE_OPTIONS=--experimental-vm-modules jest --coverage",
     "test:watch": "jest --watch",
     "lint": "eslint .",
-    "prepare": "husky install"
+    "prepare": "husky install",
+    "lint-staged": "lint-staged",
+    "postinstall": "husky install",
+    "verify:mocks": "node scripts/verify-fixture-mocks.js",
+    "ci": "node scripts/ci-runner.js"
+  },
+  "lint-staged": {
+    "**/*.{js,ts}": [
+      "npm run lint",
+      "npm run validate-fixtures"
+    ]
   },
   "repository": {
     "type": "git",
@@ -43,17 +55,20 @@
     "@semantic-release/npm": "^11.0.3",
     "@semantic-release/release-notes-generator": "^14.0.3",
     "@types/jest": "^29.5.14",
+    "codecov": "^3.8.2",
     "cross-env": "^7.0.3",
     "esbuild": "^0.25.2",
     "eslint": "^8.57.1",
     "husky": "^8.0.3",
     "jest": "^29.7.0",
+    "lint-staged": "^15.5.1",
     "prettier": "^3.5.3",
     "rimraf": "^6.0.1",
     "semantic-release": "^21.1.2",
     "ts-jest": "^29.3.2"
   },
   "dependencies": {
+    "ajv": "^8.17.1",
     "axios": "^1.8.4",
     "chalk": "^5.4.1",
     "commander": "^13.1.0",

package/plugin.config.json

@@ -0,0 +1,14 @@
+{
+  "loader": {
+    "pdf": "./src/mocks/pdf-loader.js"
+  },
+  "embedder": {
+    "openai": "./src/mocks/openai-embedder.js"
+  },
+  "retriever": {
+    "pinecone": "./src/mocks/pinecone-retriever.js"
+  },
+  "llm": {
+    "openai-gpt-4": "./src/mocks/openai-llm.js"
+  }
+}

package/scripts/banner-injector.js

@@ -0,0 +1,89 @@
+/**
+ * File: scripts/banner-injector.js
+ * Description: Script to programmatically inject file-level banners into specified project files
+ * Author: Ali Kahwaji
+ * Version: 1.0.0
+ */
+
+import fs from 'fs';
+import path from 'path';
+import glob from 'glob';
+
+const banners = {
+  '__tests__/unit/reranker/llm-reranker.test.js': `/**
+ * File: __tests__/unit/reranker/llm-reranker.test.js
+ * Description: Unit and integration tests with mocks for RAG pipeline
+ * Author: Ali Kahwaji
+ * Version: 1.0.4
+ */\n\n`,
+
+  '__tests__/unit/reranker/reranker.snapshot.test.js': `/**
+ * File: __tests__/unit/reranker/reranker.snapshot.test.js
+ * Description: Snapshot tests for LLM reranker
+ * Author: Ali Kahwaji
+ * Version: 1.0.4
+ */\n\n`,
+
+  '__tests__/unit/core/plugin-registry.test.js': `/**
+ * File: __tests__/unit/core/plugin-registry.test.js
+ * Description: Unit and integration tests with mocks for RAG pipeline
+ * Author: Ali Kahwaji
+ * Version: 1.0.4
+ */\n\n`,
+
+  '__tests__/integration/config/load-config.test.js': `/**
+ * File: __tests__/integration/config/load-config.test.js
+ * Description: Unit and integration tests with mocks for RAG pipeline
+ * Author: Ali Kahwaji
+ * Version: 1.0.4
+ */\n\n`,
+
+  '__tests__/integration/cli/config-flow.test.js': `/**
+ * File: __tests__/integration/cli/config-flow.test.js
+ * Description: Unit and integration tests with mocks for RAG pipeline
+ * Author: Ali Kahwaji
+ * Version: 1.0.4
+ */\n\n`,
+
+  'src/mocks/pdf-loader.js': `/**
+ * File: src/mocks/pdf-loader.js
+ * Description: Unit and integration tests with mocks for RAG pipeline
+ * Author: Ali Kahwaji
+ * Version: 1.0.4
+ */\n\n`,
+
+  'src/mocks/openai-embedder.js': `/**
+ * File: src/mocks/openai-embedder.js
+ * Description: Unit and integration tests with mocks for RAG pipeline
+ * Author: Ali Kahwaji
+ * Version: 1.0.4
+ */\n\n`,
+
+  'src/mocks/pinecone-retriever.js': `/**
+ * File: src/mocks/pinecone-retriever.js
+ * Description: Unit and integration tests with mocks for RAG pipeline
+ * Author: Ali Kahwaji
+ * Version: 1.0.4
+ */\n\n`
+};
+
+Object.entries(banners).forEach(([filePath, banner]) => {
+  const resolvedPath = path.resolve(filePath);
+
+  if (!fs.existsSync(resolvedPath)) {
+    console.warn(`[SKIPPED] ${filePath} does not exist.`);
+    return;
+  }
+
+  const originalContent = fs.readFileSync(resolvedPath, 'utf-8');
+
+  // Skip if banner already present
+  if (originalContent.startsWith('/**')) {
+    console.log(`[SKIPPED] ${filePath} already contains a banner.`);
+    return;
+  }
+
+  const newContent = banner + originalContent;
+  fs.writeFileSync(resolvedPath, newContent, 'utf-8');
+  console.log(`[INJECTED] Banner added to ${filePath}`);
+});

package/scripts/ci-runner.js

@@ -0,0 +1,51 @@
+/**
+ * Version: 1.1.0
+ * Description: CI runner for linting, mock validation, testing and coverage
+ * Author: Ali Kahwaji
+ * File: scripts/ci-runner.js
+ */
+
+import { execSync } from 'child_process';
+import path from 'path';
+import { pathToFileURL } from 'url';
+import fs from 'fs';
+
+// Entry helpers
+const ROOT = path.resolve();
+const resolveScript = (relPath) => pathToFileURL(path.resolve(ROOT, relPath)).href;
+
+const run = (label, command) => {
+  console.log(`\n🔧 Running: ${label}`);
+  try {
+    execSync(command, { stdio: 'inherit', cwd: ROOT });
+    console.log(`✅ ${label} passed`);
+  } catch (err) {
+    console.error(`❌ ${label} failed`);
+    process.exit(1);
+  }
+};
+
+const runDynamicModule = async (label, modulePath) => {
+  console.log(`\n🔍 Validating: ${label}`);
+  try {
+    const mod = await import(resolveScript(modulePath));
+    if (typeof mod.default === 'function') await mod.default();
+    console.log(`✅ ${label} passed`);
+  } catch (err) {
+    console.error(`❌ ${label} failed:\n${err.message}`);
+    process.exit(1);
+  }
+};
+
+(async () => {
+  // Step 1: Lint
+  run('Lint', 'npx eslint . --ext .js,.ts');
+
+  // Step 2: Verify mocks for required interface methods
+  await runDynamicModule('Fixture Mock Interface Validation', 'scripts/verify-fixture-mocks.js');
+
+  // Step 3: Test and coverage
+  run('Unit + Integration Tests w/ Coverage', 'cross-env NODE_OPTIONS=--experimental-vm-modules jest --coverage');
+
+  console.log('\n🎉 CI runner completed successfully.\n');
+})();

package/scripts/repair-fixtures.js

@@ -0,0 +1,30 @@
+/**
+ * Version: 1.0.0
+ * Description: Auto-repairs mock plugin fixtures for integration tests
+ * Author: Ali Kahwaji
+ */
+
+import { mkdirSync, copyFileSync, existsSync } from 'fs';
+import { resolve, join } from 'path';
+
+const MOCKS = [
+  'pdf-loader.js',
+  'openai-embedder.js',
+  'pinecone-retriever.js',
+  'openai-llm.js'
+];
+
+const SRC_FIXTURES = resolve('__tests__/fixtures/src/mocks');
+const DEST_FIXTURES = resolve('__tests__/__temp__/cli-config-fallback/src/mocks');
+
+mkdirSync(DEST_FIXTURES, { recursive: true });
+
+MOCKS.forEach((file) => {
+  const src = join(SRC_FIXTURES, file);
+  const dest = join(DEST_FIXTURES, file);
+  if (!existsSync(src)) throw new Error(`[FIXTURE MISSING] ${src}`);
+  copyFileSync(src, dest);
+  console.log(`[COPIED] ${file}`);
+});
+
+console.log('✅ Fixture mocks successfully repaired.');

package/scripts/setup.js

@@ -1,7 +1,31 @@
+/**
+ * Version: 1.1.0
+ * Description: Test setup script to ensure fixtures exist and validate mocks before running tests.
+ * Author: Ali Kahwaji
+ */
+
 import fs from 'fs';
-const fixturePath = './__tests__/fixtures/sample.pdf';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import { execSync } from 'child_process';
+
+// Resolve __dirname in ES module scope
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const fixturePath = path.resolve(__dirname, '../__tests__/fixtures/sample.pdf');
 
+// Ensure PDF test fixture
 if (!fs.existsSync(fixturePath)) {
-  fs.mkdirSync('./__tests__/fixtures', { recursive: true });
+  fs.mkdirSync(path.dirname(fixturePath), { recursive: true });
   fs.writeFileSync(fixturePath, 'Dummy PDF content for test');
+  console.log('[setup] Created sample.pdf fixture');
+}
+
+// Validate plugin fixture presence
+try {
+  execSync('node ./scripts/verify-fixtures.js', { stdio: 'inherit' });
+} catch (err) {
+  console.error('[setup] Fixture verification failed. Please check missing mocks.');
+  process.exit(1);
 }

package/scripts/verify-fixture-mocks.js

@@ -0,0 +1,45 @@
+/**
+ * Version: 1.0.0
+ * Description: Verifies that plugin mock implementations expose required methods
+ * Author: Ali Kahwaji
+ */
+
+import { readdirSync, statSync } from 'fs';
+import { resolve, join, extname } from 'path';
+import { pathToFileURL } from 'url';
+
+const MOCKS_DIR = resolve('__tests__/fixtures/src/mocks');
+
+const pluginContracts = {
+  'openai-embedder.js': ['embed', 'embedQuery'],
+  'openai-llm.js': ['ask'],
+  'pdf-loader.js': ['load'],
+  'pinecone-retriever.js': ['store', 'search'],
+};
+
+const failures = [];
+
+for (const file of readdirSync(MOCKS_DIR)) {
+  const fullPath = join(MOCKS_DIR, file);
+  const isFile = statSync(fullPath).isFile();
+  if (!isFile || extname(file) !== '.js') continue;
+
+  const expectedMethods = pluginContracts[file];
+  if (!expectedMethods) continue;
+
+  const mod = await import(pathToFileURL(fullPath).href);
+  const plugin = new mod.default();
+
+  for (const method of expectedMethods) {
+    if (typeof plugin[method] !== 'function') {
+      failures.push(`❌ ${file} failed: [${file}] missing method: ${method}`);
+    }
+  }
+}
+
+if (failures.length > 0) {
+  failures.forEach(msg => console.error(msg));
+  process.exit(1);
+} else {
+  console.log('✅ All plugin mocks verified.');
+}

package/scripts/verify-fixtures.js

@@ -0,0 +1,36 @@
+/**
+ * Version: 1.0.0
+ * Description: Script to verify presence of all expected fixture plugins before running integration tests.
+ * Author: Ali Kahwaji
+ * File: scripts/verify-fixtures.js
+ */
+
+import { existsSync } from 'fs';
+import { resolve } from 'path';
+
+const REQUIRED_FIXTURES = [
+  '__tests__/fixtures/sample.pdf',
+  '__tests__/fixtures/.ragrc.json',
+  '__tests__/fixtures/src/mocks/pdf-loader.js',
+  '__tests__/fixtures/src/mocks/openai-embedder.js',
+  '__tests__/fixtures/src/mocks/pinecone-retriever.js',
+  '__tests__/fixtures/src/mocks/openai-llm.js'
+];
+
+let missing = [];
+for (const file of REQUIRED_FIXTURES) {
+  const fullPath = resolve(file);
+  if (!existsSync(fullPath)) {
+    missing.push(fullPath);
+  }
+}
+
+if (missing.length) {
+  console.error('[ERROR] Missing test fixture files:');
+  for (const f of missing) {
+    console.error(' -', f);
+  }
+  process.exit(1);
+} else {
+  console.log('[OK] All fixture files are present.');
+}

package/src/config/load-config.js

@@ -1,35 +1,33 @@
 /**
- * Version: 0.1.0
- * Path: /src/config/load-config.js
- * Description: Loader and validator for `.ragrc.json` configuration files
+ * Version: 1.1.0
+ * Description: Loads validated .ragrc.json config from disk
  * Author: Ali Kahwaji
  */
 
 import fs from 'fs';
 import path from 'path';
-import { validate } from './validate-schema.js';
-
-const DEFAULT_CONFIG_FILE = '.ragrc.json';
+import { validateRagrcSchema } from './validate-schema.js';
+import { logger } from '../utils/logger.js';
 
 /**
- * Load and validate the RAG configuration from a file.
- *
- * @param {string} [configPath] - Optional custom config path
- * @returns {object} - Validated config object
- * @throws {Error} - If file not found or validation fails
+ * Loads and validates a RAG config file.
+ * @param {string} configPath - Absolute path to .ragrc.json
+ * @returns {object} Parsed config object
+ * @throws {Error} If file is missing or invalid
  */
-export function loadRagConfig(configPath = DEFAULT_CONFIG_FILE) {
-  const fullPath = path.resolve(process.cwd(), configPath);
-
-  if (!fs.existsSync(fullPath)) {
-    throw new Error(`Config file not found: ${fullPath}`);
+export function loadRagConfig(configPath) {
+  if (!fs.existsSync(configPath)) {
+    throw new Error(`Config file not found: ${configPath}`);
   }
 
-  const raw = fs.readFileSync(fullPath, 'utf-8');
-  const config = JSON.parse(raw);
+  const raw = fs.readFileSync(configPath, 'utf-8');
+  const parsed = JSON.parse(raw);
 
-  validate(config); // Throws if invalid
+  const result = validateRagrcSchema(parsed);
+  if (!result.valid) {
+    logger.error({ errors: result.errors }, 'Invalid .ragrc.json schema');
+    throw new Error('Config validation failed');
+  }
 
-  return config;
+  return parsed;
 }
-