@devilsdev/rag-pipeline-utils 1.0.0 → 2.0.0

package/__tests__/unit/config/validate-schema.test.js

@@ -0,0 +1,121 @@
+/**
+ * Version: 1.1.0
+ * Description: Schema validation tests for .ragrc.json using AJV and external fixtures
+ * Author: Ali Kahwaji
+ * File: __tests__/unit/config/validate-schema.test.js
+ */
+
+import { readFileSync } from 'fs';
+import { resolve } from 'path';
+import { validateRagrcSchema, validatePluginSchema } from '../../../src/config/validate-schema.js';
+
+const readFixture = (name) => {
+  const filePath = resolve('__tests__/fixtures',name.replace('.ragrc', '-ragrc'));
+  return JSON.parse(readFileSync(filePath, 'utf-8'));
+};
+
+describe('validateRagrcSchema', () => {
+  test('passes with valid structured plugin schema', () => {
+    const config = readFixture('valid-ragrc.json');
+    const result = validateRagrcSchema(config);
+    expect(result.valid).toBe(true);
+    expect(result.errors).toBeUndefined(); // updated: null => undefined
+  });
+
+  test('fails with missing pipeline', () => {
+    const config = {
+      loader: { pdf: './src/mocks/pdf-loader.js' },
+      namespace: 'no-pipeline'
+    };
+    const result = validateRagrcSchema(config);
+    expect(result.valid).toBe(false);
+    expect(result.errors).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          keyword: 'required',
+          params: expect.objectContaining({ missingProperty: 'pipeline' })
+        })
+      ])
+    );
+  });
+  
+  test('fails with non-object plugin sections', () => {
+    const config = {
+      loader: './src/invalid.js', // should be object
+      embedder: { openai: './src/mocks/openai-embedder.js' },
+      retriever: { pinecone: './src/mocks/pinecone-retriever.js' },
+      llm: { openai: './src/mocks/openai-llm.js' },
+      pipeline: ['loader'],
+      namespace: 'invalid-section'
+    };
+    const result = validateRagrcSchema(config);
+    expect(result.valid).toBe(false);
+    expect(result.errors).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ instancePath: '/loader' })
+      ])
+    );
+  });
+
+  test('fails with non-string namespace', () => {
+    const config = {
+      loader: { pdf: './src/mocks/pdf-loader.js' },
+      embedder: { openai: './src/mocks/openai-embedder.js' },
+      retriever: { pinecone: './src/mocks/pinecone-retriever.js' },
+      llm: { openai: './src/mocks/openai-llm.js' },
+      pipeline: ['loader'],
+      namespace: 12345
+    };
+    const result = validateRagrcSchema(config);
+    expect(result.valid).toBe(false);
+    expect(result.errors).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ instancePath: '/namespace' })
+      ])
+    );
+  });
+});
+
+describe('validatePluginSchema (subset)', () => {
+  test('passes with plugin-only valid structure', () => {
+    const config = {
+      loader: { pdf: './src/mocks/pdf-loader.js' },
+      embedder: { openai: './src/mocks/openai-embedder.js' },
+      retriever: { pinecone: './src/mocks/pinecone-retriever.js' },
+      llm: { openai: './src/mocks/openai-llm.js' }
+    };
+    const result = validatePluginSchema(config);
+    expect(result.valid).toBe(true);
+  });
+
+  test('fails when plugin section is missing', () => {
+    const config = {
+      embedder: { openai: './mock.js' },
+      retriever: { pinecone: './mock.js' },
+      llm: { openai: './mock.js' }
+    };
+    const result = validatePluginSchema(config);
+    expect(result.valid).toBe(false);
+    expect(result.errors).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ instancePath: '', message: expect.stringContaining('must have required property') })
+      ])
+    );
+  });
+
+  test('fails when plugin paths are not strings', () => {
+    const config = {
+      loader: { pdf: 12345 },
+      embedder: { openai: './mock.js' },
+      retriever: { pinecone: './mock.js' },
+      llm: { openai: './mock.js' }
+    };
+    const result = validatePluginSchema(config);
+    expect(result.valid).toBe(false);
+    expect(result.errors).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ instancePath: '/loader/pdf' })
+      ])
+    );
+  });
+});

package/__tests__/unit/core/plugin-registry.test.js

@@ -1,55 +1,55 @@
-/**
- * Version: 0.1.0
- * Path: /__tests__/unit/core/plugin-registry.test.js
- * Description: Unit tests for the PluginRegistry class
- * Author: Ali Kahwaji
- */
-
-import { PluginRegistry } from '../../../src/core/plugin-registry.js';
-
-describe('PluginRegistry', () => {
-  let registry;
-
-  beforeEach(() => {
-    registry = new PluginRegistry();
-  });
-
-  test('registers and retrieves a plugin successfully', () => {
-    const mockPlugin = { name: 'test-loader' };
-    registry.register('loader', 'test', mockPlugin);
-    const result = registry.get('loader', 'test');
-    expect(result).toBe(mockPlugin);
-  });
-
-  test('overwrites existing plugin under the same name', () => {
-    const firstPlugin = { name: 'first' };
-    const secondPlugin = { name: 'second' };
-    registry.register('embedder', 'dup', firstPlugin);
-    registry.register('embedder', 'dup', secondPlugin);
-    const result = registry.get('embedder', 'dup');
-    expect(result).toBe(secondPlugin);
-  });
-
-  test('throws error for unknown plugin type on registration', () => {
-    expect(() => registry.register('unknown-type', 'x', {})).toThrow(/Unknown plugin type/);
-  });
-
-  test('throws error when retrieving a nonexistent plugin', () => {
-    expect(() => registry.get('retriever', 'missing')).toThrow(/Plugin not found/);
-  });
-
-  test('lists all registered plugin names for a type', () => {
-    registry.register('llm', 'gpt-4', {});
-    registry.register('llm', 'gpt-3.5', {});
-    const list = registry.list('llm');
-    expect(list).toContain('gpt-4');
-    expect(list).toContain('gpt-3.5');
-    expect(list.length).toBe(2);
-  });
-
-  test('throws error when listing plugins for invalid type', () => {
-    expect(() => registry.list('invalid')).toThrow(/Unknown plugin type/);
-  });
-});
-
-
+/**
+ * Version: 0.1.0
+ * Path: /__tests__/unit/core/plugin-registry.test.js
+ * Description: Unit tests for the PluginRegistry class
+ * Author: Ali Kahwaji
+ */
+
+import { PluginRegistry } from '../../../src/core/plugin-registry.js';
+
+describe('PluginRegistry', () => {
+  let registry;
+
+  beforeEach(() => {
+    registry = new PluginRegistry();
+  });
+
+  test('registers and retrieves a plugin successfully', () => {
+    const mockPlugin = { name: 'test-loader' };
+    registry.register('loader', 'test', mockPlugin);
+    const result = registry.get('loader', 'test');
+    expect(result).toBe(mockPlugin);
+  });
+
+  test('overwrites existing plugin under the same name', () => {
+    const firstPlugin = { name: 'first' };
+    const secondPlugin = { name: 'second' };
+    registry.register('embedder', 'dup', firstPlugin);
+    registry.register('embedder', 'dup', secondPlugin);
+    const result = registry.get('embedder', 'dup');
+    expect(result).toBe(secondPlugin);
+  });
+
+  test('throws error for unknown plugin type on registration', () => {
+    expect(() => registry.register('unknown-type', 'x', {})).toThrow(/Unknown plugin type/);
+  });
+
+  test('throws error when retrieving a nonexistent plugin', () => {
+    expect(() => registry.get('retriever', 'missing')).toThrow(/Plugin not found/);
+  });
+
+  test('lists all registered plugin names for a type', () => {
+    registry.register('llm', 'gpt-4', {});
+    registry.register('llm', 'gpt-3.5', {});
+    const list = registry.list('llm');
+    expect(list).toContain('gpt-4');
+    expect(list).toContain('gpt-3.5');
+    expect(list.length).toBe(2);
+  });
+
+  test('throws error when listing plugins for invalid type', () => {
+    expect(() => registry.list('invalid')).toThrow(/Unknown plugin type/);
+  });
+});
+
+

package/__tests__/unit/reranker/llm-reranker.test.js

@@ -1,42 +1,18 @@
-/**
- * Version: 0.1.0
- * Path: /__tests__/unit/reranker/llm-reranker.test.js
- * Description: Unit tests for LLM-based context reranker
- * Author: Ali Kahwaji
- */
-
-import { LLMReranker } from '../../../src/reranker/llm-reranker.js';
-
-describe('LLMReranker', () => {
-  const mockDocs = [
-    { text: 'context about math' },
-    { text: 'context about vectors' },
-    { text: 'context about nothing' }
-  ];
-
-  test('returns reranked documents based on mock LLM JSON output', async () => {
-    const mockLLM = {
-      async generate() {
-        return '[1, 0, 2]';
-      }
-    };
-    const reranker = new LLMReranker({ llm: mockLLM });
-    const result = await reranker.rerank('What are vectors?', mockDocs);
-    expect(result.map(d => d.text)).toEqual([
-      'context about vectors',
-      'context about math',
-      'context about nothing'
-    ]);
-  });
-
-  test('gracefully falls back if LLM returns invalid JSON', async () => {
-    const mockLLM = {
-      async generate() {
-        return 'this is not JSON';
-      }
-    };
-    const reranker = new LLMReranker({ llm: mockLLM });
-    const result = await reranker.rerank('Explain.', mockDocs);
-    expect(result.length).toBe(0);
-  });
-});
+/**
+ * File: __tests__/unit/reranker/llm-reranker.test.js
+ * Version: 1.0.0
+ * Description: Validates core behavior of the LLMReranker module
+ * Author: @devilsdev
+ */
+
+import { LLMReranker } from '../../../src/reranker/llm-reranker.js';
+
+describe('LLMReranker', () => {
+  test('returns reranked documents based on mock LLM JSON output', async () => {
+    const mockLLM = { async generate() { return '[1, 0, 2]'; } };
+    const docs = [{ text: 'A' }, { text: 'B' }, { text: 'C' }];
+    const reranker = new LLMReranker({ llm: mockLLM });
+    const result = await reranker.rerank('What is A?', docs);
+    expect(result.map(d => d.text)).toEqual(['B', 'A', 'C']);
+  });
+});

package/__tests__/unit/reranker/reranker.enriched.test.js

@@ -0,0 +1,19 @@
+/**
+ * File: __tests__/unit/reranker/reranker.enriched.test.js
+ * Version: 1.0.0
+ * Description: Tests support for extended document structures
+ * Author: @devilsdev
+ */
+import { LLMReranker } from '../../../src/reranker/llm-reranker.js';
+
+test('supports documents with metadata', async () => {
+    const mockLLM = { async generate() { return '[2, 0, 1]'; } };
+    const docs = [
+      { text: 'One', source: 'alpha' },
+      { text: 'Two', source: 'beta' },
+      { text: 'Three', source: 'gamma' }
+    ];
+    const reranker = new LLMReranker({ llm: mockLLM });
+    const result = await reranker.rerank('Choose the most complete', docs);
+    expect(result[0]).toHaveProperty('source', 'gamma');
+  });

package/__tests__/unit/reranker/reranker.fallback.test.js

@@ -0,0 +1,15 @@
+/**
+ * File: __tests__/unit/reranker/reranker.fallback.test.js
+ * Version: 1.0.0
+ * Description: Validates fallback handling for invalid LLM output
+ * Author: @devilsdev
+ */
+import { LLMReranker } from '../../../src/reranker/llm-reranker.js';
+
+test('gracefully handles invalid LLM output', async () => {
+    const mockLLM = { async generate() { return 'INVALID_JSON'; } };
+    const docs = [{ text: 'X' }, { text: 'Y' }, { text: 'Z' }];
+    const reranker = new LLMReranker({ llm: mockLLM });
+    const result = await reranker.rerank('Prompt?', docs);
+    expect(result.map(d => d.text)).toEqual(['X', 'Y', 'Z']);
+  });

package/__tests__/unit/reranker/reranker.snapshot.test.js

@@ -1,17 +1,15 @@
-import { LLMReranker } from '../../../src/reranker/llm-reranker.js';
-
-test('LLMReranker output snapshot', async () => {
-  const mockLLM = {
-    async generate() {
-      return '[1, 0, 2]';
-    }
-  };
-  const mockDocs = [
-    { text: 'A' },
-    { text: 'B' },
-    { text: 'C' }
-  ];
-  const reranker = new LLMReranker({ llm: mockLLM });
-  const result = await reranker.rerank('What is A?', mockDocs);
-  expect(result).toMatchSnapshot();
-});
+/**
+ * File: __tests__/unit/reranker/reranker.snapshot.test.js
+ * Version: 1.0.0
+ * Description: Captures a snapshot baseline of reranked results
+ * Author: @devilsdev
+ */
+import { LLMReranker } from '../../../src/reranker/llm-reranker.js';
+
+test('LLMReranker output snapshot', async () => {
+  const mockLLM = { async generate() { return '[1, 0, 2]'; } };
+  const mockDocs = [{ text: 'A' }, { text: 'B' }, { text: 'C' }];
+  const reranker = new LLMReranker({ llm: mockLLM });
+  const result = await reranker.rerank('What is A?', mockDocs);
+  expect(result).toMatchSnapshot();
+});

package/__tests__/unit/reranker/reranker.structured-output.test.js

@@ -0,0 +1,17 @@
+/**
+ * File: __tests__/unit/reranker/reranker.structured-output.test.js
+ * Version: 1.0.0
+ * Description: Validates structured LLM output parsing
+ * Author: @devilsdev
+ */
+import { LLMReranker } from '../../../src/reranker/llm-reranker.js';
+
+test('parses structured LLM JSON result', async () => {
+    const mockLLM = { async generate() {
+      return JSON.stringify({ ranking: [0, 2, 1] });
+    }};
+    const docs = [{ text: 'Alpha' }, { text: 'Beta' }, { text: 'Gamma' }];
+    const reranker = new LLMReranker({ llm: mockLLM });
+    const result = await reranker.rerank('Which is alpha?', docs);
+    expect(result[0].text).toBe('Alpha');
+  });

package/bin/cli.js

@@ -1,168 +1,62 @@
-#!/usr/bin/env node
-
-/**
- * Version: 0.1.7
- * Path: /bin/cli.js
- * Description: CLI interface for rag-pipeline utilities with loaders, reranker, and evaluation
- * Author: Ali Kahwaji
- */
-
-import { Command } from 'commander';
-import { createRagPipeline, registry } from '../src/core/create-pipeline.js';
-import { loadRagConfig } from '../src/config/load-config.js';
-import { logger } from '../src/utils/logger.js';
-import { evaluateRagDataset } from '../src/evaluate/evaluator.js';
-import { LLMReranker } from '../src/reranker/llm-reranker.js';
-
-import { MarkdownLoader } from '../src/loader/markdown-loader.js';
-import { HTMLLoader } from '../src/loader/html-loader.js';
-import { CSVLoader } from '../src/loader/csv-loader.js';
-import { DirectoryLoader } from '../src/loader/directory-loader.js';
-
-// Sample/mock plugin implementations
-class PDFLoader {
-  async load(path) {
-    return [{ chunk: () => ['Sample PDF chunk.'] }];
-  }
-}
-class OpenAIEmbedder {
-  async embed(chunks) {
-    return chunks.map((text, i) => ({ id: `v${i}`, values: [0.1, 0.2, 0.3], metadata: { text } }));
-  }
-  async embedQuery(prompt) {
-    return [0.1, 0.2, 0.3];
-  }
-}
-class PineconeRetriever {
-  async store(vectors) {}
-  async retrieve(vector) {
-    return [
-      { id: 'a', text: 'Chunk about pine trees', metadata: {} },
-      { id: 'b', text: 'Chunk about vectors', metadata: {} },
-      { id: 'c', text: 'Chunk about databases', metadata: {} }
-    ];
-  }
-}
-class OpenAILLM {
-  async generate(prompt, context) {
-    return `Generated answer using: ${context.map(d => d.text).join(', ')}`;
-  }
-}
-
-// Register plugins
-registry.register('loader', 'pdf', new PDFLoader());
-registry.register('loader', 'markdown', new MarkdownLoader());
-registry.register('loader', 'html', new HTMLLoader());
-registry.register('loader', 'csv', new CSVLoader());
-registry.register('loader', 'directory', new DirectoryLoader());
-registry.register('embedder', 'openai', new OpenAIEmbedder());
-registry.register('retriever', 'pinecone', new PineconeRetriever());
-registry.register('llm', 'openai-gpt-4', new OpenAILLM());
-
-const program = new Command();
-program
-  .name('rag-pipeline')
-  .description('CLI for RAG pipeline utilities')
-  .version('0.1.7');
-
-function resolveConfig(cliOpts) {
-  try {
-    const fileConfig = loadRagConfig();
-    return { ...fileConfig, ...cliOpts };
-  } catch (err) {
-    logger.warn({ error: err.message }, 'Config file fallback failed');
-    return cliOpts;
-  }
-}
-
-program
-  .command('ingest')
-  .argument('<path>', 'Path to document(s)')
-  .option('--loader <type>', 'Document loader type')
-  .option('--embedder <type>', 'Embedder type')
-  .option('--retriever <type>', 'Vector store retriever type')
-  .action(async (path, opts) => {
-    try {
-      const config = resolveConfig(opts);
-      const pipeline = createRagPipeline(config);
-      await pipeline.ingest(path);
-      logger.info('Ingestion complete');
-    } catch (err) {
-      logger.error({ error: err.message }, 'Ingestion failed');
-      process.exit(1);
-    }
-  });
-
-program
-  .command('query')
-  .argument('<prompt>', 'Prompt to submit')
-  .option('--embedder <type>', 'Embedder type')
-  .option('--retriever <type>', 'Retriever type')
-  .option('--llm <type>', 'LLM type')
-  .action(async (prompt, opts) => {
-    try {
-      const config = resolveConfig(opts);
-      const pipeline = createRagPipeline(config);
-      const answer = await pipeline.query(prompt);
-      logger.info({ answer }, 'Query complete');
-      console.log('\n Answer:\n', answer);
-    } catch (err) {
-      logger.error({ error: err.message }, 'Query failed');
-      process.exit(1);
-    }
-  });
-
-program
-  .command('evaluate')
-  .argument('<dataset>', 'Path to JSON file of evaluation cases')
-  .option('--embedder <type>', 'Embedder type')
-  .option('--retriever <type>', 'Retriever type')
-  .option('--llm <type>', 'LLM type')
-  .action(async (dataset, opts) => {
-    try {
-      const config = resolveConfig(opts);
-      const results = await evaluateRagDataset(dataset, config);
-      const passRate = results.filter(r => r.success).length / results.length;
-      const avgBLEU = results.reduce((sum, r) => sum + r.scores.bleu, 0) / results.length;
-      const avgROUGE = results.reduce((sum, r) => sum + r.scores.rouge, 0) / results.length;
-
-      results.forEach((r, i) => {
-        console.log(`\nCase ${i + 1}:\nPrompt: ${r.prompt}\nPass: ${r.success}\nBLEU: ${r.scores.bleu.toFixed(2)}\nROUGE: ${r.scores.rouge.toFixed(2)}`);
-      });
-
-      console.log(`\n Summary:\nPassed: ${Math.round(passRate * 100)}%\nAvg BLEU: ${avgBLEU.toFixed(2)}\nAvg ROUGE: ${avgROUGE.toFixed(2)}`);
-    } catch (err) {
-      logger.error({ error: err.message }, 'Evaluation failed');
-      process.exit(1);
-    }
-  });
-
-program
-  .command('rerank')
-  .argument('<prompt>', 'Prompt to rerank context for')
-  .option('--retriever <type>', 'Retriever type')
-  .option('--llm <type>', 'LLM type')
-  .action(async (prompt, opts) => {
-    try {
-      const config = resolveConfig(opts);
-      const retriever = registry.get('retriever', config.retriever);
-      const llm = registry.get('llm', config.llm);
-      const embedder = registry.get('embedder', config.embedder);
-
-      const queryVector = await embedder.embedQuery(prompt);
-      const docs = await retriever.retrieve(queryVector);
-
-      const reranker = new LLMReranker({ llm });
-      const reranked = await reranker.rerank(prompt, docs);
-
-      console.log('\n Reranked Results:');
-      reranked.forEach((doc, i) => {
-        console.log(`\n#${i + 1}: ${doc.text}`);
-      });
-    } catch (err) {
-      logger.error({ error: err.message }, 'Rerank failed');
-      process.exit(1);
-    }
-  });
-
-program.parse();
+/**
+ * Version: 1.1.0
+ * Description: Final CLI entrypoint with dynamic plugin loading and full config validation
+ * Author: Ali Kahwaji
+ */
+
+import { resolve } from 'path';
+import { fileURLToPath } from 'url';
+import { loadRagConfig } from '../src/config/load-config.js';
+import { loadPluginsFromJson } from '../src/config/load-plugin-config.js';
+import registry from '../src/core/plugin-registry.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = resolve(__filename, '..');
+const CONFIG_PATH = resolve('.ragrc.json');
+
+const args = process.argv.slice(2);
+const command = args[0];
+
+async function runCLI() {
+  try {
+    const config = loadRagConfig(CONFIG_PATH);
+    await loadPluginsFromJson(CONFIG_PATH);
+
+    switch (command) {
+      case 'ingest': {
+        const input = args[1];
+        const loader = registry.get('loader', Object.keys(config.loader)[0]);
+        const embedder = registry.get('embedder', Object.keys(config.embedder)[0]);
+        const retriever = registry.get('retriever', Object.keys(config.retriever)[0]);
+
+        const docs = await loader.load(input);
+        const vectors = await embedder.embed(docs);
+        await retriever.store(vectors, config.namespace);
+
+        console.log('Ingestion complete');
+        break;
+      }
+      case 'query': {
+        const question = args.slice(1).join(' ');
+        const embedder = registry.get('embedder', Object.keys(config.embedder)[0]);
+        const retriever = registry.get('retriever', Object.keys(config.retriever)[0]);
+        const llm = registry.get('llm', Object.keys(config.llm)[0]);
+
+        const queryVec = await embedder.embedQuery(question);
+        const docs = await retriever.search(queryVec, config.namespace);
+        const answer = await llm.ask(question, docs);
+
+        console.log('Answer:', answer);
+        break;
+      }
+      default:
+        console.error('Unknown command:', command);
+        process.exit(1);
+    }
+  } catch (err) {
+    console.error('Error during execution:', err.message);
+    process.exit(1);
+  }
+}
+
+await runCLI();