@devilsdev/rag-pipeline-utils 1.0.0 → 1.0.1

package/release.config.js

@@ -1,23 +1,36 @@
-/**
- * Version: 0.1.0
- * Path: /release.config.js
- * Description: Semantic release configuration for automated changelog, versioning, GitHub and npm publishing
- * Author: Ali Kahwaji
- */
-
-export default {
-  branches: ['main'],
-  plugins: [
-    '@semantic-release/commit-analyzer',
-    '@semantic-release/release-notes-generator',
-    ['@semantic-release/changelog', {
-      changelogFile: 'CHANGELOG.md'
-    }],
-    '@semantic-release/npm',
-    ['@semantic-release/git', {
-      assets: ['CHANGELOG.md', 'package.json'],
-      message: 'chore(release): ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}'
-    }],
-    '@semantic-release/github'
-  ]
-};
+/**
+ * Version: 0.1.0
+ * Description: Semantic Release configuration for CI/CD automation with changelog, versioning, GitHub releases, and npm publishing.
+ * Author: Ali Kahwaji
+ */
+
+export default {
+  branches: ['main'],
+  repositoryUrl: 'https://github.com/DevilsDev/rag-pipeline-utils',
+
+  plugins: [
+    '@semantic-release/commit-analyzer',
+    '@semantic-release/release-notes-generator',
+    [
+      '@semantic-release/changelog',
+      {
+        changelogFile: 'CHANGELOG.md',
+        changelogTitle: '# Changelog',
+      },
+    ],
+    [
+      '@semantic-release/npm',
+      {
+        npmPublish: true,
+      },
+    ],
+    [
+      '@semantic-release/git',
+      {
+        assets: ['package.json', 'package-lock.json', 'CHANGELOG.md'],
+        message: 'chore(release): ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}',
+      },
+    ],
+    '@semantic-release/github',
+  ],
+};

package/scripts/setup.js

@@ -0,0 +1,7 @@
+import fs from 'fs';
+const fixturePath = './__tests__/fixtures/sample.pdf';
+
+if (!fs.existsSync(fixturePath)) {
+  fs.mkdirSync('./__tests__/fixtures', { recursive: true });
+  fs.writeFileSync(fixturePath, 'Dummy PDF content for test');
+}

package/scripts/test-global-setup.js

@@ -0,0 +1,18 @@
+// __tests__/setup/global.js
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+// __dirname polyfill for ESM
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const pdfPath = path.resolve(__dirname, '../fixtures/sample.pdf');
+
+// Ensure the fixtures directory exists
+fs.mkdirSync(path.dirname(pdfPath), { recursive: true });
+
+// Create a minimal valid PDF file if it doesn't exist
+if (!fs.existsSync(pdfPath)) {
+  fs.writeFileSync(pdfPath, '%PDF-1.4\n%EOF');
+}

package/server.js

@@ -1,31 +1,31 @@
-/**
- * Version: 0.1.0
- * Path: /server.js
- * Description: Local static server for serving evaluation dashboard
- * Author: Ali Kahwaji
- */
-
-import express from 'express';
-import path from 'path';
-import { fileURLToPath } from 'url';
-
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = path.dirname(__filename);
-
-const app = express();
-const port = process.env.PORT || 3000;
-
-// Serve evaluation results statically
-app.use('/results', express.static(path.join(__dirname, '__tests__/fixtures')));
-
-// Serve public frontend assets
-app.use(express.static(path.join(__dirname, 'public')));
-
-// Fallback to index.html
-app.get('*', (req, res) => {
-  res.sendFile(path.join(__dirname, 'public/index.html'));
-});
-
-app.listen(port, () => {
-  console.log(` Dashboard running at http://localhost:${port}`);
-});
+/**
+ * Version: 0.1.0
+ * Path: /server.js
+ * Description: Local static server for serving evaluation dashboard
+ * Author: Ali Kahwaji
+ */
+
+import express from 'express';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const app = express();
+const port = process.env.PORT || 3000;
+
+// Serve evaluation results statically
+app.use('/results', express.static(path.join(__dirname, '__tests__/fixtures')));
+
+// Serve public frontend assets
+app.use(express.static(path.join(__dirname, 'public')));
+
+// Fallback to index.html
+app.get('*', (req, res) => {
+  res.sendFile(path.join(__dirname, 'public/index.html'));
+});
+
+app.listen(port, () => {
+  console.log(` Dashboard running at http://localhost:${port}`);
+});

package/src/config/load-config.js

@@ -1,35 +1,35 @@
-/**
- * Version: 0.1.0
- * Path: /src/config/load-config.js
- * Description: Loader and validator for `.ragrc.json` configuration files
- * Author: Ali Kahwaji
- */
-
-import fs from 'fs';
-import path from 'path';
-import { validate } from './validate-schema.js';
-
-const DEFAULT_CONFIG_FILE = '.ragrc.json';
-
-/**
- * Load and validate the RAG configuration from a file.
- *
- * @param {string} [configPath] - Optional custom config path
- * @returns {object} - Validated config object
- * @throws {Error} - If file not found or validation fails
- */
-export function loadRagConfig(configPath = DEFAULT_CONFIG_FILE) {
-  const fullPath = path.resolve(process.cwd(), configPath);
-
-  if (!fs.existsSync(fullPath)) {
-    throw new Error(`Config file not found: ${fullPath}`);
-  }
-
-  const raw = fs.readFileSync(fullPath, 'utf-8');
-  const config = JSON.parse(raw);
-
-  validate(config); // Throws if invalid
-
-  return config;
-}
-
+/**
+ * Version: 0.1.0
+ * Path: /src/config/load-config.js
+ * Description: Loader and validator for `.ragrc.json` configuration files
+ * Author: Ali Kahwaji
+ */
+
+import fs from 'fs';
+import path from 'path';
+import { validate } from './validate-schema.js';
+
+const DEFAULT_CONFIG_FILE = '.ragrc.json';
+
+/**
+ * Load and validate the RAG configuration from a file.
+ *
+ * @param {string} [configPath] - Optional custom config path
+ * @returns {object} - Validated config object
+ * @throws {Error} - If file not found or validation fails
+ */
+export function loadRagConfig(configPath = DEFAULT_CONFIG_FILE) {
+  const fullPath = path.resolve(process.cwd(), configPath);
+
+  if (!fs.existsSync(fullPath)) {
+    throw new Error(`Config file not found: ${fullPath}`);
+  }
+
+  const raw = fs.readFileSync(fullPath, 'utf-8');
+  const config = JSON.parse(raw);
+
+  validate(config); // Throws if invalid
+
+  return config;
+}
+

package/src/config/validate-schema.js

@@ -1,49 +1,49 @@
-/**
- * Version: 0.1.0
- * Path: /src/config/validate-schema.js
- * Description: JSON schema validation for `.ragrc.json` config using AJV
- * Author: Ali Kahwaji
- */
-
-import Ajv from 'ajv';
-
-// Define allowed config structure
-const schema = {
-  type: 'object',
-  properties: {
-    loader: { type: 'string' },
-    embedder: { type: 'string' },
-    retriever: { type: 'string' },
-    llm: { type: 'string' },
-    chunk: {
-      type: 'object',
-      properties: {
-        strategy: { type: 'string' },
-        size: { type: 'number' }
-      },
-      required: ['strategy', 'size'],
-      additionalProperties: false
-    }
-  },
-  required: ['loader', 'embedder', 'retriever', 'llm'],
-  additionalProperties: false
-};
-
-const ajv = new Ajv();
-const validateSchema = ajv.compile(schema);
-
-/**
- * Validates a given config object against the `.ragrc.json` schema.
- * Throws if the schema is invalid.
- *
- * @param {object} config - Configuration object
- * @throws {Error} - Validation errors, if any
- */
-export function validate(config) {
-  const valid = validateSchema(config);
-  if (!valid) {
-    const messages = validateSchema.errors.map(e => `${e.instancePath} ${e.message}`).join(', ');
-    throw new Error(`Invalid config: ${messages}`);
-  }
-}
-
+/**
+ * Version: 0.1.0
+ * Path: /src/config/validate-schema.js
+ * Description: JSON schema validation for `.ragrc.json` config using AJV
+ * Author: Ali Kahwaji
+ */
+
+import Ajv from 'ajv';
+
+// Define allowed config structure
+const schema = {
+  type: 'object',
+  properties: {
+    loader: { type: 'string' },
+    embedder: { type: 'string' },
+    retriever: { type: 'string' },
+    llm: { type: 'string' },
+    chunk: {
+      type: 'object',
+      properties: {
+        strategy: { type: 'string' },
+        size: { type: 'number' }
+      },
+      required: ['strategy', 'size'],
+      additionalProperties: false
+    }
+  },
+  required: ['loader', 'embedder', 'retriever', 'llm'],
+  additionalProperties: false
+};
+
+const ajv = new Ajv();
+const validateSchema = ajv.compile(schema);
+
+/**
+ * Validates a given config object against the `.ragrc.json` schema.
+ * Throws if the schema is invalid.
+ *
+ * @param {object} config - Configuration object
+ * @throws {Error} - Validation errors, if any
+ */
+export function validate(config) {
+  const valid = validateSchema(config);
+  if (!valid) {
+    const messages = validateSchema.errors.map(e => `${e.instancePath} ${e.message}`).join(', ');
+    throw new Error(`Invalid config: ${messages}`);
+  }
+}
+

package/src/core/create-pipeline.js

@@ -1,82 +1,82 @@
-/**
- * Version: 0.1.3
- * Path: /src/core/create-pipeline.js
- * Description: RAG pipeline factory with optional LLM reranker integration
- * Author: Ali Kahwaji
- */
-
-import { PluginRegistry } from './plugin-registry.js';
-import { logger } from '../utils/logger.js';
-import { withRetry } from '../utils/retry.js';
-import { LLMReranker } from '../reranker/llm-reranker.js';
-
-const registry = new PluginRegistry();
-
-/**
- * Create a RAG pipeline instance using registered plugin components
- * @param {object} config
- * @param {string} config.loader
- * @param {string} config.embedder
- * @param {string} config.retriever
- * @param {string} config.llm
- * @param {boolean} [config.useReranker=false]
- * @returns {object} RAG pipeline instance
- */
-export function createRagPipeline({ loader, embedder, retriever, llm, useReranker = false }) {
-  const rerankerInstance = useReranker ? new LLMReranker({ llm: registry.get('llm', llm) }) : null;
-
-  return {
-    async ingest(path) {
-      logger.info({ loader, embedder, retriever }, 'Pipeline ingest start');
-      const loaderInstance = registry.get('loader', loader);
-      const embedderInstance = registry.get('embedder', embedder);
-      const retrieverInstance = registry.get('retriever', retriever);
-
-      const documents = await loaderInstance.load(path);
-      const chunks = documents.flatMap(doc => doc.chunk());
-      const vectors = await embedderInstance.embed(chunks);
-
-      await withRetry(() => retrieverInstance.store(vectors), {
-        label: 'vector-store',
-        retries: 3,
-        initialDelay: 500
-      });
-
-      logger.info('Ingestion pipeline completed');
-    },
-
-    async query(prompt) {
-      logger.info({ prompt, embedder, retriever, llm, useReranker }, 'Pipeline query start');
-      const embedderInstance = registry.get('embedder', embedder);
-      const retrieverInstance = registry.get('retriever', retriever);
-      const llmInstance = registry.get('llm', llm);
-
-      const queryVector = await embedderInstance.embedQuery(prompt);
-      let retrieved = await withRetry(() => retrieverInstance.retrieve(queryVector), {
-        label: 'vector-retrieve',
-        retries: 3,
-        initialDelay: 500
-      });
-
-      if (rerankerInstance) {
-        logger.info('Applying LLM reranker to retrieved chunks');
-        retrieved = await withRetry(() => rerankerInstance.rerank(prompt, retrieved), {
-          label: 'rerank',
-          retries: 2,
-          initialDelay: 400
-        });
-      }
-
-      const result = await withRetry(() => llmInstance.generate(prompt, retrieved), {
-        label: 'llm-generate',
-        retries: 3,
-        initialDelay: 500
-      });
-
-      return result;
-    }
-  };
-}
-
-export { registry };
-
+/**
+ * Version: 0.1.3
+ * Path: /src/core/create-pipeline.js
+ * Description: RAG pipeline factory with optional LLM reranker integration
+ * Author: Ali Kahwaji
+ */
+
+import { PluginRegistry } from './plugin-registry.js';
+import { logger } from '../utils/logger.js';
+import { withRetry } from '../utils/retry.js';
+import { LLMReranker } from '../reranker/llm-reranker.js';
+
+const registry = new PluginRegistry();
+
+/**
+ * Create a RAG pipeline instance using registered plugin components
+ * @param {object} config
+ * @param {string} config.loader
+ * @param {string} config.embedder
+ * @param {string} config.retriever
+ * @param {string} config.llm
+ * @param {boolean} [config.useReranker=false]
+ * @returns {object} RAG pipeline instance
+ */
+export function createRagPipeline({ loader, embedder, retriever, llm, useReranker = false }) {
+  const rerankerInstance = useReranker ? new LLMReranker({ llm: registry.get('llm', llm) }) : null;
+
+  return {
+    async ingest(path) {
+      logger.info({ loader, embedder, retriever }, 'Pipeline ingest start');
+      const loaderInstance = registry.get('loader', loader);
+      const embedderInstance = registry.get('embedder', embedder);
+      const retrieverInstance = registry.get('retriever', retriever);
+
+      const documents = await loaderInstance.load(path);
+      const chunks = documents.flatMap(doc => doc.chunk());
+      const vectors = await embedderInstance.embed(chunks);
+
+      await withRetry(() => retrieverInstance.store(vectors), {
+        label: 'vector-store',
+        retries: 3,
+        initialDelay: 500
+      });
+
+      logger.info('Ingestion pipeline completed');
+    },
+
+    async query(prompt) {
+      logger.info({ prompt, embedder, retriever, llm, useReranker }, 'Pipeline query start');
+      const embedderInstance = registry.get('embedder', embedder);
+      const retrieverInstance = registry.get('retriever', retriever);
+      const llmInstance = registry.get('llm', llm);
+
+      const queryVector = await embedderInstance.embedQuery(prompt);
+      let retrieved = await withRetry(() => retrieverInstance.retrieve(queryVector), {
+        label: 'vector-retrieve',
+        retries: 3,
+        initialDelay: 500
+      });
+
+      if (rerankerInstance) {
+        logger.info('Applying LLM reranker to retrieved chunks');
+        retrieved = await withRetry(() => rerankerInstance.rerank(prompt, retrieved), {
+          label: 'rerank',
+          retries: 2,
+          initialDelay: 400
+        });
+      }
+
+      const result = await withRetry(() => llmInstance.generate(prompt, retrieved), {
+        label: 'llm-generate',
+        retries: 3,
+        initialDelay: 500
+      });
+
+      return result;
+    }
+  };
+}
+
+export { registry };
+

package/src/core/plugin-registry.js

@@ -1,61 +1,61 @@
-/**
- * Version: 1.0.0
- * Path: /src/core/plugin-registry.js
- * Description: Registry for managing pluggable components (loader, embedder, retriever, llm)
- * Author: Ali Kahwaji
- */
-
-export class PluginRegistry {
-  #registry = {
-    loader: new Map(),
-    embedder: new Map(),
-    retriever: new Map(),
-    llm: new Map(),
-    reranker: new Map(),
-  };
-
-  /**
-   * Registers a plugin instance under a specific type and name.
-   * @param {'loader' | 'embedder' | 'retriever' | 'llm' | 'reranker'} type - Plugin type category.
-   * @param {string} name - Unique name/key for this plugin.
-   * @param {object} plugin - Plugin implementation (e.g., instance or factory).
-   */
-  register(type, name, plugin) {
-    const group = this.#registry[type];
-    if (!group) {
-      throw new Error(`Unknown plugin type: '${type}'`);
-    }
-    group.set(name, plugin);
-  }
-
-  /**
-   * Retrieves a plugin by type and name.
-   * @param {'loader' | 'embedder' | 'retriever' | 'llm' | 'reranker'} type - Plugin type.
-   * @param {string} name - Plugin name/key.
-   * @returns {object} - The plugin instance.
-   */
-  get(type, name) {
-    const group = this.#registry[type];
-    if (!group) {
-      throw new Error(`Unknown plugin type: '${type}'`);
-    }
-    const plugin = group.get(name);
-    if (!plugin) {
-      throw new Error(`Plugin not found: [${type}:${name}]`);
-    }
-    return plugin;
-  }
-
-  /**
-   * Lists all registered plugin names for a type.
-   * @param {'loader' | 'embedder' | 'retriever' | 'llm' | 'reranker'} type - Plugin category.
-   * @returns {string[]} - Array of registered plugin names.
-   */
-  list(type) {
-    const group = this.#registry[type];
-    if (!group) {
-      throw new Error(`Unknown plugin type: '${type}'`);
-    }
-    return [...group.keys()];
-  }
-}
+/**
+ * Version: 1.0.0
+ * Path: /src/core/plugin-registry.js
+ * Description: Registry for managing pluggable components (loader, embedder, retriever, llm)
+ * Author: Ali Kahwaji
+ */
+
+export class PluginRegistry {
+  #registry = {
+    loader: new Map(),
+    embedder: new Map(),
+    retriever: new Map(),
+    llm: new Map(),
+    reranker: new Map(),
+  };
+
+  /**
+   * Registers a plugin instance under a specific type and name.
+   * @param {'loader' | 'embedder' | 'retriever' | 'llm' | 'reranker'} type - Plugin type category.
+   * @param {string} name - Unique name/key for this plugin.
+   * @param {object} plugin - Plugin implementation (e.g., instance or factory).
+   */
+  register(type, name, plugin) {
+    const group = this.#registry[type];
+    if (!group) {
+      throw new Error(`Unknown plugin type: '${type}'`);
+    }
+    group.set(name, plugin);
+  }
+
+  /**
+   * Retrieves a plugin by type and name.
+   * @param {'loader' | 'embedder' | 'retriever' | 'llm' | 'reranker'} type - Plugin type.
+   * @param {string} name - Plugin name/key.
+   * @returns {object} - The plugin instance.
+   */
+  get(type, name) {
+    const group = this.#registry[type];
+    if (!group) {
+      throw new Error(`Unknown plugin type: '${type}'`);
+    }
+    const plugin = group.get(name);
+    if (!plugin) {
+      throw new Error(`Plugin not found: [${type}:${name}]`);
+    }
+    return plugin;
+  }
+
+  /**
+   * Lists all registered plugin names for a type.
+   * @param {'loader' | 'embedder' | 'retriever' | 'llm' | 'reranker'} type - Plugin category.
+   * @returns {string[]} - Array of registered plugin names.
+   */
+  list(type) {
+    const group = this.#registry[type];
+    if (!group) {
+      throw new Error(`Unknown plugin type: '${type}'`);
+    }
+    return [...group.keys()];
+  }
+}