@devilsdev/rag-pipeline-utils 2.3.0 → 2.4.0

package/src/mocks/pinecone-retriever.js

@@ -1,106 +0,0 @@
-/**
- * Version: 0.1.0
- * File: /src/mocks/pinecone-retriever.js
- * Description: Mock implementation of a Pinecone retriever
- * Author: Ali Kahwaji
- */
-
-class PineconeRetriever {
-  constructor(options = {}) {
-    this.options = options;
-    this.storage = [];
-
-    // Configurable batch size from environment or options
-    this.batchSize =
-      parseInt(process.env.RAG_RETRIEVER_BATCH_SIZE) ||
-      options.batchSize ||
-      1000; // Default batch size for Pinecone
-  }
-
-  /**
-   * Mocks vector storage with batch processing
-   * @param {Array} vectors - Vectors to store
-   */
-  async store(vectors = []) {
-    if (!Array.isArray(vectors) || vectors.length === 0) {
-      return true;
-    }
-
-    // Process storage in batches for better memory management
-    for (let i = 0; i < vectors.length; i += this.batchSize) {
-      const batch = vectors.slice(i, i + this.batchSize);
-      this.storage.push(...batch);
-
-      // Optional progress callback
-      if (this.options.onProgress) {
-        this.options.onProgress({
-          processed: Math.min(i + this.batchSize, vectors.length),
-          total: vectors.length,
-          stage: "storing",
-        });
-      }
-
-      // Simulate network delay for large batches
-      if (vectors.length > this.batchSize) {
-        await new Promise((resolve) => setTimeout(resolve, 20));
-      }
-    }
-
-    return true;
-  }
-
-  /**
-   * Returns mock retrieved documents with batch processing for large result sets
-   * @param {Array} queryVector - Query vector
-   * @param {number} k - Number of results to return
-   * @returns {Promise<Array<{ id: string, text: string, metadata: object }>>}
-   */
-  async retrieve(queryVector = [], k = 5) {
-    const mockResults = [
-      { id: "a", text: "Chunk about pine trees", metadata: {} },
-      { id: "b", text: "Chunk about vectors", metadata: {} },
-      { id: "c", text: "Chunk about databases", metadata: {} },
-      { id: "d", text: "Chunk about machine learning", metadata: {} },
-      { id: "e", text: "Chunk about embeddings", metadata: {} },
-    ];
-
-    // Extend mock results for large k values
-    const results = [];
-    for (let i = 0; i < k; i++) {
-      const baseResult = mockResults[i % mockResults.length];
-      results.push({
-        ...baseResult,
-        id: `${baseResult.id}_${Math.floor(i / mockResults.length)}`,
-        text: `${baseResult.text} (result ${i + 1})`,
-      });
-    }
-
-    // For large result sets, process in batches
-    if (k > this.batchSize) {
-      const batchedResults = [];
-      for (let i = 0; i < results.length; i += this.batchSize) {
-        const batch = results.slice(i, i + this.batchSize);
-        batchedResults.push(...batch);
-
-        // Optional progress callback
-        if (this.options.onProgress) {
-          this.options.onProgress({
-            processed: Math.min(i + this.batchSize, results.length),
-            total: results.length,
-            stage: "retrieving",
-          });
-        }
-
-        // Simulate network delay for large retrievals
-        await new Promise((resolve) => setTimeout(resolve, 10));
-      }
-      return batchedResults;
-    }
-
-    return results;
-  }
-}
-
-// Default export
-
-module.exports = PineconeRetriever;

package/src/scripts/script-utilities.js

@@ -1,36 +0,0 @@
-// src/scripts/script-utilities.js
-
-// default sleeper uses real timers (prod) - unref'd to not block process exit
-const sleep = (ms) =>
-  new Promise((resolve) => {
-    const t = setTimeout(resolve, ms);
-    // critical: don't keep the process alive just because of a timer
-    if (typeof t.unref === "function") t.unref();
-  });
-
-async function retry(fn, opts = {}) {
-  const {
-    retries = 3,
-    baseDelay = 100,
-    factor = 2,
-    onDelay = () => {},
-    sleeper = sleep, // <-- inject for tests
-  } = opts;
-
-  let lastErr;
-  for (let attempt = 0; attempt <= retries; attempt++) {
-    try {
-      // pass attempt if useful for logging/logic
-      return await fn(attempt);
-    } catch (err) {
-      lastErr = err;
-      if (attempt === retries) break;
-      const delay = baseDelay * Math.pow(factor, attempt);
-      onDelay(delay, attempt);
-      await sleeper(delay); // <-- no-op in tests
-    }
-  }
-  throw lastErr;
-}
-
-module.exports = { retry, sleep };

package/src/utils/ci/diagnostic-reporter.js

@@ -1,57 +0,0 @@
-/**
- * Version: 1.0.0
- * Description: Enhanced CI diagnostics for plugin mock verification
- * Author: Ali Kahwaji
- */
-
-const path = require('path');
-// eslint-disable-line global-require
-const chalk = require('chalk');
-// eslint-disable-line global-require
-
-/**
- * Pretty prints CI validation errors and guidance
- * @param {Array<{ file: string, missing: string[] }>} failures
- */
-function reportPluginValidationFailures(failures) {
-  if (!failures.length) return;
-
-  console.log(`\n${chalk.redBright.bold('❌ Plugin Mock Validation Failed')}`);
-  // eslint-disable-line no-console
-
-  failures.forEach(({ file, missing }) => {
-    const fileName = path.basename(file);
-    console.log(
-      // eslint-disable-line no-console
-      `${chalk.red(`  ✖ ${fileName}`)} ${chalk.gray(`missing: ${missing.join(', ')}`)}`,
-    );
-
-    // Add method-level fix hints
-    missing.forEach((method) => {
-      const pluginType = fileName.split('.')[0].split('-')[1] || 'unknown';
-      console.error(
-        `    → Fix: implement method ${chalk.cyan(`${method}()`)} for plugin _type ${chalk.yellow(pluginType)}`,
-      );
-      // eslint-disable-line no-console
-    });
-  });
-
-  console.log(`\n${chalk.yellowBright('Recommended Fixes:')}`);
-  // eslint-disable-line no-console
-  console.log(
-    '  • Implement all required methods defined in /src/core/plugin-contracts.js',
-  );
-  // eslint-disable-line no-console
-  console.log(`  • Re-run: ${chalk.cyan('npm run ci')} to confirm validation`);
-  // eslint-disable-line no-console
-  console.log(
-    `  • Optionally use: ${chalk.cyan('scripts/repair-fixtures.js')} to auto-repair mocks\n`,
-  );
-  // eslint-disable-line no-console
-}
-
-// Default export
-
-module.exports = {
-  reportPluginValidationFailures,
-};

package/src/utils/examples/error-formatter-usage.js

@@ -1,351 +0,0 @@
-'use strict';
-
-/**
- * Error Formatter Usage Examples
- *
- * Demonstrates how to use the enhanced error formatter
- * throughout the codebase
- */
-
-const {
-  createError,
-  wrapError,
-  formatErrorMessage,
-  logError,
-  ERROR_CODES,
-} = require('../error-formatter');
-
-// ============================================================================
-// Example 1: Basic Error Creation
-// ============================================================================
-
-function basicErrorExample() {
-  console.log('\n=== Example 1: Basic Error Creation ===\n');
-
-  try {
-    throw createError(ERROR_CODES.CONFIG_NOT_FOUND, {
-      path: '.ragrc.json',
-    });
-  } catch (error) {
-    console.log(formatErrorMessage(error));
-  }
-}
-
-// ============================================================================
-// Example 2: Error with Rich Context
-// ============================================================================
-
-function richContextExample() {
-  console.log('\n=== Example 2: Error with Rich Context ===\n');
-
-  try {
-    const input = { invalid: 'data' };
-
-    throw createError(ERROR_CODES.PLUGIN_EXECUTION_FAILED, {
-      plugin: 'embedder-openai',
-      reason: 'Invalid API key',
-      input: JSON.stringify(input),
-      details: 'Check your OPENAI_API_KEY environment variable',
-    });
-  } catch (error) {
-    console.log(
-      formatErrorMessage(error, {
-        includeContext: true,
-        includeSuggestions: true,
-        includeDocumentation: true,
-      }),
-    );
-  }
-}
-
-// ============================================================================
-// Example 3: Wrapping Existing Errors
-// ============================================================================
-
-function wrapErrorExample() {
-  console.log('\n=== Example 3: Wrapping Existing Errors ===\n');
-
-  try {
-    try {
-      // Simulate file read error
-      throw new Error('ENOENT: no such file or directory');
-    } catch (originalError) {
-      throw wrapError(originalError, ERROR_CODES.FS_FILE_NOT_FOUND, {
-        path: '/path/to/config.json',
-      });
-    }
-  } catch (error) {
-    console.log(
-      formatErrorMessage(error, {
-        includeStack: false,
-      }),
-    );
-  }
-}
-
-// ============================================================================
-// Example 4: Configuration Errors
-// ============================================================================
-
-function configErrorExample() {
-  console.log('\n=== Example 4: Configuration Errors ===\n');
-
-  try {
-    throw createError(ERROR_CODES.CONFIG_INVALID_TYPE, {
-      field: 'pipeline.stages',
-      expected: 'array',
-      actual: 'string',
-      currentValue: '"fetch,process,store"',
-    });
-  } catch (error) {
-    console.log(formatErrorMessage(error));
-  }
-}
-
-// ============================================================================
-// Example 5: Plugin Contract Violation
-// ============================================================================
-
-function pluginContractExample() {
-  console.log('\n=== Example 5: Plugin Contract Violation ===\n');
-
-  try {
-    throw createError(ERROR_CODES.PLUGIN_CONTRACT_VIOLATION, {
-      plugin: 'my-custom-plugin',
-      violation: 'Missing execute() method',
-      missingMethod: 'execute',
-      expectedType: 'function',
-    });
-  } catch (error) {
-    console.log(formatErrorMessage(error));
-  }
-}
-
-// ============================================================================
-// Example 6: Pipeline Execution Error
-// ============================================================================
-
-function pipelineErrorExample() {
-  console.log('\n=== Example 6: Pipeline Execution Error ===\n');
-
-  try {
-    throw createError(ERROR_CODES.PIPELINE_STAGE_FAILED, {
-      stage: 'embedding',
-      reason: 'API rate limit exceeded',
-      input: JSON.stringify({ text: 'Sample text...' }).slice(0, 50),
-    });
-  } catch (error) {
-    // Log with appropriate severity
-    logError(error);
-  }
-}
-
-// ============================================================================
-// Example 7: Validation Error with Multiple Issues
-// ============================================================================
-
-function validationErrorExample() {
-  console.log('\n=== Example 7: Validation Error ===\n');
-
-  try {
-    const errors = [
-      { field: 'metadata.name', message: 'Required field missing' },
-      { field: 'pipeline.stages', message: 'Must be an array' },
-    ];
-
-    throw createError(ERROR_CODES.VALIDATION_FAILED, {
-      reason: 'Configuration validation failed',
-      errors: JSON.stringify(errors, null, 2),
-    });
-  } catch (error) {
-    console.log(
-      formatErrorMessage(error, {
-        includeContext: true,
-      }),
-    );
-  }
-}
-
-// ============================================================================
-// Example 8: Hot Reload Error
-// ============================================================================
-
-function hotReloadErrorExample() {
-  console.log('\n=== Example 8: Hot Reload Error ===\n');
-
-  try {
-    throw createError(ERROR_CODES.HOT_RELOAD_FAILED, {
-      path: './src/plugins/my-plugin.js',
-      reason: 'SyntaxError: Unexpected token',
-      watchPaths: './src/plugins, ./src/mocks',
-    });
-  } catch (error) {
-    console.log(formatErrorMessage(error));
-  }
-}
-
-// ============================================================================
-// Example 9: DAG Cycle Detection
-// ============================================================================
-
-function dagCycleExample() {
-  console.log('\n=== Example 9: DAG Cycle Detection ===\n');
-
-  try {
-    throw createError(ERROR_CODES.DAG_CYCLE_DETECTED, {
-      cycle: 'A -> B -> C -> A',
-    });
-  } catch (error) {
-    console.log(formatErrorMessage(error));
-  }
-}
-
-// ============================================================================
-// Example 10: Authentication Error
-// ============================================================================
-
-function authErrorExample() {
-  console.log('\n=== Example 10: Authentication Error ===\n');
-
-  try {
-    throw createError(ERROR_CODES.AUTH_TOKEN_EXPIRED, {
-      expiredAt: new Date(Date.now() - 3600000).toISOString(),
-    });
-  } catch (error) {
-    console.log(formatErrorMessage(error));
-  }
-}
-
-// ============================================================================
-// Example 11: Custom Error Handling in Try-Catch
-// ============================================================================
-
-async function asyncErrorHandlingExample() {
-  console.log('\n=== Example 11: Async Error Handling ===\n');
-
-  async function loadPlugin(name) {
-    // Simulate plugin loading
-    throw new Error(`Cannot find module '${name}'`);
-  }
-
-  try {
-    await loadPlugin('@rag-pipeline/embedder-openai');
-  } catch (error) {
-    const enhanced = wrapError(error, ERROR_CODES.PLUGIN_LOAD_FAILED, {
-      plugin: '@rag-pipeline/embedder-openai',
-      path: './node_modules/@rag-pipeline/embedder-openai',
-      reason: error.message,
-    });
-
-    console.log(formatErrorMessage(enhanced));
-  }
-}
-
-// ============================================================================
-// Example 12: Error Serialization for Logging
-// ============================================================================
-
-function serializationExample() {
-  console.log('\n=== Example 12: Error Serialization ===\n');
-
-  try {
-    throw createError(ERROR_CODES.PLUGIN_NOT_FOUND, {
-      plugin: 'my-plugin',
-      path: './plugins/my-plugin.js',
-    });
-  } catch (error) {
-    // Serialize for logging to external service
-    const serialized = error.toJSON();
-    console.log(JSON.stringify(serialized, null, 2));
-  }
-}
-
-// ============================================================================
-// Example 13: Custom Error Formatting
-// ============================================================================
-
-function customFormattingExample() {
-  console.log('\n=== Example 13: Custom Formatting ===\n');
-
-  try {
-    throw createError(ERROR_CODES.CONFIG_VALIDATION_FAILED, {
-      reason: 'Invalid schema',
-      requiredFields: 'metadata.name, pipeline.stages',
-    });
-  } catch (error) {
-    // Minimal format for CI logs
-    console.log('CI Format:');
-    console.log(
-      formatErrorMessage(error, {
-        includeCode: true,
-        includeSuggestions: false,
-        includeDocumentation: false,
-        includeStack: false,
-      }),
-    );
-
-    console.log('\n---\n');
-
-    // Detailed format for debugging
-    console.log('Debug Format:');
-    console.log(
-      formatErrorMessage(error, {
-        includeCode: true,
-        includeSuggestions: true,
-        includeDocumentation: true,
-        includeContext: true,
-        includeStack: true,
-      }),
-    );
-  }
-}
-
-// ============================================================================
-// Run Examples
-// ============================================================================
-
-async function runAllExamples() {
-  console.log('='.repeat(70));
-  console.log('Error Formatter Usage Examples');
-  console.log('='.repeat(70));
-
-  basicErrorExample();
-  richContextExample();
-  wrapErrorExample();
-  configErrorExample();
-  pluginContractExample();
-  pipelineErrorExample();
-  validationErrorExample();
-  hotReloadErrorExample();
-  dagCycleExample();
-  authErrorExample();
-  await asyncErrorHandlingExample();
-  serializationExample();
-  customFormattingExample();
-
-  console.log('\n' + '='.repeat(70));
-  console.log('Examples Complete');
-  console.log('='.repeat(70) + '\n');
-}
-
-// Run if executed directly
-if (require.main === module) {
-  runAllExamples().catch(console.error);
-}
-
-module.exports = {
-  basicErrorExample,
-  richContextExample,
-  wrapErrorExample,
-  configErrorExample,
-  pluginContractExample,
-  pipelineErrorExample,
-  validationErrorExample,
-  hotReloadErrorExample,
-  dagCycleExample,
-  authErrorExample,
-  asyncErrorHandlingExample,
-  serializationExample,
-  customFormattingExample,
-  runAllExamples,
-};