@devilsdev/rag-pipeline-utils 2.3.1 → 2.4.1

package/src/cli/wizard/preview-save.js

@@ -0,0 +1,172 @@
+/**
+ * Preview and Save Step
+ * Handles configuration preview, validation, saving, and testing
+ */
+
+const fs = require("fs/promises");
+const {
+  validateRagrc,
+  extractPluginDependencies,
+} = require("../../config/enhanced-ragrc-schema.js");
+
+/**
+ * Preview configuration and save
+ * @param {object} config - Current wizard configuration state
+ * @param {object} inquirer - Inquirer instance for prompting
+ * @param {object} options - Wizard options (outputPath, etc.)
+ * @returns {Promise<object>} Updated configuration state (possibly normalized)
+ */
+async function previewAndSave(config, inquirer, options) {
+  console.log("👀 Configuration Preview\n");
+  // eslint-disable-line no-console
+
+  // Show configuration summary
+  showConfigSummary(config);
+
+  const answers = await inquirer.prompt([
+    {
+      _type: "confirm",
+      name: "save",
+      message: "Save this configuration?",
+      default: true,
+    },
+    {
+      _type: "input",
+      name: "filename",
+      message: "Configuration filename:",
+      default: options.outputPath,
+      when: (answers) => answers.save,
+    },
+    {
+      _type: "confirm",
+      name: "testRun",
+      message: "Run a test to validate the configuration?",
+      default: false,
+      when: (answers) => answers.save,
+    },
+  ]);
+
+  if (!answers.save) {
+    console.log("Configuration not saved."); // eslint-disable-line no-console
+    return config;
+  }
+
+  // Validate and normalize configuration
+  const validation = validateRagrc(config);
+  if (!validation.valid) {
+    console.error("❌ Configuration validation failed:"); // eslint-disable-line no-console
+    validation.errors?.forEach((error) => {
+      console.error(`  ${error.instancePath}: ${error.message}`); // eslint-disable-line no-console
+    });
+    throw new Error("Invalid configuration generated");
+  }
+
+  // Use normalized configuration
+  const normalizedConfig = validation.normalized;
+
+  // Save configuration
+  await fs.writeFile(
+    answers.filename,
+    JSON.stringify(normalizedConfig, null, 2),
+    "utf-8",
+  );
+
+  options.outputPath = answers.filename;
+
+  if (answers.testRun) {
+    await runConfigurationTest(normalizedConfig);
+  }
+
+  return normalizedConfig;
+}
+
+/**
+ * Show configuration summary
+ * @param {object} config - Configuration to summarize
+ */
+function showConfigSummary(config) {
+  console.log("📋 Configuration Summary:");
+  // eslint-disable-line no-console
+  console.log(`   Project: ${config.metadata?.name || "Unnamed"}`);
+  // eslint-disable-line no-console
+  console.log(
+    `   Environment: ${config.metadata?.environment || "development"}`,
+  );
+  // eslint-disable-line no-console
+
+  console.log("\n🔌 Plugins:");
+  // eslint-disable-line no-console
+  for (const [_type, plugins] of Object.entries(config.plugins)) {
+    if (Object.keys(plugins).length > 0) {
+      console.log(`   ${_type}: ${Object.keys(plugins).join(", ")}`);
+      // eslint-disable-line no-console
+    }
+  }
+
+  console.log("\n🔄 Pipeline:");
+  // eslint-disable-line no-console
+  console.log(
+    `   Stages: ${config.pipeline?.stages?.join(" → ") || "default"}`,
+  );
+  // eslint-disable-line no-console
+
+  if (config.performance?.parallel?.enabled) {
+    console.log("\n⚡ Performance:");
+    // eslint-disable-line no-console
+    console.log(
+      `   Parallel processing: ${config.performance.parallel.maxConcurrency} concurrent`,
+    );
+    // eslint-disable-line no-console
+  }
+
+  if (
+    config.observability?.tracing?.enabled ||
+    config.observability?.metrics?.enabled
+  ) {
+    console.log("\n📊 Observability:");
+    // eslint-disable-line no-console
+    if (config.observability.tracing?.enabled)
+      console.log("   Tracing: enabled");
+    // eslint-disable-line no-console
+    if (config.observability.metrics?.enabled)
+      console.log("   Metrics: enabled");
+    // eslint-disable-line no-console
+  }
+
+  console.log("");
+  // eslint-disable-line no-console
+}
+
+/**
+ * Run configuration test
+ * @param {object} config - Configuration to test
+ */
+async function runConfigurationTest(config) {
+  console.log("🧪 Running configuration test...\n");
+  // eslint-disable-line no-console
+
+  try {
+    // Simulate pipeline creation and basic validation
+    console.log("✅ Configuration syntax: valid");
+    // eslint-disable-line no-console
+    console.log("✅ Plugin dependencies: resolved");
+    // eslint-disable-line no-console
+    console.log("✅ Pipeline stages: configured");
+    // eslint-disable-line no-console
+
+    // Check for potential issues
+    const dependencies = extractPluginDependencies(config);
+    if (dependencies.length === 0) {
+      console.warn("⚠️  No plugins configured - pipeline will not function");
+      // eslint-disable-line no-console
+    }
+
+    console.log("\n🎉 Configuration test passed!");
+    // eslint-disable-line no-console
+  } catch (error) {
+    console.error("❌ Configuration test failed:", error.message);
+    // eslint-disable-line no-console
+  }
+}
+
+module.exports = { previewAndSave, showConfigSummary, runConfigurationTest };

package/src/cli/wizard/project-setup.js

@@ -0,0 +1,66 @@
+/**
+ * Project Setup Step
+ * Handles project metadata configuration in the interactive wizard
+ */
+
+const path = require("path");
+
+/**
+ * Run the project setup step
+ * @param {object} config - Current wizard configuration state
+ * @param {object} inquirer - Inquirer instance for prompting
+ * @returns {Promise<object>} Updated configuration state
+ */
+async function setupProject(config, inquirer) {
+  console.log("📋 Project Setup\n");
+  // eslint-disable-line no-console
+
+  const answers = await inquirer.prompt([
+    {
+      _type: "input",
+      name: "name",
+      message: "Project name:",
+      default: path.basename(process.cwd()),
+      validate: (input) => input.length > 0 || "Project name is required",
+    },
+    {
+      _type: "input",
+      name: "description",
+      message: "Project description:",
+      default: "A RAG pipeline project",
+    },
+    {
+      _type: "input",
+      name: "author",
+      message: "Author:",
+      default: "Unknown",
+    },
+    {
+      _type: "list",
+      name: "environment",
+      message: "Target environment:",
+      choices: [
+        { name: "Development", value: "development" },
+        { name: "Production", value: "production" },
+        { name: "Testing", value: "testing" },
+      ],
+      default: "development",
+    },
+  ]);
+
+  config.metadata = {
+    name: answers.name,
+    version: "1.0.0",
+    description: answers.description,
+    author: answers.author,
+    environment: answers.environment,
+    createdAt: new Date().toISOString(),
+  };
+
+  console.log(`\n✅ Project "${answers.name}" configured\n`);
+  // eslint-disable-line no-console
+
+  return config;
+}
+
+module.exports = { setupProject };

package/src/config/memory-optimization.js

@@ -10,6 +10,8 @@
  * @since 2.3.0
  */
 
+const { logger } = require("../utils/logger");
+
 /**
  * Light Profile
  *
@@ -248,9 +250,7 @@ function selectProfileByEnvironment() {
       case "standard":
         return STANDARD_PROFILE;
       default:
-        console.warn(
-          `Unknown MEMORY_PROFILE: ${memoryProfile}, using STANDARD`,
-        );
+        logger.warn(`Unknown MEMORY_PROFILE: ${memoryProfile}, using STANDARD`);
         return STANDARD_PROFILE;
     }
   }

package/src/connectors/anthropic-connector.js

@@ -0,0 +1,179 @@
+'use strict';
+
+const axios = require('axios');
+const { BaseConnector } = require('./base-connector');
+
+/**
+ * Default configuration for the Anthropic connector.
+ * @type {Object}
+ */
+const DEFAULT_CONFIG = {
+  model: 'claude-3-5-sonnet-20241022',
+  apiKey: process.env.ANTHROPIC_API_KEY || '',
+  baseURL: 'https://api.anthropic.com',
+  apiVersion: '2023-06-01',
+  maxTokens: 4096,
+  temperature: 0.7,
+};
+
+/**
+ * LLM connector for the Anthropic Claude API.
+ * Uses axios for HTTP calls and supports streaming via SSE.
+ *
+ * @extends BaseConnector
+ */
+class AnthropicConnector extends BaseConnector {
+  /**
+   * @param {Object} [options]
+   * @param {string} [options.model='claude-3-5-sonnet-20241022'] - Claude model to use
+   * @param {string} [options.apiKey] - Anthropic API key
+   * @param {string} [options.baseURL='https://api.anthropic.com'] - API base URL
+   * @param {string} [options.apiVersion='2023-06-01'] - API version header
+   * @param {number} [options.maxTokens=4096] - Max tokens in response
+   * @param {number} [options.temperature=0.7] - Sampling temperature
+   */
+  constructor(options = {}) {
+    super({ name: 'anthropic', ...options });
+    this.config = { ...DEFAULT_CONFIG, ...options };
+    this.client = axios.create({
+      baseURL: this.config.baseURL,
+      headers: {
+        'x-api-key': this.config.apiKey,
+        'anthropic-version': this.config.apiVersion,
+        'content-type': 'application/json',
+      },
+    });
+  }
+
+  /**
+   * Connect to Anthropic. Verifies the API key is present.
+   * @returns {Promise<void>}
+   * @throws {Error} If no API key is configured
+   */
+  async connect() {
+    if (!this.config.apiKey) {
+      throw new Error(
+        'Anthropic API key required. Set ANTHROPIC_API_KEY or pass apiKey option.',
+      );
+    }
+    await super.connect();
+  }
+
+  /**
+   * Generate a response from Claude using the provided query and optional RAG context.
+   *
+   * @param {string} query - The user query / prompt
+   * @param {Array<Object|string>} [context=[]] - Retrieved documents for RAG
+   * @param {Object} [options={}] - Override model, temperature, maxTokens per-call
+   * @returns {Promise<string>} The generated text
+   */
+  async generate(query, context = [], options = {}) {
+    return this.withRetry(async () => {
+      const body = {
+        model: options.model || this.config.model,
+        max_tokens: options.maxTokens ?? this.config.maxTokens,
+        temperature: options.temperature ?? this.config.temperature,
+        messages: [
+          { role: 'user', content: this._buildPrompt(query, context) },
+        ],
+      };
+
+      // Add system message when context is provided
+      if (context && context.length > 0) {
+        const contextText = context
+          .map((doc) => doc.content || doc.text || String(doc))
+          .join('\n\n');
+        body.system = `Use the following context to answer the question:\n\n${contextText}`;
+        body.messages = [{ role: 'user', content: query }];
+      }
+
+      const response = await this.client.post('/v1/messages', body);
+      return response.data.content[0].text;
+    });
+  }
+
+  /**
+   * Stream a response from Claude token-by-token via SSE.
+   *
+   * @param {string} query - The user query / prompt
+   * @param {Array<Object|string>} [context=[]] - Retrieved documents for RAG
+   * @param {Object} [options={}] - Override model, temperature, maxTokens per-call
+   * @yields {{ token: string, done: boolean }}
+   */
+  async *generateStream(query, context = [], options = {}) {
+    const body = {
+      model: options.model || this.config.model,
+      max_tokens: options.maxTokens ?? this.config.maxTokens,
+      temperature: options.temperature ?? this.config.temperature,
+      messages: [{ role: 'user', content: this._buildPrompt(query, context) }],
+      stream: true,
+    };
+
+    if (context && context.length > 0) {
+      const contextText = context
+        .map((doc) => doc.content || doc.text || String(doc))
+        .join('\n\n');
+      body.system = `Use the following context to answer the question:\n\n${contextText}`;
+      body.messages = [{ role: 'user', content: query }];
+    }
+
+    const response = await this.client.post('/v1/messages', body, {
+      responseType: 'stream',
+    });
+
+    let buffer = '';
+
+    for await (const chunk of response.data) {
+      buffer += chunk.toString();
+      const lines = buffer.split('\n');
+      // Keep the last incomplete line in the buffer
+      buffer = lines.pop() || '';
+
+      for (const line of lines) {
+        if (!line.startsWith('data: ')) continue;
+
+        const jsonStr = line.slice(6).trim();
+        if (!jsonStr || jsonStr === '[DONE]') continue;
+
+        try {
+          const event = JSON.parse(jsonStr);
+
+          if (
+            event.type === 'content_block_delta' &&
+            event.delta &&
+            event.delta.text
+          ) {
+            yield { token: event.delta.text, done: false };
+          }
+
+          if (event.type === 'message_stop') {
+            yield { token: '', done: true };
+            return;
+          }
+        } catch {
+          // Skip malformed JSON lines
+        }
+      }
+    }
+
+    yield { token: '', done: true };
+  }
+
+  /**
+   * Build a prompt string that includes RAG context when available.
+   *
+   * @param {string} query - User query
+   * @param {Array<Object|string>} context - RAG context documents
+   * @returns {string} Combined prompt
+   * @private
+   */
+  _buildPrompt(query, context) {
+    if (!context || context.length === 0) return query;
+    const contextText = context
+      .map((doc) => doc.content || doc.text || String(doc))
+      .join('\n\n');
+    return `Context:\n${contextText}\n\nQuestion: ${query}`;
+  }
+}
+
+module.exports = { AnthropicConnector, DEFAULT_CONFIG };

package/src/connectors/base-connector.js

@@ -0,0 +1,94 @@
+'use strict';
+
+const { EventEmitter } = require('events');
+
+/**
+ * Abstract base class for all connectors.
+ * Provides connection lifecycle, health checking, and retry logic.
+ * @extends EventEmitter
+ */
+class BaseConnector extends EventEmitter {
+  /**
+   * @param {Object} options
+   * @param {string} [options.name] - Connector display name
+   */
+  constructor(options = {}) {
+    super();
+    this.options = options;
+    this.connected = false;
+    this.name = options.name || this.constructor.name;
+  }
+
+  /**
+   * Establish the connection.
+   * Override in subclass for custom connection logic.
+   * @returns {Promise<void>}
+   */
+  async connect() {
+    this.connected = true;
+    this.emit('connected', { name: this.name });
+  }
+
+  /**
+   * Tear down the connection.
+   * Override in subclass for custom disconnection logic.
+   * @returns {Promise<void>}
+   */
+  async disconnect() {
+    this.connected = false;
+    this.emit('disconnected', { name: this.name });
+  }
+
+  /**
+   * Check whether the connector is healthy and operational.
+   * Override in subclass for custom health checks.
+   * @returns {Promise<boolean>}
+   */
+  async isHealthy() {
+    return this.connected;
+  }
+
+  /**
+   * Retry wrapper that attempts a function up to N times with a delay between retries.
+   * Emits a 'retry' event on each retry attempt.
+   * @param {Function} fn - Async function to execute
+   * @param {number} [retries=3] - Maximum number of retry attempts
+   * @param {number} [delay=1000] - Delay in ms between retries
+   * @returns {Promise<*>} Result of the function
+   * @throws {Error} Last error if all retries are exhausted
+   */
+  async withRetry(fn, retries = 3, delay = 1000) {
+    let lastError;
+    for (let attempt = 0; attempt <= retries; attempt++) {
+      try {
+        return await fn();
+      } catch (err) {
+        lastError = err;
+        if (attempt < retries) {
+          this.emit('retry', {
+            name: this.name,
+            attempt: attempt + 1,
+            maxRetries: retries,
+            error: err.message,
+          });
+          await new Promise((resolve) => setTimeout(resolve, delay));
+        }
+      }
+    }
+    throw lastError;
+  }
+
+  /**
+   * Return basic information about the connector.
+   * @returns {{ name: string, connected: boolean, type: string }}
+   */
+  getInfo() {
+    return {
+      name: this.name,
+      connected: this.connected,
+      type: this.constructor.name,
+    };
+  }
+}
+
+module.exports = { BaseConnector };

package/src/connectors/cohere-connector.js

@@ -0,0 +1,143 @@
+'use strict';
+
+const axios = require('axios');
+const { BaseConnector } = require('./base-connector');
+
+/**
+ * Default configuration for the Cohere connector.
+ * @type {Object}
+ */
+const DEFAULT_CONFIG = {
+  apiKey: process.env.COHERE_API_KEY || '',
+  baseURL: 'https://api.cohere.ai/v1',
+  embeddingModel: 'embed-english-v3.0',
+  rerankModel: 'rerank-english-v3.0',
+  inputType: 'search_document',
+  queryInputType: 'search_query',
+};
+
+/**
+ * Embedder + reranker connector for the Cohere API.
+ * Provides text embeddings (with separate document/query input types) and
+ * document reranking capabilities.
+ *
+ * @extends BaseConnector
+ */
+class CohereConnector extends BaseConnector {
+  /**
+   * @param {Object} [options]
+   * @param {string} [options.apiKey] - Cohere API key
+   * @param {string} [options.baseURL='https://api.cohere.ai/v1'] - API base URL
+   * @param {string} [options.embeddingModel='embed-english-v3.0'] - Embedding model
+   * @param {string} [options.rerankModel='rerank-english-v3.0'] - Reranking model
+   * @param {string} [options.inputType='search_document'] - Input type for document embeddings
+   * @param {string} [options.queryInputType='search_query'] - Input type for query embeddings
+   */
+  constructor(options = {}) {
+    super({ name: 'cohere', ...options });
+    this.config = { ...DEFAULT_CONFIG, ...options };
+    this.client = axios.create({
+      baseURL: this.config.baseURL,
+      headers: {
+        Authorization: `Bearer ${this.config.apiKey}`,
+        'Content-Type': 'application/json',
+      },
+    });
+  }
+
+  /**
+   * Connect to Cohere. Verifies the API key is present.
+   * @returns {Promise<void>}
+   * @throws {Error} If no API key is configured
+   */
+  async connect() {
+    if (!this.config.apiKey) {
+      throw new Error(
+        'Cohere API key required. Set COHERE_API_KEY or pass apiKey option.',
+      );
+    }
+    await super.connect();
+  }
+
+  /**
+   * Generate an embedding vector for document text.
+   * Uses the document input type by default.
+   *
+   * @param {string} text - Input text to embed
+   * @returns {Promise<number[]>} Embedding vector
+   */
+  async embed(text) {
+    return this.withRetry(async () => {
+      const response = await this.client.post('/embed', {
+        texts: [text],
+        model: this.config.embeddingModel,
+        input_type: this.config.inputType,
+      });
+
+      return response.data.embeddings[0];
+    });
+  }
+
+  /**
+   * Generate an embedding vector for a search query.
+   * Uses the query input type for better retrieval accuracy.
+   *
+   * @param {string} query - Query text to embed
+   * @returns {Promise<number[]>} Embedding vector
+   */
+  async embedQuery(query) {
+    return this.withRetry(async () => {
+      const response = await this.client.post('/embed', {
+        texts: [query],
+        model: this.config.embeddingModel,
+        input_type: this.config.queryInputType,
+      });
+
+      return response.data.embeddings[0];
+    });
+  }
+
+  /**
+   * Rerank documents by relevance to a query using Cohere's reranker.
+   *
+   * @param {string} query - The search query
+   * @param {Array<Object|string>} documents - Documents to rerank
+   * @param {Object} [options={}]
+   * @param {number} [options.topK] - Number of top results to return (defaults to all)
+   * @returns {Promise<Array<Object>>} Reranked documents with relevance scores
+   */
+  async rerank(query, documents, options = {}) {
+    return this.withRetry(async () => {
+      const docs = documents.map(
+        (doc) => doc.content || doc.text || String(doc),
+      );
+
+      const response = await this.client.post('/rerank', {
+        model: this.config.rerankModel,
+        query,
+        documents: docs,
+        top_n: options.topK || documents.length,
+      });
+
+      return response.data.results.map((result) => ({
+        ...documents[result.index],
+        score: result.relevance_score,
+      }));
+    });
+  }
+
+  /**
+   * Check connector health by performing a lightweight embed call.
+   * @returns {Promise<boolean>}
+   */
+  async isHealthy() {
+    try {
+      await this.embed('health check');
+      return true;
+    } catch {
+      return false;
+    }
+  }
+}
+
+module.exports = { CohereConnector, DEFAULT_CONFIG };