@devilsdev/rag-pipeline-utils 2.4.5 → 2.6.0

package/src/cli/templates/development.template.json

@@ -1,64 +0,0 @@
-{
-  "$schema": "../config/ragrc.schema.json",
-  "metadata": {
-    "name": "dev-rag-pipeline",
-    "version": "1.0.0",
-    "description": "Development RAG pipeline with verbose logging and local models",
-    "environment": "development",
-    "template": "development"
-  },
-  "plugins": {
-    "loader": {
-      "file-loader": "latest"
-    },
-    "embedder": {
-      "local-embedder": {
-        "name": "local-embedder",
-        "version": "latest",
-        "config": {
-          "modelPath": "./models/embeddings",
-          "dimensions": 768
-        }
-      }
-    },
-    "retriever": {
-      "vector-retriever": {
-        "name": "vector-retriever",
-        "version": "latest",
-        "config": {
-          "topK": 5,
-          "threshold": 0.6
-        }
-      }
-    },
-    "llm": {
-      "local-llm": {
-        "name": "local-llm",
-        "version": "latest",
-        "config": {
-          "modelPath": "./models/llm",
-          "maxTokens": 500
-        }
-      }
-    }
-  },
-  "pipeline": {
-    "stages": ["loader", "embedder", "retriever", "llm"],
-    "timeout": 30000
-  },
-  "performance": {
-    "caching": {
-      "enabled": true,
-      "maxSize": 100,
-      "ttl": 600
-    }
-  },
-  "observability": {
-    "logging": {
-      "level": "debug",
-      "structured": true,
-      "events": true,
-      "console": true
-    }
-  }
-}

package/src/cli/templates/minimal.template.json

@@ -1,33 +0,0 @@
-{
-  "$schema": "../config/ragrc.schema.json",
-  "metadata": {
-    "name": "minimal-rag-pipeline",
-    "version": "1.0.0",
-    "description": "Minimal RAG pipeline configuration with essential plugins",
-    "environment": "development",
-    "template": "minimal"
-  },
-  "plugins": {
-    "loader": {
-      "file-loader": "latest"
-    },
-    "embedder": {
-      "openai-embedder": "latest"
-    },
-    "retriever": {
-      "vector-retriever": "latest"
-    },
-    "llm": {
-      "openai-llm": "latest"
-    }
-  },
-  "pipeline": {
-    "stages": ["loader", "embedder", "retriever", "llm"]
-  },
-  "observability": {
-    "logging": {
-      "level": "info",
-      "structured": true
-    }
-  }
-}

package/src/cli/templates/production.template.json

@@ -1,104 +0,0 @@
-{
-  "$schema": "../config/ragrc.schema.json",
-  "metadata": {
-    "name": "production-rag-pipeline",
-    "version": "1.0.0",
-    "description": "Production-grade RAG pipeline with full observability and optimization",
-    "environment": "production",
-    "template": "production"
-  },
-  "plugins": {
-    "loader": {
-      "file-loader": "latest"
-    },
-    "embedder": {
-      "openai-embedder": {
-        "name": "openai-embedder",
-        "version": "latest",
-        "config": {
-          "batchSize": 100,
-          "timeout": 30000,
-          "retryAttempts": 3
-        }
-      }
-    },
-    "retriever": {
-      "vector-retriever": {
-        "name": "vector-retriever",
-        "version": "latest",
-        "config": {
-          "topK": 10,
-          "threshold": 0.7,
-          "includeMetadata": true
-        }
-      }
-    },
-    "llm": {
-      "openai-llm": {
-        "name": "openai-llm",
-        "version": "latest",
-        "config": {
-          "model": "gpt-4",
-          "temperature": 0.7,
-          "maxTokens": 1000
-        }
-      }
-    },
-    "reranker": {
-      "similarity-reranker": "latest"
-    }
-  },
-  "pipeline": {
-    "stages": ["loader", "embedder", "retriever", "reranker", "llm"],
-    "retries": {
-      "enabled": true,
-      "maxAttempts": 3,
-      "backoff": "exponential"
-    },
-    "timeout": 60000
-  },
-  "performance": {
-    "caching": {
-      "enabled": true,
-      "maxSize": 1000,
-      "ttl": 3600
-    },
-    "parallel": {
-      "enabled": true,
-      "maxConcurrency": 5,
-      "batchSize": 10
-    },
-    "streaming": {
-      "enabled": true,
-      "maxMemoryMB": 512,
-      "bufferSize": 100
-    }
-  },
-  "observability": {
-    "logging": {
-      "level": "info",
-      "structured": true,
-      "events": true
-    },
-    "tracing": {
-      "enabled": true,
-      "sampleRate": 0.1,
-      "exportUrl": "http://localhost:4318"
-    },
-    "metrics": {
-      "enabled": true,
-      "interval": 60000,
-      "exportUrl": "http://localhost:4318"
-    }
-  },
-  "security": {
-    "encryption": {
-      "enabled": true,
-      "algorithm": "aes-256-gcm"
-    },
-    "authentication": {
-      "enabled": true,
-      "provider": "jwt"
-    }
-  }
-}

package/src/cli/templates/testing.template.json

@@ -1,45 +0,0 @@
-{
-  "$schema": "../config/ragrc.schema.json",
-  "metadata": {
-    "name": "test-rag-pipeline",
-    "version": "1.0.0",
-    "description": "Testing RAG pipeline with mock plugins",
-    "environment": "testing",
-    "template": "testing"
-  },
-  "plugins": {
-    "loader": {
-      "mock-loader": {
-        "source": "local",
-        "path": "./src/mocks/pdf-loader.js"
-      }
-    },
-    "embedder": {
-      "mock-embedder": {
-        "source": "local",
-        "path": "./src/mocks/openai-embedder.js"
-      }
-    },
-    "retriever": {
-      "mock-retriever": {
-        "source": "local",
-        "path": "./src/mocks/pinecone-retriever.js"
-      }
-    },
-    "llm": {
-      "mock-llm": {
-        "source": "local",
-        "path": "./src/mocks/openai-llm.js"
-      }
-    }
-  },
-  "pipeline": {
-    "stages": ["loader", "embedder", "retriever", "llm"]
-  },
-  "observability": {
-    "logging": {
-      "level": "warn",
-      "structured": true
-    }
-  }
-}

package/src/cli/wizard/configuration.js

@@ -1,148 +0,0 @@
-/**
- * Configuration Step
- * Handles general settings configuration (caching, timeout, pipeline stages)
- */
-
-/**
- * Configure general settings
- * @param {object} config - Current wizard configuration state
- * @param {object} inquirer - Inquirer instance for prompting
- * @returns {Promise<object>} Updated configuration state
- */
-async function configureSettings(config, inquirer) {
-  console.log("⚙️  General Settings\n");
-  // eslint-disable-line no-console
-
-  const answers = await inquirer.prompt([
-    {
-      _type: "confirm",
-      name: "enableCaching",
-      message: "Enable result caching?",
-      default: false,
-    },
-    {
-      _type: "number",
-      name: "cacheSize",
-      message: "Cache size (number of entries):",
-      default: 1000,
-      when: (answers) => answers.enableCaching,
-    },
-    {
-      _type: "number",
-      name: "cacheTtl",
-      message: "Cache TTL (seconds):",
-      default: 3600,
-      when: (answers) => answers.enableCaching,
-    },
-    {
-      _type: "number",
-      name: "timeout",
-      message: "Pipeline timeout (milliseconds):",
-      default: 30000,
-    },
-  ]);
-
-  if (answers.enableCaching) {
-    config.performance = config.performance || {};
-    config.performance.caching = {
-      enabled: true,
-      maxSize: answers.cacheSize,
-      ttl: answers.cacheTtl,
-    };
-  }
-
-  if (answers.timeout !== 30000) {
-    config.pipeline = config.pipeline || {};
-    config.pipeline.timeout = answers.timeout;
-  }
-
-  console.log("✅ General settings configured\n");
-  // eslint-disable-line no-console
-
-  return config;
-}
-
-/**
- * Configure pipeline stages
- * @param {object} config - Current wizard configuration state
- * @param {object} inquirer - Inquirer instance for prompting
- * @returns {Promise<object>} Updated configuration state
- */
-async function configurePipeline(config, inquirer) {
-  console.log("🔄 Pipeline Configuration\n");
-  // eslint-disable-line no-console
-
-  const availableStages = Object.keys(config.plugins).filter(
-    (_type) => Object.keys(config.plugins[_type]).length > 0,
-  );
-
-  const answers = await inquirer.prompt([
-    {
-      _type: "checkbox",
-      name: "stages",
-      message: "Select pipeline stages (in order):",
-      choices: [
-        {
-          name: "Document Loading",
-          value: "loader",
-          checked: availableStages.includes("loader"),
-        },
-        {
-          name: "Text Embedding",
-          value: "embedder",
-          checked: availableStages.includes("embedder"),
-        },
-        {
-          name: "Vector Retrieval",
-          value: "retriever",
-          checked: availableStages.includes("retriever"),
-        },
-        {
-          name: "Language Model",
-          value: "llm",
-          checked: availableStages.includes("llm"),
-        },
-        {
-          name: "Result Reranking",
-          value: "reranker",
-          checked: availableStages.includes("reranker"),
-        },
-      ],
-      validate: (input) =>
-        input.length > 0 || "At least one stage must be selected",
-    },
-    {
-      _type: "confirm",
-      name: "enableRetries",
-      message: "Enable automatic retries on failures?",
-      default: true,
-    },
-    {
-      _type: "number",
-      name: "maxRetries",
-      message: "Maximum retry attempts:",
-      default: 3,
-      when: (answers) => answers.enableRetries,
-    },
-  ]);
-
-  config.pipeline = {
-    stages: answers.stages,
-    ...config.pipeline,
-  };
-
-  if (answers.enableRetries) {
-    config.pipeline.retries = {
-      enabled: true,
-      maxAttempts: answers.maxRetries,
-      backoff: "exponential",
-    };
-  }
-
-  console.log("✅ Pipeline configuration complete\n");
-  // eslint-disable-line no-console
-
-  return config;
-}
-
-module.exports = { configureSettings, configurePipeline };

package/src/cli/wizard/observability.js

@@ -1,79 +0,0 @@
-/**
- * Observability Step
- * Handles observability settings configuration (logging, tracing, metrics)
- */
-
-/**
- * Configure observability settings
- * @param {object} config - Current wizard configuration state
- * @param {object} inquirer - Inquirer instance for prompting
- * @returns {Promise<object>} Updated configuration state
- */
-async function configureObservability(config, inquirer) {
-  console.log("📊 Observability Settings\n");
-  // eslint-disable-line no-console
-
-  const answers = await inquirer.prompt([
-    {
-      _type: "list",
-      name: "logLevel",
-      message: "Log level:",
-      choices: [
-        { name: "Debug (verbose)", value: "debug" },
-        { name: "Info (default)", value: "info" },
-        { name: "Warning (minimal)", value: "warn" },
-        { name: "Error (errors only)", value: "error" },
-      ],
-      default: "info",
-    },
-    {
-      _type: "confirm",
-      name: "enableTracing",
-      message: "Enable distributed tracing?",
-      default: false,
-    },
-    {
-      _type: "confirm",
-      name: "enableMetrics",
-      message: "Enable metrics collection?",
-      default: false,
-    },
-    {
-      _type: "input",
-      name: "exportUrl",
-      message: "Metrics/tracing export URL (optional):",
-      when: (answers) => answers.enableTracing || answers.enableMetrics,
-    },
-  ]);
-
-  config.observability = {
-    logging: {
-      level: answers.logLevel,
-      structured: true,
-      events: answers.enableTracing,
-    },
-  };
-
-  if (answers.enableTracing) {
-    config.observability.tracing = {
-      enabled: true,
-      exportUrl: answers.exportUrl || undefined,
-      sampleRate: 1,
-    };
-  }
-
-  if (answers.enableMetrics) {
-    config.observability.metrics = {
-      enabled: true,
-      exportUrl: answers.exportUrl || undefined,
-      interval: 60000,
-    };
-  }
-
-  console.log("✅ Observability settings configured\n");
-  // eslint-disable-line no-console
-
-  return config;
-}
-
-module.exports = { configureObservability };

package/src/cli/wizard/performance.js

@@ -1,76 +0,0 @@
-/**
- * Performance Step
- * Handles performance settings configuration (parallel processing, streaming)
- */
-
-/**
- * Configure performance settings
- * @param {object} config - Current wizard configuration state
- * @param {object} inquirer - Inquirer instance for prompting
- * @returns {Promise<object>} Updated configuration state
- */
-async function configurePerformance(config, inquirer) {
-  console.log("⚡ Performance Settings\n");
-  // eslint-disable-line no-console
-
-  const answers = await inquirer.prompt([
-    {
-      _type: "confirm",
-      name: "enableParallel",
-      message: "Enable parallel processing?",
-      default: false,
-    },
-    {
-      _type: "number",
-      name: "maxConcurrency",
-      message: "Maximum concurrent operations:",
-      default: 3,
-      when: (answers) => answers.enableParallel,
-    },
-    {
-      _type: "number",
-      name: "batchSize",
-      message: "Batch size for parallel processing:",
-      default: 10,
-      when: (answers) => answers.enableParallel,
-    },
-    {
-      _type: "confirm",
-      name: "enableStreaming",
-      message: "Enable streaming for large documents?",
-      default: false,
-    },
-    {
-      _type: "number",
-      name: "maxMemoryMB",
-      message: "Maximum memory usage (MB):",
-      default: 512,
-      when: (answers) => answers.enableStreaming,
-    },
-  ]);
-
-  config.performance = config.performance || {};
-
-  if (answers.enableParallel) {
-    config.performance.parallel = {
-      enabled: true,
-      maxConcurrency: answers.maxConcurrency,
-      batchSize: answers.batchSize,
-    };
-  }
-
-  if (answers.enableStreaming) {
-    config.performance.streaming = {
-      enabled: true,
-      maxMemoryMB: answers.maxMemoryMB,
-      bufferSize: 100,
-    };
-  }
-
-  console.log("✅ Performance settings configured\n");
-  // eslint-disable-line no-console
-
-  return config;
-}
-
-module.exports = { configurePerformance };