@devilsdev/rag-pipeline-utils 2.1.12 → 2.2.0

package/.ragrc.schema.json

@@ -0,0 +1,218 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://raw.githubusercontent.com/DevilsDev/rag-pipeline-utils/main/.ragrc.schema.json",
+  "title": "RAG Pipeline Configuration Schema",
+  "description": "Configuration schema for @devilsdev/rag-pipeline-utils",
+  "type": "object",
+  "required": ["plugins"],
+  "properties": {
+    "plugins": {
+      "type": "object",
+      "description": "Plugin specifications for RAG pipeline components",
+      "required": ["loader", "embedder", "retriever", "llm"],
+      "properties": {
+        "loader": {
+          "type": "object",
+          "description": "Document loader plugins",
+          "minProperties": 1,
+          "additionalProperties": {
+            "oneOf": [
+              { "type": "string" },
+              {
+                "type": "object",
+                "required": ["name"],
+                "properties": {
+                  "name": { "type": "string" },
+                  "version": { "type": "string" },
+                  "source": {
+                    "type": "string",
+                    "enum": ["registry", "local", "git", "npm"]
+                  },
+                  "path": { "type": "string" },
+                  "config": { "type": "object" },
+                  "enabled": { "type": "boolean" }
+                }
+              }
+            ]
+          }
+        },
+        "embedder": {
+          "type": "object",
+          "description": "Text embedding plugins",
+          "minProperties": 1,
+          "additionalProperties": {
+            "oneOf": [
+              { "type": "string" },
+              {
+                "type": "object",
+                "required": ["name"],
+                "properties": {
+                  "name": { "type": "string" },
+                  "version": { "type": "string" },
+                  "source": {
+                    "type": "string",
+                    "enum": ["registry", "local", "git", "npm"]
+                  },
+                  "path": { "type": "string" },
+                  "config": { "type": "object" },
+                  "enabled": { "type": "boolean" }
+                }
+              }
+            ]
+          }
+        },
+        "retriever": {
+          "type": "object",
+          "description": "Vector retrieval plugins",
+          "minProperties": 1,
+          "additionalProperties": {
+            "oneOf": [
+              { "type": "string" },
+              {
+                "type": "object",
+                "required": ["name"],
+                "properties": {
+                  "name": { "type": "string" },
+                  "version": { "type": "string" },
+                  "source": {
+                    "type": "string",
+                    "enum": ["registry", "local", "git", "npm"]
+                  },
+                  "path": { "type": "string" },
+                  "config": { "type": "object" },
+                  "enabled": { "type": "boolean" }
+                }
+              }
+            ]
+          }
+        },
+        "llm": {
+          "type": "object",
+          "description": "Language model plugins",
+          "minProperties": 1,
+          "additionalProperties": {
+            "oneOf": [
+              { "type": "string" },
+              {
+                "type": "object",
+                "required": ["name"],
+                "properties": {
+                  "name": { "type": "string" },
+                  "version": { "type": "string" },
+                  "source": {
+                    "type": "string",
+                    "enum": ["registry", "local", "git", "npm"]
+                  },
+                  "path": { "type": "string" },
+                  "config": { "type": "object" },
+                  "enabled": { "type": "boolean" }
+                }
+              }
+            ]
+          }
+        },
+        "reranker": {
+          "type": "object",
+          "description": "Result reranking plugins (optional)",
+          "minProperties": 1,
+          "additionalProperties": {
+            "oneOf": [
+              { "type": "string" },
+              {
+                "type": "object",
+                "required": ["name"],
+                "properties": {
+                  "name": { "type": "string" },
+                  "version": { "type": "string" },
+                  "source": {
+                    "type": "string",
+                    "enum": ["registry", "local", "git", "npm"]
+                  },
+                  "path": { "type": "string" },
+                  "config": { "type": "object" },
+                  "enabled": { "type": "boolean" }
+                }
+              }
+            ]
+          }
+        }
+      },
+      "additionalProperties": false
+    },
+    "pipeline": {
+      "type": "object",
+      "description": "Pipeline execution configuration",
+      "properties": {
+        "stages": {
+          "type": "array",
+          "description": "Pipeline execution stages",
+          "items": {
+            "type": "string",
+            "enum": ["loader", "embedder", "retriever", "llm", "reranker"]
+          }
+        },
+        "mode": {
+          "type": "string",
+          "description": "Pipeline execution mode",
+          "enum": ["sequential", "parallel", "dag"],
+          "default": "sequential"
+        }
+      }
+    },
+    "performance": {
+      "type": "object",
+      "description": "Performance and optimization settings",
+      "properties": {
+        "parallel": {
+          "type": "object",
+          "properties": {
+            "enabled": { "type": "boolean", "default": false },
+            "maxConcurrency": { "type": "number", "default": 5 },
+            "batchSize": { "type": "number", "default": 10 }
+          }
+        },
+        "streaming": {
+          "type": "object",
+          "properties": {
+            "enabled": { "type": "boolean", "default": false },
+            "maxMemoryMB": { "type": "number", "default": 512 },
+            "bufferSize": { "type": "number", "default": 100 }
+          }
+        }
+      }
+    },
+    "observability": {
+      "type": "object",
+      "description": "Logging and monitoring configuration",
+      "properties": {
+        "logging": {
+          "type": "object",
+          "properties": {
+            "level": {
+              "type": "string",
+              "enum": ["debug", "info", "warn", "error"],
+              "default": "info"
+            },
+            "structured": { "type": "boolean", "default": true }
+          }
+        },
+        "metrics": {
+          "type": "object",
+          "properties": {
+            "enabled": { "type": "boolean", "default": false },
+            "exportFormat": {
+              "type": "string",
+              "enum": ["prometheus", "json", "statsd"],
+              "default": "json"
+            }
+          }
+        }
+      }
+    },
+    "namespace": {
+      "type": "string",
+      "description": "Configuration namespace for isolation"
+    }
+  },
+  "additionalProperties": false
+}

package/README.md

@@ -8,6 +8,60 @@
 
 Enterprise-grade toolkit for building Retrieval-Augmented Generation (RAG) pipelines in Node.js with modular plugin architecture, streaming support, and observability.
 
+## 🎉 What's New in v2.2.0
+
+### Major Improvements
+
+**✅ Fixed ESM Build (Critical)**
+
+- Implemented CJS→ESM interop pattern for proper module compatibility
+- All 15 exports now working correctly in both CommonJS and ES Modules
+- Verified compatibility with Node.js 18.x, 20.x, and 22.x
+
+**📦 50% Smaller Install Size**
+
+- Reduced from ~30MB to ~15MB by optimizing dependencies
+- Moved CLI-only tools to devDependencies
+- Zero production vulnerabilities (npm audit clean)
+
+**📝 Enhanced TypeScript Support**
+
+- Added 5 complete plugin contract interfaces:
+  - `LoaderPlugin` - Document loading interface
+  - `EmbedderPlugin` - Text embedding interface
+  - `RetrieverPlugin` - Vector search interface
+  - `LLMPlugin` - Language model interface
+  - `RerankerPlugin` - Result reranking interface
+- Added 4 helper types: `Document`, `SearchResult`, `RetrieveOptions`, `LLMResponse`
+- 236 lines of comprehensive type definitions
+- Full IDE autocomplete and type checking support
+
+**🔧 Real-World Example**
+
+- Complete OpenAI + Pinecone integration example in `examples/openai-pinecone/`
+- Mock mode for testing without API keys
+- Production-ready error handling and cost estimation guidance
+- See [OpenAI + Pinecone Example](#openai--pinecone-example) below
+
+**📚 Documentation Enhancements**
+
+- Platform compatibility matrix (Linux, macOS, Windows, Docker)
+- Module system examples (CommonJS + ES Modules)
+- Docker usage examples
+- Troubleshooting guides
+
+### Migration from v2.1.x
+
+**No breaking changes!** v2.2.0 is fully backward compatible. Simply update:
+
+```bash
+npm install @devilsdev/rag-pipeline-utils@latest
+```
+
+All existing code will continue to work without modifications.
+
+---
+
 ## Installation
 
 ```bash
@@ -16,6 +70,128 @@ npm install @devilsdev/rag-pipeline-utils
 
 **Requirements:** Node.js >= 18.0.0
 
+## Compatibility
+
+### Platform Support
+
+| Platform    | Status             | Notes                                                         |
+| ----------- | ------------------ | ------------------------------------------------------------- |
+| **Linux**   | ✅ Fully Supported | Tested on Ubuntu 20.04+, Debian 11+, RHEL 8+                  |
+| **macOS**   | ✅ Fully Supported | Tested on macOS 12+ (Intel & Apple Silicon)                   |
+| **Windows** | ✅ Fully Supported | Tested on Windows 10/11, WSL2 recommended for best experience |
+| **Docker**  | ✅ Fully Supported | Alpine & Debian-based images available                        |
+
+### Module Systems
+
+This package supports both CommonJS and ES Modules:
+
+**CommonJS (require)**
+
+```javascript
+const {
+  createRagPipeline,
+  DAGEngine,
+} = require("@devilsdev/rag-pipeline-utils");
+
+const pipeline = createRagPipeline({
+  /* config */
+});
+const dag = new DAGEngine();
+```
+
+**ES Modules (import)**
+
+```javascript
+import { createRagPipeline, DAGEngine } from "@devilsdev/rag-pipeline-utils";
+
+const pipeline = createRagPipeline({
+  /* config */
+});
+const dag = new DAGEngine();
+```
+
+### TypeScript Support
+
+**New in v2.2.0:** Complete plugin contract interfaces with 236 lines of type definitions:
+
+```typescript
+import {
+  createRagPipeline,
+  DAGEngine,
+  RagPipelineConfig,
+  // Plugin contract interfaces (NEW in v2.2.0)
+  LoaderPlugin,
+  EmbedderPlugin,
+  RetrieverPlugin,
+  LLMPlugin,
+  RerankerPlugin,
+  // Helper types (NEW in v2.2.0)
+  Document,
+  SearchResult,
+  RetrieveOptions,
+  LLMResponse,
+} from "@devilsdev/rag-pipeline-utils";
+
+// Type-safe plugin implementation
+class MyLoader implements LoaderPlugin {
+  async load(source: string, options?: any): Promise<Document[]> {
+    // Your implementation with full type checking
+    return [
+      {
+        id: "doc-1",
+        content: "Document content",
+        metadata: { source },
+      },
+    ];
+  }
+}
+
+const config: RagPipelineConfig = {
+  loader: new MyLoader(),
+  embedder: "my-embedder",
+};
+
+const pipeline = createRagPipeline(config);
+```
+
+### Package Size & Dependencies
+
+- **Package size**: 244KB (gzipped), 1.1MB (unpacked)
+- **Runtime dependencies**: Only 11 production dependencies
+- **Total install size**: ~15MB (including dependencies) - **50% smaller than v2.1.x**
+- **Tree-shakeable**: ES module exports support tree-shaking
+- **Zero vulnerabilities**: npm audit clean (production dependencies)
+
+### Docker Usage
+
+**Basic Docker setup:**
+
+```dockerfile
+FROM node:18-alpine
+
+WORKDIR /app
+COPY package*.json ./
+RUN npm ci --production
+COPY . .
+
+CMD ["node", "index.js"]
+```
+
+**Docker Compose example:**
+
+```yaml
+version: "3.8"
+services:
+  rag-pipeline:
+    image: node:18-alpine
+    working_dir: /app
+    volumes:
+      - ./:/app
+    environment:
+      - NODE_ENV=production
+    command: npm start
+```
+
 ## Quick Start
 
 ### Basic Pipeline Setup
@@ -233,17 +409,25 @@ This package provides a plugin-based architecture. You need to implement plugins
 
 ### Plugin Contracts
 
+**New in v2.2.0:** TypeScript interfaces for all plugin contracts. See [TypeScript Support](#typescript-support) for full type definitions.
+
 #### Loader Plugin
 
 ```javascript
+/**
+ * Loader plugin interface (NEW TypeScript interface in v2.2.0)
+ * Implements: LoaderPlugin
+ */
 class MyLoader {
-  async load(filePath) {
+  async load(source, options) {
     // Load and parse document
-    // Return array of document chunks
+    // Return array of Document objects
     return [
       {
+        id: "doc-1",
         content: "Document text...",
-        metadata: { page: 1, source: filePath },
+        metadata: { page: 1, source },
+        // embedding: optional pre-computed embedding
       },
     ];
   }
@@ -253,19 +437,21 @@ class MyLoader {
 #### Embedder Plugin
 
 ```javascript
+/**
+ * Embedder plugin interface (NEW TypeScript interface in v2.2.0)
+ * Implements: EmbedderPlugin
+ */
 class MyEmbedder {
-  async embed(texts) {
-    // Generate embeddings for array of texts
-    // Return array of vectors
-    return [
-      [0.1, 0.2, 0.3, ...], // vector for text[0]
-      [0.4, 0.5, 0.6, ...]  // vector for text[1]
-    ];
+  async embed(text, options) {
+    // Generate embedding for a single text string
+    // Return: Promise<number[]> - embedding vector
+    return [0.1, 0.2, 0.3 /* ... 1536 dimensions for OpenAI */];
   }
 
-  async embedQuery(query) {
-    // Generate embedding for single query
-    return [0.7, 0.8, 0.9, ...];
+  // Optional batch embedding method
+  async embedBatch(texts) {
+    // Generate embeddings for multiple texts
+    return texts.map((text) => this.embed(text));
   }
 }
 ```
@@ -273,23 +459,31 @@ class MyEmbedder {
 #### Retriever Plugin
 
 ```javascript
+/**
+ * Retriever plugin interface (NEW TypeScript interface in v2.2.0)
+ * Implements: RetrieverPlugin
+ */
 class MyRetriever {
-  async store(vectors, metadata) {
-    // Store vectors with metadata in your vector database
-  }
-
-  async retrieve(queryVector, options = {}) {
-    const { topK = 5, threshold = 0.7 } = options;
+  async retrieve(query, options) {
+    // query can be:
+    // - string: search by text
+    // - { embedding: number[] }: search by vector
+    const { topK = 5, minScore = 0.7, filter } = options || {};
 
     // Search for similar vectors
-    // Return documents with similarity scores
+    // Return: Promise<SearchResult[]>
     return [
       {
-        content: "Document text...",
+        id: "result-1",
+        score: 0.95,
+        document: {
+          id: "doc-1",
+          content: "Document text...",
+          metadata: { source: "file.pdf" },
+        },
         metadata: {
-          /* document metadata */
+          /* additional search metadata */
         },
-        score: 0.95,
       },
     ];
   }
@@ -299,16 +493,30 @@ class MyRetriever {
 #### LLM Plugin
 
 ```javascript
+/**
+ * LLM plugin interface (NEW TypeScript interface in v2.2.0)
+ * Implements: LLMPlugin
+ */
 class MyLLM {
-  async generate(prompt, options = {}) {
+  async generate(prompt, options) {
     // Generate text from prompt
+    // Return: Promise<LLMResponse>
     return {
       text: "Generated response...",
-      usage: { tokens: 150 },
+      usage: {
+        promptTokens: 100,
+        completionTokens: 50,
+        totalTokens: 150,
+      },
+      metadata: {
+        model: "gpt-4",
+        finishReason: "stop",
+      },
     };
   }
 
-  async *stream(prompt, options = {}) {
+  // Optional streaming support
+  async *stream(prompt, options) {
     // Stream response chunks
     yield "Generated ";
     yield "response ";
@@ -320,13 +528,18 @@ class MyLLM {
 #### Reranker Plugin (Optional)
 
 ```javascript
+/**
+ * Reranker plugin interface (NEW TypeScript interface in v2.2.0)
+ * Implements: RerankerPlugin
+ */
 class MyReranker {
-  async rerank(query, documents, options = {}) {
-    // Rerank documents based on relevance to query
-    return documents
-      .map((doc) => ({
-        ...doc,
-        score: calculateRelevanceScore(query, doc),
+  async rerank(results, query, options) {
+    // Rerank search results based on relevance to query
+    // Return: Promise<SearchResult[]> - reranked results
+    return results
+      .map((result) => ({
+        ...result,
+        score: calculateRelevanceScore(query, result),
       }))
       .sort((a, b) => b.score - a.score);
   }
@@ -525,6 +738,32 @@ console.log("Average latency:", stats.queries?.avgLatency || 0);
 
 ## Common Integration Examples
 
+### OpenAI + Pinecone Example
+
+**New in v2.2.0:** Complete working example with OpenAI embeddings and Pinecone vector database.
+
+See the full example in [`examples/openai-pinecone/`](examples/openai-pinecone/) with:
+
+- ✅ Complete plugin implementations
+- ✅ Mock mode (no API keys required for testing)
+- ✅ Real OpenAI integration
+- ✅ Real Pinecone integration
+- ✅ Error handling and cost estimation
+- ✅ Comprehensive documentation
+
+**Quick start:**
+
+```bash
+cd examples/openai-pinecone
+npm install
+
+# Test with mock mode (no API keys needed)
+USE_MOCK_MODE=true npm start
+
+# Run with real APIs (requires API keys in .env)
+npm start
+```
+
 ### With OpenAI
 
 You need to implement OpenAI plugin wrappers:
@@ -638,11 +877,32 @@ const { logger } = require("@devilsdev/rag-pipeline-utils");
 logger.setLevel("debug");
 ```
 
+## Examples
+
+### Real-World Integration Examples
+
+**[OpenAI + Pinecone Example](examples/openai-pinecone/)** (NEW in v2.2.0)
+
+- Complete RAG pipeline implementation
+- OpenAI embeddings (text-embedding-3-small)
+- Pinecone vector database integration
+- Mock mode for testing without API keys
+- Production-ready with error handling
+
+**Basic Pipeline Example**
+
+- See `examples/basic-pipeline/` for minimal setup
+- Simple in-memory implementation
+- Perfect for learning and prototyping
+
 ## Documentation
 
 - [GitHub Repository](https://github.com/DevilsDev/rag-pipeline-utils)
 - [Issue Tracker](https://github.com/DevilsDev/rag-pipeline-utils/issues)
 - [Discussions](https://github.com/DevilsDev/rag-pipeline-utils/discussions)
+- [OpenAI + Pinecone Example](examples/openai-pinecone/) (NEW)
+- [Release Notes](https://github.com/DevilsDev/rag-pipeline-utils/releases)
+- [CHANGELOG](CHANGELOG.md)
 
 ## Contributing
 

package/bin/cli.js

@@ -1,14 +1,8 @@
-#!/usr/bin/env node
-
-/**
- * Enhanced RAG Pipeline CLI with comprehensive UX improvements
- * Integrates observability, plugin marketplace, interactive wizard, and diagnostics
- */
-
-import { runEnhancedCLI } from '../src/cli/enhanced-cli-commands.js';
-
-// Run the enhanced CLI
-runEnhancedCLI().catch(error => {
-  console.error('❌ CLI error:', error.message);
-  process.exit(1);
-});
+#!/usr/bin/env node
+"use strict";
+require("../src/cli.js")
+  .run(process.argv)
+  .catch((err) => {
+    console.error(err);
+    process.exit(1);
+  });