@devilsdev/rag-pipeline-utils 2.3.0 → 2.4.0

package/dist/index.d.ts

@@ -13,18 +13,36 @@ export interface RagPipelineConfig {
   retriever?: string | RetrieverPlugin;
   llm?: string | LLMPlugin;
   reranker?: string | RerankerPlugin;
+  registry?: PluginRegistry;
   [key: string]: any;
 }
 
-export interface LoadConfigOptions {
-  configPath?: string;
-  validate?: boolean;
+export interface PipelineRunOptions {
+  query?: string;
+  queryVector?: number[];
+  options?: {
+    timeout?: number;
+    stream?: boolean;
+    topK?: number;
+    [key: string]: any;
+  };
 }
 
-export interface PipelineExecuteOptions {
+export interface PipelineRunResult {
+  success: boolean;
   query?: string;
-  documents?: string[];
-  [key: string]: any;
+  results?: any;
+  error?: string;
+}
+
+export interface Pipeline {
+  run(options: PipelineRunOptions): Promise<PipelineRunResult>;
+  cleanup(): void;
+}
+
+export interface LoadConfigOptions {
+  configPath?: string;
+  validate?: boolean;
 }
 
 // =============================================================================
@@ -66,69 +84,23 @@ export interface LLMResponse {
 // Plugin Contract Interfaces
 // =============================================================================
 
-/**
- * Loader plugin interface for loading documents from various sources
- */
 export interface LoaderPlugin {
-  /**
-   * Load documents from a source
-   * @param source - Path or identifier of the source
-   * @param options - Optional loading parameters
-   * @returns Promise resolving to array of documents
-   */
   load(source: string, options?: any): Promise<Document[]>;
 }
 
-/**
- * Embedder plugin interface for generating vector embeddings
- */
 export interface EmbedderPlugin {
-  /**
-   * Generate embedding vector for text
-   * @param text - Input text to embed
-   * @param options - Optional embedding parameters
-   * @returns Promise resolving to embedding vector
-   */
   embed(text: string, options?: any): Promise<number[]>;
 }
 
-/**
- * Retriever plugin interface for vector search and document retrieval
- */
 export interface RetrieverPlugin {
-  /**
-   * Retrieve relevant documents based on query
-   * @param query - Search query (text or embedding)
-   * @param options - Retrieval options (topK, filters, etc.)
-   * @returns Promise resolving to search results
-   */
   retrieve(query: string | { embedding: number[] }, options?: RetrieveOptions): Promise<SearchResult[]>;
 }
 
-/**
- * LLM plugin interface for text generation
- */
 export interface LLMPlugin {
-  /**
-   * Generate text response based on prompt
-   * @param prompt - Input prompt for generation
-   * @param options - Generation parameters (temperature, maxTokens, etc.)
-   * @returns Promise resolving to LLM response
-   */
   generate(prompt: string, options?: any): Promise<LLMResponse>;
 }
 
-/**
- * Reranker plugin interface for improving retrieval relevance
- */
 export interface RerankerPlugin {
-  /**
-   * Rerank search results based on query
-   * @param results - Initial search results
-   * @param query - Original query
-   * @param options - Reranking parameters
-   * @returns Promise resolving to reranked results
-   */
   rerank(results: SearchResult[], query: string, options?: any): Promise<SearchResult[]>;
 }
 
@@ -161,7 +133,7 @@ export interface DAGOptions {
 // =============================================================================
 
 export interface PluginRegistry {
-  register(type: string, name: string, implementation: any): void;
+  register(type: string, name: string, implementation: any, contract?: any): void;
   get(type: string, name: string): any;
   has(type: string, name: string): boolean;
   list(type: string): any[];
@@ -174,12 +146,36 @@ export interface Logger {
   debug(message: string, meta?: any): void;
 }
 
+export interface ErrorFormatter {
+  formatError(error: Error): string;
+  createError(code: string, message: string, details?: any): Error;
+  wrapError(error: Error, context: string): Error;
+  ERROR_CODES: Record<string, string>;
+}
+
+// =============================================================================
+// Evaluation Types
+// =============================================================================
+
+export interface EvalResult {
+  question: string;
+  expected: string;
+  actual: string;
+  score: number;
+  bleu?: number;
+  rouge?: number;
+}
+
+// =============================================================================
 // Core API
-export function createRagPipeline(config: RagPipelineConfig): any;
+// =============================================================================
+
+export function createRagPipeline(config: RagPipelineConfig): Pipeline;
 export { createRagPipeline as createPipeline };
 
 // Configuration
 export function loadConfig(options?: LoadConfigOptions): Promise<RagPipelineConfig>;
+export { loadConfig as loadRagConfig };
 export function validateRagrc(config: any): boolean;
 export function normalizeConfig(config: any): RagPipelineConfig;
 
@@ -188,36 +184,174 @@ export const pluginRegistry: PluginRegistry;
 
 // Utilities
 export const logger: Logger;
+export const errorFormatter: ErrorFormatter;
+export function createError(code: string, message: string, details?: any): Error;
+export function wrapError(error: Error, context: string): Error;
+export const ERROR_CODES: Record<string, string>;
 
-// DAG Engine
-export class DAGEngine {
-  constructor(options?: DAGOptions);
-  addNode(id: string, fn: (input: any) => Promise<any>, options?: Partial<DAGNode>): void;
-  connect(fromId: string, toId: string): void;
-  execute(seed?: any, options?: DAGOptions): Promise<Map<string, any>>;
-  validate(): void;
+// =============================================================================
+// Utility Classes
+// =============================================================================
+
+export class BatchProcessor {
+  constructor(options?: {
+    batchSize?: number;
+    maxConcurrent?: number;
+    retryAttempts?: number;
+    onProgress?: (progress: any) => void;
+    [key: string]: any;
+  });
+  process(items: any[], handler: (batch: any[]) => Promise<any[]>): Promise<any[]>;
+  cancel(): void;
+}
+
+export class RateLimiter {
+  constructor(options?: {
+    maxRequests?: number;
+    windowMs?: number;
+    [key: string]: any;
+  });
+  tryAcquire(identifier?: string): boolean;
+  reset(identifier?: string): void;
+}
+
+export function retry<T>(fn: () => Promise<T>, options?: {
+  maxAttempts?: number;
+  delay?: number;
+  backoff?: number;
+  retryIf?: (error: Error) => boolean;
+}): Promise<T>;
+
+export function withRetry<T>(fn: () => Promise<T>, options?: {
+  maxAttempts?: number;
+  delay?: number;
+  backoff?: number;
+}): Promise<T>;
+
+export class ConnectionPoolManager {
+  constructor(options?: {
+    maxSockets?: number;
+    maxFreeSockets?: number;
+    timeout?: number;
+    [key: string]: any;
+  });
+  getPool(hostname: string): any;
+  getMetrics(): any;
+  destroy(): void;
 }
 
+// =============================================================================
+// Security
+// =============================================================================
+
+export class JWTValidator {
+  constructor(options?: {
+    secret?: string;
+    algorithms?: string[];
+    issuer?: string;
+    audience?: string;
+    clockTolerance?: number;
+    maxAge?: string;
+    [key: string]: any;
+  });
+  sign(payload: any, options?: any): string;
+  verify(token: string, options?: any): any;
+}
+
+export class PluginSandbox {
+  constructor(options?: {
+    memoryLimit?: number;
+    timeout?: number;
+    permissions?: string[];
+    [key: string]: any;
+  });
+  execute(code: string, context?: any): Promise<any>;
+}
+
+export class PluginSignatureVerifier {
+  constructor(options?: {
+    trustedKeysPath?: string;
+    failClosed?: boolean;
+    [key: string]: any;
+  });
+  verify(pluginPath: string): Promise<boolean>;
+}
+
+export function validateInput(input: string, options?: any): { valid: boolean; errors?: string[] };
+
+// =============================================================================
 // AI/ML
+// =============================================================================
+
 export class MultiModalProcessor {
   constructor(options?: any);
-  process(input: any): Promise<any>;
+  processContent(tenantId: string, content: any): Promise<any>;
+  generateEmbeddings(tenantId: string, content: any): Promise<any>;
+  crossModalSearch(tenantId: string, query: any): Promise<any>;
 }
 
 export class AdaptiveRetrievalEngine {
   constructor(options?: any);
-  retrieve(query: string, options?: any): Promise<any>;
+  retrieveDocuments(tenantId: string, query: string, options?: any): Promise<any>;
+  optimizeRetrieval(tenantId: string, performanceData: any): Promise<any>;
 }
 
-// Performance
+export class FederatedLearningCoordinator {
+  constructor(options?: any);
+  createSession(config: any): Promise<any>;
+  submitUpdate(sessionId: string, update: any): Promise<any>;
+  aggregate(sessionId: string): Promise<any>;
+}
+
+export class ModelTrainingOrchestrator {
+  constructor(options?: any);
+  createTrainingJob(tenantId: string, config: any): Promise<any>;
+  getJobStatus(jobId: string): Promise<any>;
+  optimizeModel(config: any): Promise<any>;
+  deployModel(jobId: string, deployConfig: any): Promise<any>;
+}
+
+// =============================================================================
+// Workflow Engine
+// =============================================================================
+
+export class DAGEngine {
+  constructor(options?: DAGOptions);
+  addNode(id: string, fn: (input: any) => Promise<any>, options?: Partial<DAGNode>): void;
+  connect(fromId: string, toId: string): void;
+  execute(seed?: any, options?: DAGOptions): Promise<Map<string, any>>;
+  validate(): void;
+}
+
+export { DAGEngine as DAG };
+
+// =============================================================================
+// Evaluation
+// =============================================================================
+
+export function evaluateRagDataset(dataset: Array<{
+  question: string;
+  expected: string;
+  actual: string;
+}>): EvalResult[];
+
+export function scoreAnswer(expected: string, actual: string): number;
+export function computeBLEU(reference: string, hypothesis: string): number;
+export function computeROUGE(reference: string, hypothesis: string): number;
+
+// =============================================================================
+// Performance & Observability
+// =============================================================================
+
 export class ParallelProcessor {
   constructor(options?: any);
   process(items: any[], handler: (item: any) => Promise<any>): Promise<any[]>;
 }
 
-// Observability
 export const eventLogger: {
   log(event: string, data?: any): void;
+  info(event: string, data?: any): void;
+  error(event: string, data?: any): void;
 };
 
 export const metrics: {
@@ -225,13 +359,109 @@ export const metrics: {
   timer(name: string): { end(): void };
 };
 
+// =============================================================================
 // Enterprise
+// =============================================================================
+
 export class AuditLogger {
   constructor(options?: any);
-  log(event: string, data?: any): void;
+  logAuthentication(event: any): Promise<void>;
+  logDataAccess(event: any): Promise<void>;
+  logDataModification(event: any): Promise<void>;
 }
 
 export class DataGovernance {
   constructor(options?: any);
-  enforce(policy: any): void;
-}
+  classifyData(data: any): any;
+  enforceRetention(policy: any): Promise<void>;
+}
+
+export { DataGovernance as DataGovernanceManager };
+
+export class TenantManager {
+  constructor(options?: any);
+  createTenant(config: any): Promise<any>;
+  getTenant(tenantId: string): any;
+  deleteTenant(tenantId: string): Promise<void>;
+}
+
+export class ResourceMonitor {
+  constructor(options?: any);
+  getUsage(tenantId: string): any;
+  checkQuota(tenantId: string): boolean;
+}
+
+export class SSOManager {
+  constructor(options?: any);
+  authenticate(provider: string, credentials: any): Promise<any>;
+  getSession(sessionId: string): any;
+}
+
+export class SAMLProvider {
+  constructor(options?: any);
+  authenticate(request: any): Promise<any>;
+}
+
+export class OAuth2Provider {
+  constructor(options?: any);
+  authenticate(request: any): Promise<any>;
+}
+
+export class ActiveDirectoryProvider {
+  constructor(options?: any);
+  authenticate(request: any): Promise<any>;
+}
+
+export class OIDCProvider {
+  constructor(options?: any);
+  authenticate(request: any): Promise<any>;
+}
+
+// =============================================================================
+// Ecosystem
+// =============================================================================
+
+export class PluginHub {
+  constructor(options?: any);
+  search(query: string, options?: any): Promise<any[]>;
+  install(pluginName: string, options?: any): Promise<any>;
+  publish(pluginPath: string, options?: any): Promise<any>;
+}
+
+export class PluginAnalytics {
+  constructor(options?: any);
+  trackDownload(pluginName: string): void;
+  getStats(pluginName: string): any;
+}
+
+export class PluginCertification {
+  constructor(options?: any);
+  certify(pluginName: string): Promise<any>;
+  verify(pluginName: string): Promise<boolean>;
+}
+
+export class PluginAnalyticsDashboard {
+  constructor(options?: any);
+  generateReport(options?: any): Promise<any>;
+  getMetrics(): any;
+}
+
+// =============================================================================
+// Development Tools
+// =============================================================================
+
+export class HotReloadManager {
+  constructor(options?: any);
+  watch(paths: string[]): void;
+  stop(): void;
+}
+
+export function createHotReloadManager(options?: any): HotReloadManager;
+
+export class DevServer {
+  constructor(options?: any);
+  start(port?: number): Promise<void>;
+  stop(): Promise<void>;
+}
+
+export function createDevServer(options?: any): DevServer;

package/dist/index.mjs

@@ -1,80 +1,13 @@
-/**
- * RAG Pipeline Utils - ESM Build
- * Main entry point for ES module consumers
- */
+import { createRequire } from 'module';
+import { fileURLToPath } from 'url';
+import { dirname } from 'path';
 
-/**
- * RAG Pipeline Utils - Public API
- * Main entry point for consumers of the library
- */
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const require = createRequire(import.meta.url);
 
-// Core pipeline functionality
-import createpipelineModule from '../src/core/create-pipeline.js';
-const { createRagPipeline  } = createpipelineModule;
+const mod = require('./index.cjs');
 
-// Configuration utilities
-import loadconfigModule from '../src/config/load-config.js';
-const { loadConfig  } = loadconfigModule;
-import enhancedragrcschemaModule from '../src/config/enhanced-ragrc-schema.js';
-const { validateRagrc  } = enhancedragrcschemaModule;
-import normalizeconfigModule from '../src/config/normalize-config.js';
-const { normalizeConfig  } = normalizeconfigModule;
+export const { createRagPipeline, createPipeline, loadConfig, loadRagConfig, validateRagrc, normalizeConfig, pluginRegistry, logger, errorFormatter, createError, wrapError, ERROR_CODES, BatchProcessor, RateLimiter, retry, withRetry, ConnectionPoolManager, JWTValidator, PluginSandbox, PluginSignatureVerifier, validateInput, MultiModalProcessor, AdaptiveRetrievalEngine, FederatedLearningCoordinator, ModelTrainingOrchestrator, DAGEngine, DAG, evaluateRagDataset, scoreAnswer, computeBLEU, computeROUGE, computeFaithfulness, computeAnswerRelevance, computeContextPrecision, computeContextRecall, computeGroundedness, PipelineEvaluator, ChunkingEngine, CitationTracker, mapSentenceToSources, buildIDFWeights, detectHallucinations, BM25Search, reciprocalRankFusion, HybridRetriever, PreRetrievalGuard, RetrievalGuard, PostGenerationGuard, GuardrailsPipeline, QueryPlanner, IterativeRetriever, SelfCritiqueChecker, AgenticPipeline, CostCalculator, CostTracker, TokenBudget, ExecutionTracer, TraceInspector, MCPServer, MCPToolBuilder, ProjectScaffolder, TemplateRegistry, SSEAdapter, WebSocketAdapter, StreamRouter, BaseConnector, OllamaConnector, LocalEmbedder, MemoryRetriever, OpenAIConnector, AnthropicConnector, CohereConnector, StreamingEmbedder, createEmbeddingStream, ScoringReranker, EmbeddingReranker, CascadeReranker, KnowledgeGraph, EntityExtractor, GraphRetriever, GraphIndex, DashboardGenerator, MetricsAggregator, ParallelProcessor, eventLogger, metrics, AuditLogger, DataGovernance, DataGovernanceManager, TenantManager, ResourceMonitor, SSOManager, SAMLProvider, OAuth2Provider, ActiveDirectoryProvider, OIDCProvider, PluginHub, PluginAnalytics, PluginCertification, PluginAnalyticsDashboard, HotReloadManager, createHotReloadManager, DevServer, createDevServer } = mod;
 
-// Plugin system
-import pluginRegistry from '../src/core/plugin-registry.js';
-
-// Utilities
-import logger from '../src/utils/logger.js';
-import errorFormatter from '../src/utils/error-formatter.js';
-
-// AI/ML capabilities
-import MultiModalProcessor from '../src/ai/multimodal.js';
-import AdaptiveRetrievalEngine from '../src/ai/retrieval-engine.js';
-
-// DAG engine for complex workflows
-import dagengineModule from '../src/dag/dag-engine.js';
-const { DAGEngine  } = dagengineModule;
-
-// Performance and observability
-import ParallelProcessor from '../src/core/performance/parallel-processor.js';
-import eventLogger from '../src/core/observability/event-logger.js';
-import metrics from '../src/core/observability/metrics.js';
-
-// Enterprise features
-import auditloggingModule from '../src/enterprise/audit-logging.js';
-const { AuditLogger  } = auditloggingModule;
-import datagovernanceModule from '../src/enterprise/data-governance.js';
-const { DataGovernance  } = datagovernanceModule;
-
-// Development tools
-import hotreloadModule from '../src/dev/hot-reload.js';
-const { HotReloadManager,
-  createHotReloadManager,
- } = hotreloadModule;
-import devserverModule from '../src/dev/dev-server.js';
-const { DevServer, createDevServer  } = devserverModule;
-
-export {
-  createRagPipeline,
-  loadConfig,
-  validateRagrc,
-  normalizeConfig,
-  pluginRegistry,
-  logger,
-  errorFormatter,
-  MultiModalProcessor,
-  AdaptiveRetrievalEngine,
-  DAGEngine,
-  ParallelProcessor,
-  eventLogger,
-  metrics,
-  AuditLogger,
-  DataGovernance,
-  HotReloadManager,
-  createHotReloadManager,
-  DevServer,
-  createDevServer
-};
-
-// Backward compatibility aliases
-export const createPipeline = createRagPipeline;
+export default mod;

package/package.json

@@ -1,15 +1,15 @@
 {
   "name": "@devilsdev/rag-pipeline-utils",
-  "version": "2.3.0",
+  "version": "2.4.0",
   "description": "A comprehensive enterprise-grade toolkit for building RAG (Retrieval-Augmented Generation) pipelines with advanced AI/ML capabilities",
   "main": "dist/index.cjs",
   "type": "commonjs",
   "types": "dist/index.d.ts",
   "exports": {
     ".": {
+      "types": "./dist/index.d.ts",
       "require": "./dist/index.cjs",
-      "import": "./dist/index.mjs",
-      "types": "./dist/index.d.ts"
+      "import": "./dist/index.mjs"
     },
     "./cli": "./bin/cli.js"
   },
@@ -22,7 +22,9 @@
   "scripts": {
     "build": "node scripts/build.js && node scripts/generate-types.js",
     "prepack": "npm run build",
-    "prepublishOnly": "npm run lint && npm test",
+    "docs:validate": "node scripts/validate-docs-examples.js",
+    "verify:esm": "node scripts/verify-esm.mjs",
+    "prepublishOnly": "npm run lint && npm test && npm run docs:validate && npm run verify:esm",
     "guard:cjs": "node scripts/guard-cjs.js",
     "pretest": "npm run guard:cjs && node scripts/setup.js",
     "test": "cross-env NODE_OPTIONS=--experimental-vm-modules jest --runInBand --config jest.config.js",
@@ -50,7 +52,7 @@
     "watch:lqip": "node docs-site/scripts/watch-images.js",
     "docs:clear": "docusaurus clear --site-dir docs-site",
     "docs:start": "docusaurus start docs-site",
-    "docs:build": "docusaurus build docs-site",
+    "docs:build": "cd docs-site && npm run build",
     "docs:serve": "docusaurus serve docs-site",
     "docs:deploy": "docusaurus deploy docs-site",
     "roadmap:close": "node scripts/close-done-roadmap-issues.js",
@@ -121,6 +123,16 @@
   "files": [
     "dist/",
     "src/",
+    "!src/mocks/",
+    "!src/ai/module-template.js",
+    "!src/ai/index-fixed.js",
+    "!src/dev/examples/",
+    "!src/dev/README.md",
+    "!src/utils/examples/",
+    "!src/utils/ci/",
+    "!src/ci/",
+    "!src/scripts/",
+    "!src/cli/plugin-marketplace-commands.js",
     "bin/",
     "contracts/",
     ".ragrc.schema.json",
@@ -137,8 +149,8 @@
   "devDependencies": {
     "@babel/core": "^7.26.10",
     "@babel/preset-env": "^7.26.9",
-    "@docusaurus/core": "^3.7.0",
-    "@docusaurus/preset-classic": "^3.7.0",
+    "@docusaurus/core": "3.5.2",
+    "@docusaurus/preset-classic": "3.5.2",
     "@octokit/rest": "^21.1.1",
     "@opentelemetry/api": "^1.9.0",
     "@opentelemetry/auto-instrumentations-node": "^0.52.1",
@@ -159,7 +171,7 @@
     "@types/jest": "^29.5.14",
     "acorn": "8.15.0",
     "babel-jest": "^29.7.0",
-    "codecov": "^3.8.2",
+    "codecov": "3.6.2",
     "conventional-changelog-conventionalcommits": "^8.0.0",
     "cross-env": "7.0.3",
     "esbuild": "^0.25.2",