@devilsdev/rag-pipeline-utils 2.1.7 → 2.1.9

package/contracts/embedder-contract.json

@@ -0,0 +1,52 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "embedder",
+  "version": "1.0.0",
+  "title": "Embedder Contract",
+  "description": "Contract for embedder plugins",
+  "methods": [
+    {
+      "name": "embed",
+      "parameters": ["chunks"],
+      "returns": "vectors"
+    }
+  ],
+  "properties": {
+    "embed": {
+      "type": "function",
+      "description": "Generate embeddings for text chunks",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "chunks": {
+            "type": "array",
+            "items": {
+              "type": "object",
+              "properties": {
+                "id": { "type": "string" },
+                "content": { "type": "string" }
+              },
+              "required": ["id", "content"]
+            }
+          }
+        },
+        "required": ["chunks"]
+      },
+      "returns": {
+        "type": "array",
+        "items": {
+          "type": "object",
+          "properties": {
+            "id": { "type": "string" },
+            "vector": {
+              "type": "array",
+              "items": { "type": "number" }
+            }
+          },
+          "required": ["id", "vector"]
+        }
+      }
+    }
+  },
+  "required": ["embed"]
+}

package/contracts/llm-contract.json

@@ -0,0 +1,36 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "llm",
+  "version": "1.0.0",
+  "title": "LLM Contract",
+  "description": "Contract for LLM plugins",
+  "methods": [
+    {
+      "name": "generate",
+      "parameters": ["prompt"],
+      "returns": "response"
+    }
+  ],
+  "properties": {
+    "generate": {
+      "type": "function",
+      "description": "Generate text response",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "prompt": { "type": "string" },
+          "context": {
+            "type": "array",
+            "items": { "type": "string" }
+          },
+          "options": { "type": "object" }
+        },
+        "required": ["prompt"]
+      },
+      "returns": {
+        "type": "string"
+      }
+    }
+  },
+  "required": ["generate"]
+}

package/contracts/loader-contract.json

@@ -0,0 +1,43 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "loader",
+  "version": "1.0.0",
+  "title": "Loader Contract",
+  "description": "Contract for loader plugins",
+  "methods": [
+    {
+      "name": "load",
+      "parameters": ["source"],
+      "returns": "documents"
+    }
+  ],
+  "properties": {
+    "load": {
+      "type": "function",
+      "description": "Load documents from source",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "source": {
+            "type": "string",
+            "description": "Source path or identifier"
+          }
+        },
+        "required": ["source"]
+      },
+      "returns": {
+        "type": "array",
+        "items": {
+          "type": "object",
+          "properties": {
+            "id": { "type": "string" },
+            "content": { "type": "string" },
+            "metadata": { "type": "object" }
+          },
+          "required": ["id", "content"]
+        }
+      }
+    }
+  },
+  "required": ["load"]
+}

package/contracts/retriever-contract.json

@@ -0,0 +1,68 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "retriever",
+  "version": "1.0.0",
+  "title": "Retriever Contract",
+  "description": "Contract for retriever plugins",
+  "methods": [
+    {
+      "name": "store",
+      "parameters": ["vectors"],
+      "returns": "success"
+    },
+    {
+      "name": "retrieve",
+      "parameters": ["query", "k"],
+      "returns": "results"
+    }
+  ],
+  "properties": {
+    "store": {
+      "type": "function",
+      "description": "Store vectors in the retrieval system",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "vectors": {
+            "type": "array",
+            "items": {
+              "type": "object",
+              "properties": {
+                "id": { "type": "string" },
+                "vector": { "type": "array", "items": { "type": "number" } },
+                "metadata": { "type": "object" }
+              },
+              "required": ["id", "vector"]
+            }
+          }
+        },
+        "required": ["vectors"]
+      }
+    },
+    "retrieve": {
+      "type": "function",
+      "description": "Retrieve relevant vectors",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "query": { "type": "string" },
+          "k": { "type": "number", "default": 10 }
+        },
+        "required": ["query"]
+      },
+      "returns": {
+        "type": "array",
+        "items": {
+          "type": "object",
+          "properties": {
+            "id": { "type": "string" },
+            "score": { "type": "number" },
+            "metadata": { "type": "object" }
+          },
+          "required": ["id", "score"]
+        }
+      }
+    }
+  },
+  "required": ["store", "retrieve"]
+}

package/dist/index.cjs

@@ -0,0 +1,67 @@
+/**
+ * RAG Pipeline Utils - Public API
+ * Main entry point for consumers of the library
+ */
+
+// Core pipeline functionality
+const { createRagPipeline } = require('../src/core/create-pipeline');
+
+// Configuration utilities
+const { loadConfig } = require('../src/config/load-config');
+const { validateRagrc } = require('../src/config/enhanced-ragrc-schema');
+const { normalizeConfig } = require('../src/config/normalize-config');
+
+// Plugin system
+const pluginRegistry = require('../src/core/plugin-registry');
+
+// Utilities
+const logger = require('../src/utils/logger');
+
+// AI/ML capabilities
+const MultiModalProcessor = require('../src/ai/multimodal');
+const AdaptiveRetrievalEngine = require('../src/ai/retrieval-engine');
+
+// DAG engine for complex workflows
+const { DAGEngine } = require('../src/dag/dag-engine');
+
+// Performance and observability
+const ParallelProcessor = require('../src/core/performance/parallel-processor');
+const eventLogger = require('../src/core/observability/event-logger');
+const metrics = require('../src/core/observability/metrics');
+
+// Enterprise features
+const { AuditLogger } = require('../src/enterprise/audit-logging');
+const { DataGovernance } = require('../src/enterprise/data-governance');
+
+module.exports = {
+  // Core API
+  createRagPipeline,
+  createPipeline: createRagPipeline, // Alias for backward compatibility
+
+  // Configuration
+  loadConfig,
+  validateRagrc,
+  normalizeConfig,
+
+  // Plugin system
+  pluginRegistry,
+
+  // Utilities
+  logger,
+
+  // AI/ML
+  MultiModalProcessor,
+  AdaptiveRetrievalEngine,
+
+  // Workflow engine
+  DAGEngine,
+
+  // Performance & Observability
+  ParallelProcessor,
+  eventLogger,
+  metrics,
+
+  // Enterprise
+  AuditLogger,
+  DataGovernance,
+};

package/dist/index.mjs

@@ -0,0 +1,51 @@
+/**
+ * RAG Pipeline Utils - ESM Build
+ * Main entry point for ES module consumers
+ */
+
+/**
+ * RAG Pipeline Utils - Public API
+ * Main entry point for consumers of the library
+ */
+
+// Core pipeline functionality
+import { createRagPipeline  } from '../src/core/create-pipeline.js';
+
+// Configuration utilities
+import { loadConfig  } from '../src/config/load-config.js';
+import { validateRagrc  } from '../src/config/enhanced-ragrc-schema.js';
+import { normalizeConfig  } from '../src/config/normalize-config.js';
+
+// Plugin system
+import pluginRegistry from '../src/core/plugin-registry.js';
+
+// Utilities
+import logger from '../src/utils/logger.js';
+
+// AI/ML capabilities
+import MultiModalProcessor from '../src/ai/multimodal.js';
+import AdaptiveRetrievalEngine from '../src/ai/retrieval-engine.js';
+
+// DAG engine for complex workflows
+import { DAGEngine  } from '../src/dag/dag-engine.js';
+
+// Performance and observability
+import ParallelProcessor from '../src/core/performance/parallel-processor.js';
+import eventLogger from '../src/core/observability/event-logger.js';
+import metrics from '../src/core/observability/metrics.js';
+
+// Enterprise features
+import { AuditLogger  } from '../src/enterprise/audit-logging.js';
+import { DataGovernance  } from '../src/enterprise/data-governance.js';
+
+export {
+  validateRagrc,
+  normalizeConfig,
+  AdaptiveRetrievalEngine,
+  eventLogger,
+  metrics,
+  DataGovernance
+};
+
+// Backward compatibility aliases
+export const createPipeline = createRagPipeline;

package/package.json

@@ -1,20 +1,43 @@
 {
   "name": "@devilsdev/rag-pipeline-utils",
-  "version": "2.1.7",
-  "description": "A modular toolkit for building RAG (Retrieval-Augmented Generation) pipelines in Node.js",
-  "main": "src/core/create-pipeline.js",
+  "version": "2.1.9",
+  "description": "A comprehensive enterprise-grade toolkit for building RAG (Retrieval-Augmented Generation) pipelines with advanced AI/ML capabilities",
+  "main": "dist/index.cjs",
+  "type": "commonjs",
+  "exports": {
+    ".": {
+      "require": "./dist/index.cjs",
+      "import": "./dist/index.mjs"
+    },
+    "./cli": "./bin/cli.js"
+  },
   "bin": {
-    "rag-pipeline": "./bin/cli.js"
+    "rag-pipeline": "bin/cli.js"
+  },
+  "engines": {
+    "node": ">=18"
   },
-  "type": "module",
   "scripts": {
-    "validate-fixtures": "node ./scripts/verify-fixtures.js",
-    "setup": "node ./scripts/setup.js",
-    "test": "node ./scripts/setup.js && cross-env NODE_OPTIONS=--experimental-vm-modules jest --coverage",
+    "build": "node scripts/build.js",
+    "prepack": "npm run build",
+    "prepublishOnly": "echo 'Package ready for publishing'",
+    "guard:cjs": "node scripts/guard-cjs.js",
+    "pretest": "npm run guard:cjs && node scripts/setup.js",
+    "test": "cross-env NODE_OPTIONS=--experimental-vm-modules jest --runInBand --config jest.config.js",
+    "test:ci": "jest --coverage",
+    "test:open-handles": "jest --detectOpenHandles --runInBand",
+    "test:report": "node scripts/test-reporter.js",
+    "posttest": "node scripts/generate-test-reports.js",
+    "format": "prettier --write \"src/**/*.{js,json}\"",
     "test:watch": "jest --watch",
-    "lint": "eslint . --ext .js,.jsx,.ts,.tsx",
-    "lint:fix": "eslint . --ext .js,.jsx,.ts,.tsx --fix",
-    "prepare": "husky install",
+    "lint": "eslint src --ext .js --ignore-path .gitignore",
+    "lint:fix": "eslint . --ext .js,.jsx --ignore-path .gitignore --fix",
+    "lint:strict": "eslint . --ext .js,.jsx --ignore-path .gitignore --max-warnings 0",
+    "lint:errors-only": "eslint . --ext .js,.jsx --ignore-path .gitignore --quiet",
+    "lint:diagnostic": "eslint . --ext .js,.jsx --ignore-path .gitignore --format=stylish",
+    "lint:strict:ts": "eslint . --ext .js,.jsx,.ts,.tsx --max-warnings 0",
+    "lint:diagnostic:ts": "eslint . --ext .js,.jsx,.ts,.tsx --format=stylish",
+    "lint:errors-only:ts": "eslint . --ext .js,.jsx,.ts,.tsx --quiet",
     "lint-staged": "lint-staged",
     "postinstall": "husky install",
     "verify:mocks": "node scripts/verify-fixture-mocks.js",
@@ -30,17 +53,53 @@
     "docs:deploy": "docusaurus deploy docs-site",
     "roadmap:close": "node scripts/close-done-roadmap-issues.js",
     "labels:sync": "node scripts/ensure-roadmap-labels.js",
-    "release:tag": "node scripts/generate-release-note.js v$(node -p \"require('./package.json').version\")"
-  },
-  "lint-staged": {
-    "**/*.{js,ts}": [
-      "npm run lint",
-      "npm run validate-fixtures"
-    ]
+    "release:tag": "node scripts/generate-release-note.js v$(node -p \"require('./package.json').version\")",
+    "security:audit": "npm audit --audit-level=moderate",
+    "security:audit:critical": "npm audit --audit-level=critical --production || echo 'Development vulnerabilities found but acceptable'",
+    "security:audit:json": "npm audit --json > security-audit-report.json || true",
+    "docs:generate": "node scripts/generate-docs.js --verbose",
+    "docs:api": "node scripts/generate-docs.js --formats=html,markdown,json",
+    "benchmark": "node scripts/benchmark.js --iterations=100 --verbose",
+    "benchmark:quick": "node scripts/benchmark.js --iterations=20",
+    "benchmark:full": "node scripts/benchmark.js --iterations=500 --verbose",
+    "dev": "node scripts/dev-tools.js menu",
+    "dev:health": "node scripts/dev-tools.js health",
+    "dev:setup": "node scripts/dev-tools.js setup",
+    "dev:clean": "node scripts/dev-tools.js clean",
+    "dev:update": "node scripts/dev-tools.js update",
+    "dev:report": "node scripts/dev-tools.js report",
+    "security:fix": "npm audit fix",
+    "security:fix:force": "npm audit fix --force",
+    "security:check": "npm audit --production && echo 'Production security check passed'",
+    "security:report": "npm run security:audit:json && echo 'Security report generated: security-audit-report.json'",
+    "ci:hardening": "node scripts/ci/harden-workflows.js",
+    "ci:audit": "node scripts/audit/gha-audit.js",
+    "ci:targeted-fixes": "node scripts/ci/targeted-security-fixes.js",
+    "ci:comprehensive-hardening": "node scripts/ci/comprehensive-hardening.js",
+    "ci:shell-security": "node scripts/ci/shell-security-hardening.js",
+    "ci:final-fixes": "node scripts/ci/final-shell-injection-fixes.js",
+    "ci:test-workflows": "node scripts/ci/test-workflows.js",
+    "ci:fix-remaining": "node scripts/ci/fix-remaining-actions.js",
+    "tests:categorize": "node scripts/test-categorizer.js",
+    "tests:affected": "node scripts/run-affected.js",
+    "tests:full": "jest --runInBand --reporters=default --json --outputFile=test-results-current.json",
+    "tests:ci": "jest --maxWorkers=50% --reporters=default --json --outputFile=test-results-current.json",
+    "test:unit": "jest __tests__/unit",
+    "test:integration": "jest __tests__/integration",
+    "test:simple": "jest --maxWorkers=1 --no-coverage",
+    "verify:pack": "node scripts/verify-npm-pack.js",
+    "verify": "npm run lint && npm run typecheck --if-present && npm test",
+    "sbom": "node scripts/generate-sbom.js",
+    "observability:setup": "node scripts/setup-observability.js",
+    "observability:start": "cd observability && ./scripts/start.sh",
+    "observability:stop": "cd observability && ./scripts/stop.sh",
+    "telemetry:init": "node -e \"require('./src/observability/telemetry').initializeTelemetry()\"",
+    "security:monitor": "node -e \"require('./src/observability/security-monitor').initializeSecurityMonitoring()\"",
+    "audit:compliance": "node -e \"require('./src/observability/audit-logger').initializeAuditLogging()\""
   },
   "repository": {
     "type": "git",
-    "url": "https://github.com/DevilsDev/rag-pipeline-utils.git"
+    "url": "git+https://github.com/DevilsDev/rag-pipeline-utils.git"
   },
   "keywords": [
     "rag",
@@ -52,19 +111,40 @@
     "openai"
   ],
   "author": "Ali Kahwaji",
-  "license": "Apache-2.0",
+  "license": "GPL-3.0",
   "bugs": {
     "url": "https://github.com/DevilsDev/rag-pipeline-utils/issues"
   },
   "homepage": "https://github.com/DevilsDev/rag-pipeline-utils#readme",
+  "files": [
+    "dist/",
+    "bin/",
+    "contracts/",
+    "README.md",
+    "LICENSE"
+  ],
   "publishConfig": {
     "access": "public"
   },
+  "lint-staged": {
+    "src/**/*.{js,ts,jsx,tsx}": "eslint --fix",
+    "**/*.{js,ts,jsx,tsx,json,md,mdx,yml,yaml,css,scss,html}": "prettier --write --ignore-unknown"
+  },
   "devDependencies": {
     "@babel/core": "^7.26.10",
     "@babel/preset-env": "^7.26.9",
     "@docusaurus/core": "^3.7.0",
     "@docusaurus/preset-classic": "^3.7.0",
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/auto-instrumentations-node": "^0.52.1",
+    "@opentelemetry/exporter-jaeger": "^1.26.0",
+    "@opentelemetry/exporter-prometheus": "^0.53.0",
+    "@opentelemetry/exporter-trace-otlp-http": "^0.53.0",
+    "@opentelemetry/resources": "^1.26.0",
+    "@opentelemetry/sdk-metrics": "^1.26.0",
+    "@opentelemetry/sdk-node": "^0.53.0",
+    "@opentelemetry/sdk-trace-base": "^1.26.0",
+    "@opentelemetry/semantic-conventions": "^1.27.0",
     "@semantic-release/changelog": "^6.0.3",
     "@semantic-release/commit-analyzer": "^13.0.1",
     "@semantic-release/git": "^10.0.1",
@@ -72,22 +152,25 @@
     "@semantic-release/npm": "^11.0.3",
     "@semantic-release/release-notes-generator": "^14.0.3",
     "@types/jest": "^29.5.14",
+    "acorn": "8.15.0",
     "babel-jest": "^29.7.0",
     "codecov": "^3.8.2",
-    "cross-env": "^7.0.3",
-    "docusaurus": "^1.14.7",
+    "conventional-changelog-conventionalcommits": "^8.0.0",
+    "cross-env": "7.0.3",
     "esbuild": "^0.25.2",
     "eslint": "^8.57.1",
     "eslint-plugin-storybook": "^0.12.0",
     "husky": "^8.0.3",
     "jest": "^29.7.0",
+    "jest-junit": "^16.0.0",
     "jest-transform-stub": "^2.0.0",
     "lint-staged": "^15.5.1",
     "prettier": "^3.5.3",
     "prop-types": "^15.8.1",
     "rimraf": "^6.0.1",
     "semantic-release": "^21.1.2",
-    "ts-jest": "^29.3.2"
+    "ts-jest": "^29.3.2",
+    "why-is-node-running": "^3.2.2"
   },
   "dependencies": {
     "@octokit/rest": "^21.1.1",
@@ -99,6 +182,8 @@
     "dotenv": "^16.5.0",
     "fast-glob": "^3.3.3",
     "framer-motion": "^12.7.4",
+    "inquirer": "^8.2.5",
+    "isolated-vm": "6.0.1",
     "jsdom": "^26.1.0",
     "octokit": "^4.1.3",
     "openai": "^4.93.0",

package/.codecov.yml

@@ -1,21 +0,0 @@
-# .codecov.yml
-
-codecov:
-  notify:
-    after_n_builds: 1
-
-coverage:
-  status:
-    project:
-      default:
-        target: 90%
-        threshold: 5%
-    patch:
-      default:
-        target: 90%
-        threshold: 5%
-
-comment:
-  layout: "reach, diff, flags, files"
-  behavior: default
-  require_changes: false

package/.dockerignore

@@ -1,14 +0,0 @@
-# Ignore test files, fixtures, and local config
-__tests__/
-coverage/
-.env
-.env.*
-node_modules/
-.dockerignore
-.git
-.gitignore
-.vscode
-*.log
-*.md
-/public/
-/docs-site/

package/.eslintignore

@@ -1,5 +0,0 @@
-node_modules
-docs-site
-public
-dist
-coverage

package/.eslintrc.json

@@ -1,28 +0,0 @@
-{
-  "env": {
-    "es2022": true,
-    "node": true,
-    "jest": true,
-    "browser": true
-  },
-  "extends": [
-    "eslint:recommended",
-    "plugin:storybook/recommended"
-  ],
-  "parserOptions": {
-    "ecmaVersion": "latest",
-    "sourceType": "module"
-  },
-  "rules": {
-    "no-unused-vars": "warn",
-    "no-console": "off",
-    "semi": [
-      "error",
-      "always"
-    ],
-    "quotes": [
-      "error",
-      "single"
-    ]
-  }
-}