@devilsdev/rag-pipeline-utils 2.4.5 → 2.6.0

package/dist/types/utils/rate-limiter.d.ts

@@ -0,0 +1,66 @@
+export = RateLimiter;
+declare class RateLimiter {
+    /**
+     * Create a rate limiter instance
+     * @param {Object} options - Configuration options
+     * @param {number} options.maxAttempts - Maximum attempts allowed (default: 5)
+     * @param {number} options.windowMs - Time window in milliseconds (default: 15 minutes)
+     * @param {number} options.blockDurationMs - How long to block after exceeding limit (default: 1 hour)
+     * @param {number} options.cleanupIntervalMs - How often to cleanup expired entries (default: 5 minutes)
+     */
+    constructor(options?: {
+        maxAttempts: number;
+        windowMs: number;
+        blockDurationMs: number;
+        cleanupIntervalMs: number;
+    });
+    maxAttempts: number;
+    windowMs: number;
+    blockDurationMs: number;
+    cleanupIntervalMs: number;
+    store: Map<any, any>;
+    /**
+     * Check if a request should be allowed
+     * @param {string} identifier - Unique identifier (IP, userId, tenantId, etc.)
+     * @returns {Object} { allowed: boolean, remaining: number, resetAt: Date, blockedUntil: Date|null }
+     */
+    allowRequest(identifier: string): any;
+    /**
+     * Manually reset rate limit for an identifier
+     * @param {string} identifier - Identifier to reset
+     */
+    reset(identifier: string): void;
+    /**
+     * Get current status for an identifier
+     * @param {string} identifier - Identifier to check
+     * @returns {Object} Current rate limit status
+     */
+    getStatus(identifier: string): any;
+    /**
+     * Clean up expired entries
+     */
+    cleanup(): {
+        cleaned: number;
+        remaining: number;
+    };
+    /**
+     * Get statistics about rate limiter usage
+     * @returns {Object} Statistics
+     */
+    getStats(): any;
+    /**
+     * Stop the rate limiter and cleanup
+     */
+    destroy(): void;
+    _cleanupInterval: NodeJS.Timeout;
+    /**
+     * Hash identifier for storage (privacy)
+     * @private
+     */
+    private _hashIdentifier;
+    /**
+     * Start automatic cleanup timer
+     * @private
+     */
+    private _startCleanup;
+}

package/dist/types/utils/retry.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Retry a function with exponential backoff.
+ *
+ * @param {Function} fn - async or sync function to call; receives (attempt)
+ * @param {Object} [opts]
+ * @param {number} [opts.retries=3] - number of *retries* (total attempts = retries + 1)
+ * @param {number} [opts.baseDelay=100] - initial delay in ms
+ * @param {number} [opts.multiplier=2] - backoff multiplier
+ * @param {boolean} [opts.jitter=false] - add +/-50% jitter if true
+ * @param {Function} [opts.sleep=defaultSleep] - injected sleeper for tests
+ * @param {Function} [opts.onDelay] - callback (delayMs, attempt, error) before each sleep
+ * @returns {Promise<*>}
+ */
+export function retry(fn: Function, opts?: {
+    retries?: number;
+    baseDelay?: number;
+    multiplier?: number;
+    jitter?: boolean;
+    sleep?: Function;
+    onDelay?: Function;
+}): Promise<any>;
+export function withRetry(_fn: any, { retries, initialDelay, label }?: {
+    retries?: number;
+    initialDelay?: number;
+    label?: string;
+}): Promise<any>;
+export function defaultSleep(ms: any): Promise<any>;

package/dist/types/utils/run-abort.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Per-run cancellation: one AbortController that is aborted when `timeout`
+ * elapses or when the caller's `signal` aborts. Its `signal` is handed to
+ * every component call; `race()` makes the run settle as soon as it is
+ * aborted even if a component ignores the signal; `throwIfAborted()` stops
+ * work between stages.
+ *
+ * @param {object} [options]
+ * @param {number} [options.timeout] - Milliseconds before the run is aborted
+ * @param {AbortSignal} [options.signal] - Caller's signal to follow
+ * @returns {{
+ *   signal: AbortSignal,
+ *   race: (promise: Promise<any>) => Promise<any>,
+ *   throwIfAborted: () => void,
+ *   clearTimer: () => void,
+ *   detach: () => void,
+ *   cleanup: () => void
+ * }}
+ */
+export function createRunAbort({ timeout, signal }?: {
+    timeout?: number;
+    signal?: AbortSignal;
+}): {
+    signal: AbortSignal;
+    race: (promise: Promise<any>) => Promise<any>;
+    throwIfAborted: () => void;
+    clearTimer: () => void;
+    detach: () => void;
+    cleanup: () => void;
+};
+/** Message used for a run that exceeded its timeout. */
+export const TIMEOUT_MESSAGE: "Execution timeout";
+/** Message used for a run cancelled through the caller's AbortSignal. */
+export const ABORTED_MESSAGE: "Execution aborted";

package/dist/types/utils/secure-logging.d.ts

@@ -0,0 +1,51 @@
+/**
+ * Redact sensitive fields in an object
+ * @param {any} obj - Object to redact
+ * @param {number} depth - Current recursion depth
+ * @param {number} maxDepth - Maximum recursion depth
+ * @returns {any} Redacted object
+ */
+export function redactObject(obj: any, depth?: number, maxDepth?: number): any;
+/**
+ * Redact a string value if it contains secrets
+ * @param {string} value - String value to redact
+ * @returns {string} Redacted value if sensitive, original otherwise
+ */
+export function redactStringValue(value: string): string;
+/**
+ * Redact sensitive information from log data
+ * @param {string} message - Log message
+ * @param {object} meta - Log metadata object
+ * @returns {object} Redacted message and metadata
+ */
+export function redactLogData(message: string, meta?: object): object;
+/**
+ * Create a redacting wrapper for any logger
+ * @param {object} logger - Logger instance to wrap
+ * @returns {object} Wrapped logger with automatic redaction
+ */
+export function createSecureLogger(logger: object): object;
+/**
+ * Check if a field name is sensitive
+ * @param {string} fieldName - Field name to check
+ * @returns {boolean} True if sensitive
+ */
+export function isSensitiveFieldName(fieldName: string): boolean;
+/**
+ * Check if a value matches a secret pattern
+ * @param {string} value - Value to check
+ * @returns {boolean} True if matches secret pattern
+ */
+export function matchesSecretPattern(value: string): boolean;
+/**
+ * Sensitive field names that should be redacted
+ */
+export const SENSITIVE_FIELD_NAMES: Set<string>;
+/**
+ * Patterns that match secret-like values
+ */
+export const SECRET_PATTERNS: RegExp[];
+/**
+ * Redaction marker
+ */
+export const REDACTED: "[REDACTED]";

package/dist/types/utils/structured-logger.d.ts

@@ -0,0 +1,126 @@
+/**
+ * Enterprise structured logger with correlation tracking
+ */
+export class StructuredLogger {
+    constructor(options?: {});
+    serviceName: any;
+    environment: any;
+    logLevel: any;
+    enableCorrelation: boolean;
+    enableMetrics: boolean;
+    enableSecureLogging: boolean;
+    outputFormat: any;
+    metrics: {
+        logsWritten: number;
+        errorCount: number;
+        warnCount: number;
+        avgLatency: number;
+    };
+    /**
+     * Create correlation context for request tracking
+     * @param {string} correlationId - Optional correlation ID
+     * @param {Function} fn - Function to execute with context
+     * @returns {Promise} Result of function execution
+     */
+    withCorrelation(correlationId: string, fn: Function): Promise<any>;
+    /**
+     * Generate correlation ID
+     * @returns {string} Correlation ID
+     */
+    generateCorrelationId(): string;
+    /**
+     * Get current correlation context
+     * @returns {object|null} Current context
+     */
+    getContext(): object | null;
+    /**
+     * Create structured log entry
+     * @param {string} level - Log level
+     * @param {string} message - Log message
+     * @param {object} meta - Additional metadata
+     * @returns {object} Structured log entry
+     */
+    createLogEntry(level: string, message: string, meta?: object): object;
+    /**
+     * Check if log level should be written
+     * @param {string} level - Log level to check
+     * @returns {boolean} Whether to write log
+     */
+    shouldLog(level: string): boolean;
+    /**
+     * Write log entry to output
+     * @param {object} entry - Log entry
+     */
+    writeLog(entry: object): void;
+    /**
+     * Log error message
+     * @param {string} message - Error message
+     * @param {object} meta - Additional metadata
+     */
+    error(message: string, meta?: object): void;
+    /**
+     * Log warning message
+     * @param {string} message - Warning message
+     * @param {object} meta - Additional metadata
+     */
+    warn(message: string, meta?: object): void;
+    /**
+     * Log info message
+     * @param {string} message - Info message
+     * @param {object} meta - Additional metadata
+     */
+    info(message: string, meta?: object): void;
+    /**
+     * Log debug message
+     * @param {string} message - Debug message
+     * @param {object} meta - Additional metadata
+     */
+    debug(message: string, meta?: object): void;
+    /**
+     * Log trace message
+     * @param {string} message - Trace message
+     * @param {object} meta - Additional metadata
+     */
+    trace(message: string, meta?: object): void;
+    /**
+     * Create child logger with additional context
+     * @param {object} childMeta - Additional metadata for child logger
+     * @returns {ChildLogger} Child logger instance
+     */
+    child(childMeta?: object): ChildLogger;
+    /**
+     * Get logger metrics
+     * @returns {object} Logger performance metrics
+     */
+    getMetrics(): object;
+    /**
+     * Reset logger metrics
+     */
+    resetMetrics(): void;
+}
+/**
+ * Child logger with additional context
+ */
+export class ChildLogger {
+    constructor(parent: any, childMeta: any);
+    parent: any;
+    childMeta: any;
+    error(message: any, meta?: {}): void;
+    warn(message: any, meta?: {}): void;
+    info(message: any, meta?: {}): void;
+    debug(message: any, meta?: {}): void;
+    trace(message: any, meta?: {}): void;
+    child(additionalMeta?: {}): ChildLogger;
+    withCorrelation(correlationId: any, fn: any): any;
+    getContext(): any;
+    getMetrics(): any;
+}
+/**
+ * Create structured logger instance
+ * @param {object} options - Logger configuration
+ * @returns {StructuredLogger} Logger instance
+ */
+export function createLogger(options?: object): StructuredLogger;
+declare const defaultLogger: StructuredLogger;
+export declare let withCorrelation: any;
+export { defaultLogger as logger };

package/package.json

@@ -1,16 +1,36 @@
 {
   "name": "@devilsdev/rag-pipeline-utils",
-  "version": "2.4.5",
-  "description": "Composable RAG for Node.js — with built-in evaluation, citations, guardrails, and observability.",
+  "version": "2.6.0",
+  "description": "Composable retrieval-augmented generation (RAG) for Node.js: grounded citations, evaluation, guardrails, hybrid retrieval and a CLI.",
   "main": "dist/index.cjs",
   "type": "commonjs",
   "types": "dist/index.d.ts",
+  "typesVersions": {
+    "*": {
+      "experimental": [
+        "dist/experimental.d.ts"
+      ],
+      "testing": [
+        "dist/testing.d.ts"
+      ]
+    }
+  },
   "exports": {
     ".": {
       "types": "./dist/index.d.ts",
       "require": "./dist/index.cjs",
       "import": "./dist/index.mjs"
     },
+    "./experimental": {
+      "types": "./dist/experimental.d.ts",
+      "require": "./dist/experimental.cjs",
+      "import": "./dist/experimental.mjs"
+    },
+    "./testing": {
+      "types": "./dist/testing.d.ts",
+      "require": "./dist/testing.cjs",
+      "import": "./dist/testing.mjs"
+    },
     "./cli": "./bin/cli.js"
   },
   "bin": {
@@ -20,19 +40,21 @@
     "node": ">=18"
   },
   "scripts": {
-    "build": "node scripts/build.js && node scripts/generate-types.js",
+    "build": "node scripts/build.js && node scripts/generate-types.js && node scripts/check-type-exports.js",
     "prepack": "npm run build",
     "docs:validate": "node scripts/validate-docs-examples.js",
     "verify:esm": "node scripts/verify-esm.mjs",
     "prepublishOnly": "npm run lint && npm test && npm run docs:validate && npm run verify:esm",
     "guard:cjs": "node scripts/guard-cjs.js",
     "pretest": "npm run guard:cjs && node scripts/setup.js",
-    "test": "cross-env NODE_OPTIONS=--experimental-vm-modules jest --runInBand --config jest.config.js",
+    "test": "cross-env NODE_OPTIONS=--experimental-vm-modules jest --maxWorkers=50% --config jest.config.js",
+    "test:serial": "cross-env NODE_OPTIONS=--experimental-vm-modules jest --runInBand --config jest.config.js",
     "test:ci": "jest --coverage",
     "test:open-handles": "jest --detectOpenHandles --runInBand",
     "test:report": "node scripts/test-reporter.js",
     "posttest": "node scripts/generate-test-reports.js",
-    "format": "prettier --write \"src/**/*.{js,json}\"",
+    "format": "prettier --write --ignore-unknown \"{__tests__,scripts,bin,docs-site,examples,contracts}/**/*.{js,jsx,ts,tsx,mjs,cjs}\" \"*.{js,mjs,cjs}\" \"**/*.{json,md,mdx,yml,yaml,css,scss,html}\"",
+    "format:check": "prettier --check --ignore-unknown \"{__tests__,scripts,bin,docs-site,examples,contracts}/**/*.{js,jsx,ts,tsx,mjs,cjs}\" \"*.{js,mjs,cjs}\" \"**/*.{json,md,mdx,yml,yaml,css,scss,html}\"",
     "test:watch": "jest --watch",
     "lint": "eslint src --ext .js --ignore-path .gitignore",
     "lint:fix": "eslint . --ext .js,.jsx --ignore-path .gitignore --fix",
@@ -45,16 +67,16 @@
     "lint-staged": "lint-staged",
     "postinstall": "husky install",
     "verify:mocks": "node scripts/verify-fixture-mocks.js",
-    "build:docs": "npx docusaurus build docs-site",
+    "build:docs": "npm run docs:build",
     "ci": "node scripts/ci-runner.js",
-    "docs:dev": "npx docusaurus start docs-site",
+    "docs:dev": "cd docs-site && npm run start",
     "docs:version": "npx docusaurus docs:version",
     "watch:lqip": "node docs-site/scripts/watch-images.js",
     "docs:clear": "docusaurus clear --site-dir docs-site",
-    "docs:start": "docusaurus start docs-site",
+    "docs:start": "cd docs-site && npm run start",
     "docs:build": "cd docs-site && npm run build",
     "docs:serve": "docusaurus serve docs-site",
-    "docs:deploy": "docusaurus deploy docs-site",
+    "docs:deploy": "node docs-site/scripts/sync-changelog.js && docusaurus deploy docs-site",
     "roadmap:close": "node scripts/close-done-roadmap-issues.js",
     "labels:sync": "node scripts/ensure-roadmap-labels.js",
     "release:tag": "node scripts/generate-release-note.js v$(node -p \"require('./package.json').version\")",
@@ -76,14 +98,8 @@
     "security:fix:force": "npm audit fix --force",
     "security:check": "npm audit --production && echo 'Production security check passed'",
     "security:report": "npm run security:audit:json && echo 'Security report generated: security-audit-report.json'",
-    "ci:hardening": "node scripts/ci/harden-workflows.js",
     "ci:audit": "node scripts/audit/gha-audit.js",
-    "ci:targeted-fixes": "node scripts/ci/targeted-security-fixes.js",
-    "ci:comprehensive-hardening": "node scripts/ci/comprehensive-hardening.js",
-    "ci:shell-security": "node scripts/ci/shell-security-hardening.js",
-    "ci:final-fixes": "node scripts/ci/final-shell-injection-fixes.js",
     "ci:test-workflows": "node scripts/ci/test-workflows.js",
-    "ci:fix-remaining": "node scripts/ci/fix-remaining-actions.js",
     "tests:categorize": "node scripts/test-categorizer.js",
     "tests:affected": "node scripts/run-affected.js",
     "tests:full": "jest --runInBand --reporters=default --json --outputFile=test-results-current.json",
@@ -92,6 +108,8 @@
     "test:integration": "jest __tests__/integration",
     "test:simple": "jest --maxWorkers=1 --no-coverage",
     "verify:pack": "node scripts/verify-npm-pack.js",
+    "smoke:pack": "node scripts/smoke-test-pack.js",
+    "demo:record": "vhs assets/demo.tape",
     "verify": "npm run lint && npm run typecheck --if-present && npm test",
     "sbom": "node scripts/generate-sbom.js",
     "observability:setup": "node scripts/setup-observability.js",
@@ -138,15 +156,12 @@
     "dist/",
     "src/",
     "!src/mocks/",
-    "!src/ai/module-template.js",
-    "!src/ai/index-fixed.js",
     "!src/dev/examples/",
     "!src/dev/README.md",
     "!src/utils/examples/",
     "!src/utils/ci/",
     "!src/ci/",
     "!src/scripts/",
-    "!src/cli/plugin-marketplace-commands.js",
     "bin/",
     "contracts/",
     ".ragrc.schema.json",
@@ -162,13 +177,19 @@
   },
   "lint-staged": {
     "src/**/*.{js,ts,jsx,tsx}": "eslint --fix",
-    "**/*.{js,ts,jsx,tsx,json,md,mdx,yml,yaml,css,scss,html}": "prettier --write --ignore-unknown"
+    "{__tests__,scripts,bin,docs-site,examples,contracts}/**/*.{js,jsx,ts,tsx,mjs,cjs}": "prettier --write --ignore-unknown",
+    "*.{js,mjs,cjs}": "prettier --write --ignore-unknown",
+    "**/*.{json,md,mdx,yml,yaml,css,scss,html}": "prettier --write --ignore-unknown"
   },
   "devDependencies": {
     "@babel/core": "^7.26.10",
     "@babel/preset-env": "^7.26.9",
-    "@docusaurus/core": "3.5.2",
-    "@docusaurus/preset-classic": "3.5.2",
+    "@docusaurus/core": "3.10.2",
+    "@docusaurus/preset-classic": "3.10.2",
+    "@docusaurus/theme-mermaid": "3.10.2",
+    "@easyops-cn/docusaurus-search-local": "0.55.3",
+    "@fontsource/inter": "5.3.0",
+    "@fontsource/jetbrains-mono": "5.3.0",
     "@octokit/rest": "^21.1.1",
     "@opentelemetry/api": "^1.9.0",
     "@opentelemetry/auto-instrumentations-node": "^0.52.1",
@@ -180,18 +201,14 @@
     "@opentelemetry/sdk-node": "^0.53.0",
     "@opentelemetry/sdk-trace-base": "^1.26.0",
     "@opentelemetry/semantic-conventions": "^1.27.0",
-    "@semantic-release/changelog": "^6.0.3",
-    "@semantic-release/commit-analyzer": "^13.0.1",
-    "@semantic-release/git": "^10.0.1",
-    "@semantic-release/github": "^9.2.6",
-    "@semantic-release/npm": "^11.0.3",
-    "@semantic-release/release-notes-generator": "^14.0.3",
     "@types/jest": "^29.5.14",
+    "@types/node": "22.14.1",
     "acorn": "8.15.0",
+    "ai": "6.0.317",
+    "ai-v7": "npm:ai@7.0.156",
     "babel-jest": "^29.7.0",
-    "codecov": "3.6.2",
-    "conventional-changelog-conventionalcommits": "^8.0.0",
     "cross-env": "7.0.3",
+    "docusaurus-plugin-llms": "0.6.1",
     "esbuild": "^0.25.2",
     "eslint": "^8.57.1",
     "eslint-plugin-storybook": "^0.12.0",
@@ -207,24 +224,46 @@
     "prettier": "^3.5.3",
     "prop-types": "^15.8.1",
     "rimraf": "^6.0.1",
-    "semantic-release": "^21.1.2",
     "sharp": "^0.34.1",
     "ts-jest": "^29.3.2",
+    "typescript": "5.8.3",
     "why-is-node-running": "^3.2.2"
   },
   "dependencies": {
     "ajv": "^8.17.1",
-    "axios": "^1.8.4",
-    "chalk": "^5.4.1",
+    "ajv-formats": "2.1.1",
+    "axios": "^1.21.0",
+    "chalk": "4.1.2",
     "chokidar": "^4.0.3",
+    "cli-table3": "0.6.5",
     "commander": "^13.1.0",
     "csv-parse": "^5.6.0",
-    "dotenv": "^16.5.0",
     "fast-glob": "^3.3.3",
     "inquirer": "^8.2.5",
     "jsdom": "^26.1.0",
     "jsonwebtoken": "9.0.2",
+    "marked": "15.0.12",
     "openai": "^4.93.0",
+    "ora": "5.4.1",
     "pino": "^9.6.0"
+  },
+  "peerDependencies": {
+    "isolated-vm": ">=5.0.0",
+    "pino-pretty": ">=10.0.0"
+  },
+  "peerDependenciesMeta": {
+    "isolated-vm": {
+      "optional": true
+    },
+    "pino-pretty": {
+      "optional": true
+    }
+  },
+  "overrides": {
+    "fast-uri": "^3.1.9",
+    "follow-redirects": "^1.16.2",
+    "form-data": "^4.0.6",
+    "ws@^8.0.0": "^8.21.0",
+    "zod": "^4.6.7"
   }
 }