@devilsdev/rag-pipeline-utils 2.6.1 → 2.7.0

package/dist/types/utils/error-formatter.d.ts

@@ -0,0 +1,312 @@
+export namespace ERROR_CODES {
+    let CONFIG_NOT_FOUND: string;
+    let CONFIG_INVALID_JSON: string;
+    let CONFIG_VALIDATION_FAILED: string;
+    let CONFIG_MISSING_REQUIRED: string;
+    let CONFIG_INVALID_TYPE: string;
+    let CONFIG_INVALID_VALUE: string;
+    let PLUGIN_NOT_FOUND: string;
+    let PLUGIN_LOAD_FAILED: string;
+    let PLUGIN_INVALID_EXPORT: string;
+    let PLUGIN_EXECUTION_FAILED: string;
+    let PLUGIN_INITIALIZATION_FAILED: string;
+    let PLUGIN_CONTRACT_VIOLATION: string;
+    let PLUGIN_DEPENDENCY_MISSING: string;
+    let PIPELINE_CREATION_FAILED: string;
+    let PIPELINE_EXECUTION_FAILED: string;
+    let PIPELINE_STAGE_FAILED: string;
+    let PIPELINE_INVALID_STAGE: string;
+    let VALIDATION_FAILED: string;
+    let VALIDATION_SCHEMA_ERROR: string;
+    let VALIDATION_CONSTRAINT_ERROR: string;
+    let FS_FILE_NOT_FOUND: string;
+    let FS_DIRECTORY_NOT_FOUND: string;
+    let FS_PERMISSION_DENIED: string;
+    let FS_READ_ERROR: string;
+    let FS_WRITE_ERROR: string;
+    let NETWORK_REQUEST_FAILED: string;
+    let NETWORK_TIMEOUT: string;
+    let NETWORK_INVALID_RESPONSE: string;
+    let HOT_RELOAD_FAILED: string;
+    let HOT_RELOAD_STATE_ERROR: string;
+    let HOT_RELOAD_WATCH_ERROR: string;
+    let DAG_CYCLE_DETECTED: string;
+    let DAG_NODE_NOT_FOUND: string;
+    let DAG_EXECUTION_FAILED: string;
+    let DAG_INVALID_TOPOLOGY: string;
+    let AUTH_INVALID_TOKEN: string;
+    let AUTH_TOKEN_EXPIRED: string;
+    let AUTH_INSUFFICIENT_PERMISSIONS: string;
+    let UNKNOWN_ERROR: string;
+}
+/**
+ * Message, suggestion and documentation templates keyed by error code.
+ * Placeholders such as {path} are filled from the context passed to
+ * formatError. Codes without an entry fall back to UNKNOWN_ERROR.
+ */
+export const ERROR_TEMPLATES: {
+    [ERROR_CODES.CONFIG_NOT_FOUND]: {
+        message: string;
+        suggestions: string[];
+        documentation: string;
+    };
+    [ERROR_CODES.CONFIG_INVALID_JSON]: {
+        message: string;
+        suggestions: string[];
+        documentation: string;
+    };
+    [ERROR_CODES.CONFIG_VALIDATION_FAILED]: {
+        message: string;
+        suggestions: string[];
+        documentation: string;
+    };
+    [ERROR_CODES.CONFIG_MISSING_REQUIRED]: {
+        message: string;
+        suggestions: string[];
+        documentation: string;
+    };
+    [ERROR_CODES.CONFIG_INVALID_TYPE]: {
+        message: string;
+        suggestions: string[];
+        documentation: string;
+    };
+    [ERROR_CODES.PLUGIN_NOT_FOUND]: {
+        message: string;
+        suggestions: string[];
+        documentation: string;
+    };
+    [ERROR_CODES.PLUGIN_LOAD_FAILED]: {
+        message: string;
+        suggestions: string[];
+        documentation: string;
+    };
+    [ERROR_CODES.PLUGIN_INVALID_EXPORT]: {
+        message: string;
+        suggestions: string[];
+        documentation: string;
+    };
+    [ERROR_CODES.PLUGIN_EXECUTION_FAILED]: {
+        message: string;
+        suggestions: string[];
+        documentation: string;
+    };
+    [ERROR_CODES.PLUGIN_CONTRACT_VIOLATION]: {
+        message: string;
+        suggestions: string[];
+        documentation: string;
+    };
+    [ERROR_CODES.PLUGIN_DEPENDENCY_MISSING]: {
+        message: string;
+        suggestions: string[];
+        documentation: string;
+    };
+    [ERROR_CODES.PIPELINE_CREATION_FAILED]: {
+        message: string;
+        suggestions: string[];
+        documentation: string;
+    };
+    [ERROR_CODES.PIPELINE_EXECUTION_FAILED]: {
+        message: string;
+        suggestions: string[];
+        documentation: string;
+    };
+    [ERROR_CODES.PIPELINE_STAGE_FAILED]: {
+        message: string;
+        suggestions: string[];
+        documentation: string;
+    };
+    [ERROR_CODES.FS_FILE_NOT_FOUND]: {
+        message: string;
+        suggestions: string[];
+        documentation: string;
+    };
+    [ERROR_CODES.FS_DIRECTORY_NOT_FOUND]: {
+        message: string;
+        suggestions: string[];
+        documentation: string;
+    };
+    [ERROR_CODES.FS_PERMISSION_DENIED]: {
+        message: string;
+        suggestions: string[];
+        documentation: string;
+    };
+    [ERROR_CODES.HOT_RELOAD_FAILED]: {
+        message: string;
+        suggestions: string[];
+        documentation: string;
+    };
+    [ERROR_CODES.HOT_RELOAD_STATE_ERROR]: {
+        message: string;
+        suggestions: string[];
+        documentation: string;
+    };
+    [ERROR_CODES.DAG_CYCLE_DETECTED]: {
+        message: string;
+        suggestions: string[];
+        documentation: string;
+    };
+    [ERROR_CODES.DAG_NODE_NOT_FOUND]: {
+        message: string;
+        suggestions: string[];
+        documentation: string;
+    };
+    [ERROR_CODES.VALIDATION_FAILED]: {
+        message: string;
+        suggestions: string[];
+        documentation: string;
+    };
+    [ERROR_CODES.AUTH_INVALID_TOKEN]: {
+        message: string;
+        suggestions: string[];
+        documentation: string;
+    };
+    [ERROR_CODES.AUTH_TOKEN_EXPIRED]: {
+        message: string;
+        suggestions: string[];
+        documentation: string;
+    };
+    [ERROR_CODES.UNKNOWN_ERROR]: {
+        message: string;
+        suggestions: string[];
+        documentation: string;
+    };
+};
+/**
+ * Error carrying a code, the context used to build its message, suggested
+ * fixes and a documentation path, all taken from ERROR_TEMPLATES.
+ */
+export class EnhancedError extends Error {
+    /**
+     * @param {string} code - Error code, normally one of ERROR_CODES
+     * @param {Object<string, *>} [context={}] - Values for the template placeholders
+     * @param {Error|null} [originalError=null] - Underlying error, if any
+     */
+    constructor(code: string, context?: {
+        [x: string]: any;
+    }, originalError?: Error | null);
+    code: string;
+    context: {
+        [x: string]: any;
+    };
+    suggestions: string[];
+    documentation: string;
+    originalError: Error;
+    timestamp: string;
+    /**
+     * Format this error for display with formatErrorMessage's default options.
+     * @returns {string} Multi-line message with code, suggestions and documentation link
+     */
+    getFormattedMessage(): string;
+    /**
+     * Plain object for JSON logging, including the stack and the original
+     * error's message and stack (or null).
+     * @returns {{name: string, code: string, message: string, suggestions: string[], documentation: string, context: Object<string, *>, timestamp: string, stack: string, originalError: ({message: string, stack: string}|null)}}
+     */
+    toJSON(): {
+        name: string;
+        code: string;
+        message: string;
+        suggestions: string[];
+        documentation: string;
+        context: {
+            [x: string]: any;
+        };
+        timestamp: string;
+        stack: string;
+        originalError: ({
+            message: string;
+            stack: string;
+        } | null);
+    };
+}
+/**
+ * Create an EnhancedError. Same as calling its constructor.
+ *
+ * @param {string} code - Error code
+ * @param {Object<string, *>} [context={}] - Placeholder values
+ * @param {Error|null} [originalError=null] - Underlying error, if any
+ * @returns {EnhancedError} Enhanced error instance
+ */
+export function createError(code: string, context?: {
+    [x: string]: any;
+}, originalError?: Error | null): EnhancedError;
+/**
+ * Wrap an existing error in an EnhancedError. The original message is added
+ * to the context as originalMessage and the error is kept as originalError.
+ *
+ * @param {Error} error - Original error
+ * @param {string} code - Error code
+ * @param {Object<string, *>} [context={}] - Additional placeholder values
+ * @returns {EnhancedError} Enhanced error
+ */
+export function wrapError(error: Error, code: string, context?: {
+    [x: string]: any;
+}): EnhancedError;
+/**
+ * Fill the template for an error code with context values. Unknown codes use
+ * the UNKNOWN_ERROR template. Each placeholder is replaced at its first
+ * occurrence only; placeholders with no context value stay as written.
+ *
+ * @param {string} code - Error code
+ * @param {Object<string, *>} [context={}] - Placeholder values, converted with String()
+ * @returns {{code: string, message: string, suggestions: string[], documentation: string, context: Object<string, *>}} Formatted error details
+ */
+export function formatError(code: string, context?: {
+    [x: string]: any;
+}): {
+    code: string;
+    message: string;
+    suggestions: string[];
+    documentation: string;
+    context: {
+        [x: string]: any;
+    };
+};
+/**
+ * Build a multi-line display string for an error. Fields the error lacks
+ * (code, suggestions, documentation) are skipped.
+ *
+ * @param {Error|EnhancedError} error - Error to format
+ * @param {Object} [options] - Formatting options
+ * @param {boolean} [options.includeCode=true] - Prefix the message with the error code
+ * @param {boolean} [options.includeSuggestions=true] - List the suggestions
+ * @param {boolean} [options.includeDocumentation=true] - Add the documentation URL
+ * @param {boolean} [options.includeContext=false] - List the context values as JSON
+ * @param {boolean} [options.includeStack=false] - Append the stack trace
+ * @param {boolean} [options.color=false] - Accepted but unused
+ * @returns {string} Formatted error message
+ */
+export function formatErrorMessage(error: Error | EnhancedError, options?: {
+    includeCode?: boolean;
+    includeSuggestions?: boolean;
+    includeDocumentation?: boolean;
+    includeContext?: boolean;
+    includeStack?: boolean;
+    color?: boolean;
+}): string;
+/**
+ * Check whether a value is an EnhancedError instance.
+ *
+ * @param {*} error - Value to check
+ * @returns {boolean} True if enhanced error
+ */
+export function isEnhancedError(error: any): boolean;
+/**
+ * Severity for an error code: 'critical' for PIPELINE_CREATION_FAILED,
+ * CONFIG_NOT_FOUND, FS_PERMISSION_DENIED and AUTH_INVALID_TOKEN, 'warning'
+ * for the HOT_RELOAD_FAILED and HOT_RELOAD_STATE_ERROR codes, 'error' otherwise.
+ *
+ * @param {string} code - Error code
+ * @returns {'critical'|'warning'|'error'} Severity level
+ */
+export function getErrorSeverity(code: string): "critical" | "warning" | "error";
+/**
+ * Format an error and write it to a logger. Warnings go to logger.warn,
+ * everything else to logger.error; critical errors include the stack trace.
+ * Errors that are not EnhancedError are treated as 'error'.
+ *
+ * @param {Error|EnhancedError} error - Error to log
+ * @param {Pick<Console, 'error'|'warn'>} [logger=console] - Logger to write to
+ * @returns {void}
+ */
+export function logError(error: Error | EnhancedError, logger?: Pick<Console, "error" | "warn">): void;

package/dist/types/utils/logger.d.ts

@@ -1,3 +1,4 @@
+/** The shared structured logger (pino), exported as `logger.logger`. */
 export const logger: StructuredLogger;
 /**
  * Create a logger with correlation ID for request tracing
@@ -31,10 +32,30 @@ export class StructuredLogger {
      * Log with automatic correlation ID and context injection
      */
     _log(level: any, message: any, meta?: {}): void;
-    info(message: any, meta: any): void;
-    error(message: any, meta: any): void;
-    warn(message: any, meta: any): void;
-    debug(message: any, meta: any): void;
+    /**
+     * Logs at info level.
+     * @param {string} message
+     * @param {*} [meta] - An object, Error or value to attach
+     */
+    info(message: string, meta?: any): void;
+    /**
+     * Logs at error level.
+     * @param {string} message
+     * @param {*} [meta] - An object, Error or value to attach
+     */
+    error(message: string, meta?: any): void;
+    /**
+     * Logs at warn level.
+     * @param {string} message
+     * @param {*} [meta] - An object, Error or value to attach
+     */
+    warn(message: string, meta?: any): void;
+    /**
+     * Logs at debug level.
+     * @param {string} message
+     * @param {*} [meta] - An object, Error or value to attach
+     */
+    debug(message: string, meta?: any): void;
     /**
      * Security audit logging with enhanced metadata
      */
@@ -48,3 +69,7 @@ export class StructuredLogger {
      */
     tenant(tenantId: any, message: any, meta?: {}): void;
 }
+export declare function info(message: string, meta?: any): void;
+export declare function warn(message: string, meta?: any): void;
+export declare function error(message: string, meta?: any): void;
+export declare function debug(message: string, meta?: any): void;

package/dist/types/utils/rate-limiter.d.ts

@@ -1,18 +1,23 @@
 export = RateLimiter;
+/**
+ * Sliding-window limiter: allows up to `maxAttempts` requests per identifier
+ * in `windowMs`, then blocks that identifier for `blockDurationMs`. Check a
+ * request with allowRequest(identifier).
+ */
 declare class RateLimiter {
     /**
      * Create a rate limiter instance
      * @param {Object} options - Configuration options
-     * @param {number} options.maxAttempts - Maximum attempts allowed (default: 5)
-     * @param {number} options.windowMs - Time window in milliseconds (default: 15 minutes)
-     * @param {number} options.blockDurationMs - How long to block after exceeding limit (default: 1 hour)
-     * @param {number} options.cleanupIntervalMs - How often to cleanup expired entries (default: 5 minutes)
+     * @param {number} [options.maxAttempts=5] - Maximum attempts allowed (default: 5)
+     * @param {number} [options.windowMs=900000] - Time window in milliseconds (default: 15 minutes)
+     * @param {number} [options.blockDurationMs=3600000] - How long to block after exceeding limit (default: 1 hour)
+     * @param {number} [options.cleanupIntervalMs=300000] - How often to cleanup expired entries (default: 5 minutes)
      */
     constructor(options?: {
-        maxAttempts: number;
-        windowMs: number;
-        blockDurationMs: number;
-        cleanupIntervalMs: number;
+        maxAttempts?: number;
+        windowMs?: number;
+        blockDurationMs?: number;
+        cleanupIntervalMs?: number;
     });
     maxAttempts: number;
     windowMs: number;

package/dist/types/utils/retry.d.ts

@@ -19,6 +19,16 @@ export function retry(fn: Function, opts?: {
     sleep?: Function;
     onDelay?: Function;
 }): Promise<any>;
+/**
+ * Older wrapper around retry(): retries `fn` with exponential backoff from
+ * `initialDelay` ms and logs each retry and the final failure under `label`.
+ * @param {function(number): *} fn - The operation; called with the attempt index
+ * @param {Object} [options]
+ * @param {number} [options.retries=3] - Retries after the first attempt
+ * @param {number} [options.initialDelay=300] - First backoff delay in ms
+ * @param {string} [options.label='operation'] - Name used in log messages
+ * @returns {Promise<*>} The operation's result
+ */
 export function withRetry(_fn: any, { retries, initialDelay, label }?: {
     retries?: number;
     initialDelay?: number;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devilsdev/rag-pipeline-utils",
-  "version": "2.6.1",
+  "version": "2.7.0",
   "description": "Composable retrieval-augmented generation (RAG) for Node.js: grounded citations, evaluation, guardrails, hybrid retrieval and a CLI.",
   "main": "dist/index.cjs",
   "type": "commonjs",
@@ -65,7 +65,7 @@
     "lint:diagnostic:ts": "eslint . --ext .js,.jsx,.ts,.tsx --format=stylish",
     "lint:errors-only:ts": "eslint . --ext .js,.jsx,.ts,.tsx --quiet",
     "lint-staged": "lint-staged",
-    "postinstall": "husky install",
+    "prepare": "husky install",
     "verify:mocks": "node scripts/verify-fixture-mocks.js",
     "build:docs": "npm run docs:build",
     "ci": "node scripts/ci-runner.js",
@@ -84,10 +84,12 @@
     "security:audit:critical": "npm audit --audit-level=critical --production || echo 'Development vulnerabilities found but acceptable'",
     "security:audit:json": "npm audit --json > security-audit-report.json || true",
     "docs:generate": "node scripts/generate-docs.js --verbose",
-    "docs:api": "node scripts/generate-docs.js --formats=html,markdown,json",
+    "docs:api": "npm run build && typedoc && prettier --write docs-site/docs/api --log-level warn",
     "benchmark": "node scripts/benchmark.js --iterations=100 --verbose",
     "benchmark:quick": "node scripts/benchmark.js --iterations=20",
     "benchmark:full": "node scripts/benchmark.js --iterations=500 --verbose",
+    "bench:speed": "node benchmarks/speed.js",
+    "bench:quality": "node benchmarks/quality.js",
     "dev": "node scripts/dev-tools.js menu",
     "dev:health": "node scripts/dev-tools.js health",
     "dev:setup": "node scripts/dev-tools.js setup",
@@ -214,18 +216,22 @@
     "eslint-plugin-storybook": "^0.12.0",
     "framer-motion": "^12.7.4",
     "husky": "^8.0.3",
+    "hyparquet": "1.32.0",
     "isolated-vm": "5.0.4",
     "jest": "^29.7.0",
     "jest-junit": "^16.0.0",
     "jest-transform-stub": "^2.0.0",
     "lint-staged": "^15.5.1",
     "octokit": "^4.1.3",
+    "pg": "8.23.1",
     "plaiceholder": "^3.0.0",
     "prettier": "^3.5.3",
     "prop-types": "^15.8.1",
     "rimraf": "^6.0.1",
     "sharp": "^0.34.1",
     "ts-jest": "^29.3.2",
+    "typedoc": "0.28.20",
+    "typedoc-plugin-markdown": "4.14.1",
     "typescript": "5.8.3",
     "why-is-node-running": "^3.2.2"
   },
@@ -249,12 +255,16 @@
   },
   "peerDependencies": {
     "isolated-vm": ">=5.0.0",
+    "pg": ">=8.11.0",
     "pino-pretty": ">=10.0.0"
   },
   "peerDependenciesMeta": {
     "isolated-vm": {
       "optional": true
     },
+    "pg": {
+      "optional": true
+    },
     "pino-pretty": {
       "optional": true
     }

package/src/cache/memory-cache.js

@@ -68,7 +68,10 @@ class MemoryCache {
     }
   }
 
-  /** @param {string} key */
+  /**
+   * Removes a key.
+   * @param {string} key
+   */
   async delete(key) {
     this._entries.delete(key);
   }

package/src/cache/redis-cache.js

@@ -67,7 +67,10 @@ class RedisCache {
     }
   }
 
-  /** @param {string} key */
+  /**
+   * Removes a key.
+   * @param {string} key
+   */
   async delete(key) {
     const del = this.client.del || this.client.unlink;
     if (typeof del === 'function')

package/src/cli/enhanced-cli-commands.js

@@ -12,6 +12,8 @@ const { logger } = require("../utils/logger.js");
 // Shared utilities
 const {
   parsePositiveInteger,
+  collectMin,
+  parseTolerance,
   generateCompletionScript,
   getVersion,
   getExtendedHelp,
@@ -28,6 +30,8 @@ const handlers = {
   config: () => require("./handlers/config-handler.js"),
   ingest: () => require("./handlers/ingest-handler.js"),
   query: () => require("./handlers/query-handler.js"),
+  eval: () => require("./handlers/eval-handler.js"),
+  generate: () => require("./handlers/generate-handler.js"),
 };
 /* eslint-enable global-require */
 
@@ -154,6 +158,81 @@ class EnhancedCLI {
           .handleQuery(this.program.opts(), question, options);
       });
 
+    this.program
+      .command("eval")
+      .description(
+        "Run an evaluation dataset; exit 1 on failures, low scores or regressions",
+      )
+      .argument("<dataset>", "Test set: .json (array or { cases }) or .jsonl")
+      .option("--baseline <report>", "Compare with a report saved by --output")
+      .option("--output <report>", "Save this run's report as JSON")
+      .option(
+        "--report <file>",
+        "Write a summary: .html for a page, .md for Markdown",
+      )
+      .option(
+        "--min <metric=value>",
+        "Minimum mean for a metric, e.g. faithfulness=0.8 (repeatable)",
+        collectMin,
+      )
+      .option(
+        "--tolerance <n>",
+        "Allowed drop in a metric's mean from the baseline (default 0.02)",
+        parseTolerance,
+      )
+      .option(
+        "--case-tolerance <n>",
+        "Drop in one case's score that lists it (default 0.1)",
+        parseTolerance,
+      )
+      .option(
+        "--concurrency <n>",
+        "Cases run at the same time",
+        parsePositiveInteger,
+        1,
+      )
+      .option(
+        "--timeout <ms>",
+        "Abort each case after this many milliseconds",
+        parsePositiveInteger,
+      )
+      .option("--top-k <n>", "Passages to retrieve", parsePositiveInteger, 3)
+      .option("--json", "Print the verdict, summary and comparison as JSON")
+      .action(async (dataset, options) => {
+        await handlers.eval().handleEval(this.program.opts(), dataset, options);
+      });
+
+    this.program
+      .command("generate-dataset")
+      .description(
+        "Write a test set from the index: the llm writes questions about each chunk",
+      )
+      .requiredOption("--output <file>", "Where to write the test set (.json)")
+      .option("--count <n>", "Cases to write at most", parsePositiveInteger, 20)
+      .option(
+        "--per-document <n>",
+        "Questions to ask per chunk",
+        parsePositiveInteger,
+        1,
+      )
+      .option(
+        "--min-faithfulness <n>",
+        "Share of an answer's sentences its chunk must support (default 1)",
+        parseTolerance,
+      )
+      .option(
+        "--concurrency <n>",
+        "Chunks processed at the same time",
+        parsePositiveInteger,
+        1,
+      )
+      .option("--force", "Replace --output if it exists")
+      .action(async (options) => {
+        await handlers
+          .generate()
+          .handleGenerateDataset(this.program.opts(), options);
+      });
+
     this.program
       .command("validate")
       .description("Validate the configuration and resolve its components")

package/src/cli/handlers/ai-handler.js

@@ -162,7 +162,7 @@ async function handleFeedback(globalOptions, question, docId, options = {}) {
         `Retriever "${components.names.retriever}" cannot list documents. Use "file-store".`,
       );
     }
-    if (!retriever.list().some((doc) => doc.id === docId)) {
+    if (!(await retriever.list()).some((doc) => doc.id === docId)) {
       throw new Error(
         `No indexed chunk with id "${docId}". Ids are shown by \`rag-pipeline ai search\`.`,
       );
@@ -197,7 +197,7 @@ async function handleSearch(globalOptions, question, options = {}) {
     const topK = options.topK || 3;
     const components = loadComponents({ config, configDir });
     const { retriever } = components;
-    const size = retriever.size ? retriever.size() : topK;
+    const size = retriever.size ? await retriever.size() : topK;
     // Re-score a larger pool so a boosted chunk can move into the top-k.
     const pool = Math.min(size, Math.max(topK * 4, topK + 10));
     const results = await retrieve({ components, question, topK: pool });