@devilsdev/rag-pipeline-utils 2.4.4 → 2.5.0

package/dist/types/utils/structured-logger.d.ts

@@ -0,0 +1,126 @@
+/**
+ * Enterprise structured logger with correlation tracking
+ */
+export class StructuredLogger {
+    constructor(options?: {});
+    serviceName: any;
+    environment: any;
+    logLevel: any;
+    enableCorrelation: boolean;
+    enableMetrics: boolean;
+    enableSecureLogging: boolean;
+    outputFormat: any;
+    metrics: {
+        logsWritten: number;
+        errorCount: number;
+        warnCount: number;
+        avgLatency: number;
+    };
+    /**
+     * Create correlation context for request tracking
+     * @param {string} correlationId - Optional correlation ID
+     * @param {Function} fn - Function to execute with context
+     * @returns {Promise} Result of function execution
+     */
+    withCorrelation(correlationId: string, fn: Function): Promise<any>;
+    /**
+     * Generate correlation ID
+     * @returns {string} Correlation ID
+     */
+    generateCorrelationId(): string;
+    /**
+     * Get current correlation context
+     * @returns {object|null} Current context
+     */
+    getContext(): object | null;
+    /**
+     * Create structured log entry
+     * @param {string} level - Log level
+     * @param {string} message - Log message
+     * @param {object} meta - Additional metadata
+     * @returns {object} Structured log entry
+     */
+    createLogEntry(level: string, message: string, meta?: object): object;
+    /**
+     * Check if log level should be written
+     * @param {string} level - Log level to check
+     * @returns {boolean} Whether to write log
+     */
+    shouldLog(level: string): boolean;
+    /**
+     * Write log entry to output
+     * @param {object} entry - Log entry
+     */
+    writeLog(entry: object): void;
+    /**
+     * Log error message
+     * @param {string} message - Error message
+     * @param {object} meta - Additional metadata
+     */
+    error(message: string, meta?: object): void;
+    /**
+     * Log warning message
+     * @param {string} message - Warning message
+     * @param {object} meta - Additional metadata
+     */
+    warn(message: string, meta?: object): void;
+    /**
+     * Log info message
+     * @param {string} message - Info message
+     * @param {object} meta - Additional metadata
+     */
+    info(message: string, meta?: object): void;
+    /**
+     * Log debug message
+     * @param {string} message - Debug message
+     * @param {object} meta - Additional metadata
+     */
+    debug(message: string, meta?: object): void;
+    /**
+     * Log trace message
+     * @param {string} message - Trace message
+     * @param {object} meta - Additional metadata
+     */
+    trace(message: string, meta?: object): void;
+    /**
+     * Create child logger with additional context
+     * @param {object} childMeta - Additional metadata for child logger
+     * @returns {ChildLogger} Child logger instance
+     */
+    child(childMeta?: object): ChildLogger;
+    /**
+     * Get logger metrics
+     * @returns {object} Logger performance metrics
+     */
+    getMetrics(): object;
+    /**
+     * Reset logger metrics
+     */
+    resetMetrics(): void;
+}
+/**
+ * Child logger with additional context
+ */
+export class ChildLogger {
+    constructor(parent: any, childMeta: any);
+    parent: any;
+    childMeta: any;
+    error(message: any, meta?: {}): void;
+    warn(message: any, meta?: {}): void;
+    info(message: any, meta?: {}): void;
+    debug(message: any, meta?: {}): void;
+    trace(message: any, meta?: {}): void;
+    child(additionalMeta?: {}): ChildLogger;
+    withCorrelation(correlationId: any, fn: any): any;
+    getContext(): any;
+    getMetrics(): any;
+}
+/**
+ * Create structured logger instance
+ * @param {object} options - Logger configuration
+ * @returns {StructuredLogger} Logger instance
+ */
+export function createLogger(options?: object): StructuredLogger;
+declare const defaultLogger: StructuredLogger;
+export declare let withCorrelation: any;
+export { defaultLogger as logger };

package/package.json

@@ -1,16 +1,28 @@
 {
   "name": "@devilsdev/rag-pipeline-utils",
-  "version": "2.4.4",
-  "description": "Composable RAG for Node.js — with built-in evaluation, citations, guardrails, and observability.",
+  "version": "2.5.0",
+  "description": "Composable retrieval-augmented generation (RAG) for Node.js: grounded citations, evaluation, guardrails, hybrid retrieval and a CLI.",
   "main": "dist/index.cjs",
   "type": "commonjs",
   "types": "dist/index.d.ts",
+  "typesVersions": {
+    "*": {
+      "experimental": [
+        "dist/experimental.d.ts"
+      ]
+    }
+  },
   "exports": {
     ".": {
       "types": "./dist/index.d.ts",
       "require": "./dist/index.cjs",
       "import": "./dist/index.mjs"
     },
+    "./experimental": {
+      "types": "./dist/experimental.d.ts",
+      "require": "./dist/experimental.cjs",
+      "import": "./dist/experimental.mjs"
+    },
     "./cli": "./bin/cli.js"
   },
   "bin": {
@@ -20,19 +32,21 @@
     "node": ">=18"
   },
   "scripts": {
-    "build": "node scripts/build.js && node scripts/generate-types.js",
+    "build": "node scripts/build.js && node scripts/generate-types.js && node scripts/check-type-exports.js",
     "prepack": "npm run build",
     "docs:validate": "node scripts/validate-docs-examples.js",
     "verify:esm": "node scripts/verify-esm.mjs",
     "prepublishOnly": "npm run lint && npm test && npm run docs:validate && npm run verify:esm",
     "guard:cjs": "node scripts/guard-cjs.js",
     "pretest": "npm run guard:cjs && node scripts/setup.js",
-    "test": "cross-env NODE_OPTIONS=--experimental-vm-modules jest --runInBand --config jest.config.js",
+    "test": "cross-env NODE_OPTIONS=--experimental-vm-modules jest --maxWorkers=50% --config jest.config.js",
+    "test:serial": "cross-env NODE_OPTIONS=--experimental-vm-modules jest --runInBand --config jest.config.js",
     "test:ci": "jest --coverage",
     "test:open-handles": "jest --detectOpenHandles --runInBand",
     "test:report": "node scripts/test-reporter.js",
     "posttest": "node scripts/generate-test-reports.js",
-    "format": "prettier --write \"src/**/*.{js,json}\"",
+    "format": "prettier --write --ignore-unknown \"{__tests__,scripts,bin,docs-site,examples,contracts}/**/*.{js,jsx,ts,tsx,mjs,cjs}\" \"*.{js,mjs,cjs}\" \"**/*.{json,md,mdx,yml,yaml,css,scss,html}\"",
+    "format:check": "prettier --check --ignore-unknown \"{__tests__,scripts,bin,docs-site,examples,contracts}/**/*.{js,jsx,ts,tsx,mjs,cjs}\" \"*.{js,mjs,cjs}\" \"**/*.{json,md,mdx,yml,yaml,css,scss,html}\"",
     "test:watch": "jest --watch",
     "lint": "eslint src --ext .js --ignore-path .gitignore",
     "lint:fix": "eslint . --ext .js,.jsx --ignore-path .gitignore --fix",
@@ -76,14 +90,8 @@
     "security:fix:force": "npm audit fix --force",
     "security:check": "npm audit --production && echo 'Production security check passed'",
     "security:report": "npm run security:audit:json && echo 'Security report generated: security-audit-report.json'",
-    "ci:hardening": "node scripts/ci/harden-workflows.js",
     "ci:audit": "node scripts/audit/gha-audit.js",
-    "ci:targeted-fixes": "node scripts/ci/targeted-security-fixes.js",
-    "ci:comprehensive-hardening": "node scripts/ci/comprehensive-hardening.js",
-    "ci:shell-security": "node scripts/ci/shell-security-hardening.js",
-    "ci:final-fixes": "node scripts/ci/final-shell-injection-fixes.js",
     "ci:test-workflows": "node scripts/ci/test-workflows.js",
-    "ci:fix-remaining": "node scripts/ci/fix-remaining-actions.js",
     "tests:categorize": "node scripts/test-categorizer.js",
     "tests:affected": "node scripts/run-affected.js",
     "tests:full": "jest --runInBand --reporters=default --json --outputFile=test-results-current.json",
@@ -92,6 +100,8 @@
     "test:integration": "jest __tests__/integration",
     "test:simple": "jest --maxWorkers=1 --no-coverage",
     "verify:pack": "node scripts/verify-npm-pack.js",
+    "smoke:pack": "node scripts/smoke-test-pack.js",
+    "demo:record": "vhs assets/demo.tape",
     "verify": "npm run lint && npm run typecheck --if-present && npm test",
     "sbom": "node scripts/generate-sbom.js",
     "observability:setup": "node scripts/setup-observability.js",
@@ -138,15 +148,12 @@
     "dist/",
     "src/",
     "!src/mocks/",
-    "!src/ai/module-template.js",
-    "!src/ai/index-fixed.js",
     "!src/dev/examples/",
     "!src/dev/README.md",
     "!src/utils/examples/",
     "!src/utils/ci/",
     "!src/ci/",
     "!src/scripts/",
-    "!src/cli/plugin-marketplace-commands.js",
     "bin/",
     "contracts/",
     ".ragrc.schema.json",
@@ -162,13 +169,15 @@
   },
   "lint-staged": {
     "src/**/*.{js,ts,jsx,tsx}": "eslint --fix",
-    "**/*.{js,ts,jsx,tsx,json,md,mdx,yml,yaml,css,scss,html}": "prettier --write --ignore-unknown"
+    "{__tests__,scripts,bin,docs-site,examples,contracts}/**/*.{js,jsx,ts,tsx,mjs,cjs}": "prettier --write --ignore-unknown",
+    "*.{js,mjs,cjs}": "prettier --write --ignore-unknown",
+    "**/*.{json,md,mdx,yml,yaml,css,scss,html}": "prettier --write --ignore-unknown"
   },
   "devDependencies": {
     "@babel/core": "^7.26.10",
     "@babel/preset-env": "^7.26.9",
-    "@docusaurus/core": "3.5.2",
-    "@docusaurus/preset-classic": "3.5.2",
+    "@docusaurus/core": "3.10.0",
+    "@docusaurus/preset-classic": "3.10.0",
     "@octokit/rest": "^21.1.1",
     "@opentelemetry/api": "^1.9.0",
     "@opentelemetry/auto-instrumentations-node": "^0.52.1",
@@ -180,17 +189,10 @@
     "@opentelemetry/sdk-node": "^0.53.0",
     "@opentelemetry/sdk-trace-base": "^1.26.0",
     "@opentelemetry/semantic-conventions": "^1.27.0",
-    "@semantic-release/changelog": "^6.0.3",
-    "@semantic-release/commit-analyzer": "^13.0.1",
-    "@semantic-release/git": "^10.0.1",
-    "@semantic-release/github": "^9.2.6",
-    "@semantic-release/npm": "^11.0.3",
-    "@semantic-release/release-notes-generator": "^14.0.3",
     "@types/jest": "^29.5.14",
+    "@types/node": "22.14.1",
     "acorn": "8.15.0",
     "babel-jest": "^29.7.0",
-    "codecov": "3.6.2",
-    "conventional-changelog-conventionalcommits": "^8.0.0",
     "cross-env": "7.0.3",
     "esbuild": "^0.25.2",
     "eslint": "^8.57.1",
@@ -207,24 +209,46 @@
     "prettier": "^3.5.3",
     "prop-types": "^15.8.1",
     "rimraf": "^6.0.1",
-    "semantic-release": "^21.1.2",
     "sharp": "^0.34.1",
     "ts-jest": "^29.3.2",
+    "typescript": "5.8.3",
     "why-is-node-running": "^3.2.2"
   },
   "dependencies": {
     "ajv": "^8.17.1",
-    "axios": "^1.8.4",
-    "chalk": "^5.4.1",
+    "ajv-formats": "2.1.1",
+    "axios": "^1.21.0",
+    "chalk": "4.1.2",
     "chokidar": "^4.0.3",
+    "cli-table3": "0.6.5",
     "commander": "^13.1.0",
     "csv-parse": "^5.6.0",
-    "dotenv": "^16.5.0",
     "fast-glob": "^3.3.3",
     "inquirer": "^8.2.5",
     "jsdom": "^26.1.0",
     "jsonwebtoken": "9.0.2",
+    "marked": "15.0.12",
     "openai": "^4.93.0",
+    "ora": "5.4.1",
     "pino": "^9.6.0"
+  },
+  "peerDependencies": {
+    "isolated-vm": ">=5.0.0",
+    "pino-pretty": ">=10.0.0"
+  },
+  "peerDependenciesMeta": {
+    "isolated-vm": {
+      "optional": true
+    },
+    "pino-pretty": {
+      "optional": true
+    }
+  },
+  "overrides": {
+    "fast-uri": "^3.1.9",
+    "follow-redirects": "^1.16.2",
+    "form-data": "^4.0.6",
+    "ws@^8.0.0": "^8.21.0",
+    "zod": "^4.6.7"
   }
 }

package/src/agentic/agentic-pipeline.js

@@ -5,6 +5,8 @@ const { DAG } = require('../dag/dag-engine');
 const { QueryPlanner } = require('./query-planner');
 const { IterativeRetriever } = require('./iterative-retriever');
 const { SelfCritiqueChecker } = require('./self-critique');
+const { embedQuery, canEmbed } = require('../utils/embed-query');
+const { createRunAbort } = require('../utils/run-abort');
 
 /**
  * Default configuration for the AgenticPipeline.
@@ -14,6 +16,7 @@ const DEFAULT_CONFIG = {
   maxConcurrency: 3,
   timeout: 60000,
   enablePlanning: true,
+  enableLlmPlanning: false,
   enableCritique: true,
   enableIterativeRetrieval: false,
   maxIterations: 3,
@@ -33,6 +36,9 @@ class AgenticPipeline extends EventEmitter {
    * @param {number} [options.maxConcurrency=3] - Max concurrent DAG node execution
    * @param {number} [options.timeout=60000] - Global timeout in ms
    * @param {boolean} [options.enablePlanning=true] - Whether to decompose queries
+   * @param {boolean} [options.enableLlmPlanning=false] - Ask the llm for the
+   *   sub-queries (strictly validated, heuristic fallback); costs one extra
+   *   llm.generate() call per run
    * @param {boolean} [options.enableCritique=true] - Whether to self-critique answers
    * @param {boolean} [options.enableIterativeRetrieval=false] - Whether to use iterative retrieval
    * @param {number} [options.maxIterations=3] - Max iterations for iterative retrieval
@@ -62,7 +68,16 @@ class AgenticPipeline extends EventEmitter {
    * @param {object} params.retriever - Retriever instance with .retrieve() method
    * @param {object} params.llm - LLM instance with .generate() method
    * @param {Array<number>} [params.queryVector] - Optional pre-computed query vector
+   * @param {object} [params.embedder] - Optional embedder; when given, each
+   *   sub-query is embedded on its own instead of reusing queryVector
    * @param {object} [params.options={}] - Extra options passed to llm.generate()
+   * @param {AbortSignal} [params.options.signal] - Cancels the run. The run
+   *   has its own AbortController (also aborted after `timeout`); its signal
+   *   is passed to retriever.retrieve({ ..., signal }), embedder.embedQuery /
+   *   embed(text, { signal }) and llm.generate(query, docs, { ...options,
+   *   signal }), and no later stage starts once it aborts. The returned
+   *   promise then rejects with Error('Execution timeout') or
+   *   Error('Execution aborted').
    * @returns {Promise<{
    *   success: boolean,
    *   query: string,
@@ -74,7 +89,7 @@ class AgenticPipeline extends EventEmitter {
    * }>}
    * @throws {Error} If retriever or llm is missing
    */
-  async run({ query, retriever, llm, queryVector, options = {} }) {
+  async run({ query, retriever, llm, queryVector, embedder, options = {} }) {
     // 1. Validate inputs
     if (!retriever || typeof retriever.retrieve !== 'function') {
       throw new Error(
@@ -92,10 +107,44 @@ class AgenticPipeline extends EventEmitter {
       );
     }
 
+    // One AbortController per run: aborted on timeout or when the caller's
+    // options.signal aborts. Its signal goes to every component call.
+    const runAbort = createRunAbort({
+      timeout: this.config.timeout,
+      signal: options && options.signal,
+    });
+    try {
+      return await runAbort.race(
+        this._execute(
+          { query, retriever, llm, queryVector, embedder, options },
+          runAbort,
+        ),
+      );
+    } finally {
+      runAbort.cleanup();
+    }
+  }
+
+  /**
+   * @param {object} params - Validated run() parameters
+   * @param {object} runAbort - Per-run cancellation (see utils/run-abort)
+   * @returns {Promise<object>}
+   * @private
+   */
+  async _execute(
+    { query, retriever, llm, queryVector, embedder, options },
+    runAbort,
+  ) {
+    const { signal } = runAbort;
+    const llmOptions = { ...(options || {}), signal };
+
     // Phase 1: Plan
     let subQueries;
     if (this.config.enablePlanning) {
-      const plan = this.planner.plan(query);
+      const plan = this.config.enableLlmPlanning
+        ? await this.planner.plan(query, { llm, signal })
+        : this.planner.plan(query);
+      runAbort.throwIfAborted();
       subQueries = plan.subQueries;
       this.emit('planned', plan);
     } else {
@@ -111,19 +160,31 @@ class AgenticPipeline extends EventEmitter {
       dag.addNode(
         `retrieve-${i}`,
         async () => {
+          runAbort.throwIfAborted();
+          // queryVector belongs to the original query; a decomposed
+          // sub-query needs its own embedding when an embedder is available.
+          const needsOwnVector =
+            canEmbed(embedder) && (sq.subQuery !== query || !queryVector);
+          const subQueryVector = needsOwnVector
+            ? await embedQuery(embedder, sq.subQuery, { signal })
+            : queryVector;
+          runAbort.throwIfAborted();
           if (this.iterativeRetriever) {
             const result = await this.iterativeRetriever.retrieve({
               query: sq.subQuery,
               retriever,
               llm,
-              queryVector,
+              queryVector: subQueryVector,
+              embedder,
+              signal,
             });
             return result.results;
           }
           return await retriever.retrieve({
             query: sq.subQuery,
-            queryVector,
+            queryVector: subQueryVector,
             topK: 10,
+            signal,
           });
         },
         { timeout: this.config.timeout / 2, priority: sq.priority },
@@ -132,6 +193,7 @@ class AgenticPipeline extends EventEmitter {
 
     // Add merge node
     dag.addNode('merge', async (inputs) => {
+      runAbort.throwIfAborted();
       const allResults = Array.isArray(inputs)
         ? inputs.flat()
         : [inputs].flat();
@@ -155,8 +217,10 @@ class AgenticPipeline extends EventEmitter {
     // Add critique + generate nodes
     if (this.config.enableCritique) {
       dag.addNode('pre-generate', async (mergedResults) => {
+        runAbort.throwIfAborted();
         // Generate initial answer for critique
-        const answer = await llm.generate(query, mergedResults, options);
+        const answer = await llm.generate(query, mergedResults, llmOptions);
+        runAbort.throwIfAborted();
         const critique = this.critiqueChecker.check(
           String(answer),
           mergedResults,
@@ -167,6 +231,7 @@ class AgenticPipeline extends EventEmitter {
       dag.connect('merge', 'pre-generate');
 
       dag.addNode('generate', async ({ answer, mergedResults, critique }) => {
+        runAbort.throwIfAborted();
         if (critique.approved) {
           return { answer, results: mergedResults, critique };
         }
@@ -178,7 +243,7 @@ class AgenticPipeline extends EventEmitter {
         const refinedAnswer = await llm.generate(
           refinedPrompt,
           mergedResults,
-          options,
+          llmOptions,
         );
         return {
           answer: String(refinedAnswer),
@@ -189,7 +254,8 @@ class AgenticPipeline extends EventEmitter {
       dag.connect('pre-generate', 'generate');
     } else {
       dag.addNode('generate', async (mergedResults) => {
-        const answer = await llm.generate(query, mergedResults, options);
+        runAbort.throwIfAborted();
+        const answer = await llm.generate(query, mergedResults, llmOptions);
         return { answer: String(answer), results: mergedResults };
       });
       dag.connect('merge', 'generate');
@@ -206,6 +272,8 @@ class AgenticPipeline extends EventEmitter {
       continueOnError: true,
       gracefulDegradation: true,
     });
+    // continueOnError lets nodes degrade; an abort must not look like success.
+    runAbort.throwIfAborted();
 
     // Extract final result from generate node
     const output = dagResult?.answer

package/src/agentic/iterative-retriever.js

@@ -3,6 +3,7 @@
 const { EventEmitter } = require('events');
 const { retry } = require('../utils/retry');
 const { tokenize, computeJaccardSimilarity } = require('../evaluate/scoring');
+const { embedQuery, canEmbed } = require('../utils/embed-query');
 
 /**
  * Default configuration for the IterativeRetriever.
@@ -110,10 +111,26 @@ class IterativeRetriever extends EventEmitter {
    * @param {object} params.retriever - Retriever instance with .retrieve() method
    * @param {object} [params.llm] - Optional LLM for query refinement
    * @param {Array<number>} [params.queryVector] - Optional pre-computed query vector
+   * @param {object} [params.embedder] - Optional embedder; when given, each
+   *   refined query is re-embedded instead of reusing queryVector
+   * @param {AbortSignal} [params.signal] - Cancels the loop: passed to
+   *   retriever.retrieve({ ..., signal }), the embedder and llm.generate(...,
+   *   { signal }); no further iteration, retry or refinement starts once it
+   *   has aborted, and the call rejects with the signal's reason
    * @returns {Promise<{results: Array<object>, iterations: number, converged: boolean, coverage: number}>}
    * @throws {Error} If query or retriever is missing
    */
-  async retrieve({ query, retriever, llm, queryVector }) {
+  async retrieve({ query, retriever, llm, queryVector, embedder, signal }) {
+    const throwIfAborted = () => {
+      if (signal && signal.aborted) {
+        throw signal.reason instanceof Error
+          ? signal.reason
+          : new Error('Execution aborted');
+      }
+    };
+    // Only add the signal when there is one, so components keep the
+    // argument shapes they had before.
+    const withSignal = (obj) => (signal ? { ...obj, signal } : obj);
     // 1. Validate inputs
     if (!query || typeof query !== 'string' || !query.trim()) {
       throw new Error(
@@ -135,13 +152,31 @@ class IterativeRetriever extends EventEmitter {
 
     // 3. Retrieval loop
     while (iterations < this.config.maxIterations) {
+      throwIfAborted();
       // a. Retrieve with retry
       const retrieveQuery = currentQuery;
-      const newResults = await retry(
-        () =>
-          retriever.retrieve({ query: retrieveQuery, queryVector, topK: 10 }),
-        this.config.retryOptions,
-      );
+      // Without an embedder the caller's vector is all we have; it only
+      // matches the original query, so refined queries need re-embedding.
+      const retrieveVector =
+        canEmbed(embedder) && (retrieveQuery !== query || !queryVector)
+          ? await embedQuery(
+              embedder,
+              retrieveQuery,
+              signal ? { signal } : undefined,
+            )
+          : queryVector;
+      const newResults = await retry(() => {
+        // Do not retry (or start) a retrieval after an abort.
+        throwIfAborted();
+        return retriever.retrieve(
+          withSignal({
+            query: retrieveQuery,
+            queryVector: retrieveVector,
+            topK: 10,
+          }),
+        );
+      }, this.config.retryOptions);
+      throwIfAborted();
 
       // b. Merge and deduplicate
       const resultsArray = Array.isArray(newResults) ? newResults : [];
@@ -173,16 +208,18 @@ class IterativeRetriever extends EventEmitter {
         // Try LLM-based refinement first
         if (llm && typeof llm.generate === 'function') {
           try {
-            refined = await llm.generate(
-              `Given the query "${query}" and that current results don't fully cover the topic, generate a refined search query to find missing information.`,
-              accumulatedResults,
-            );
+            const prompt = `Given the query "${query}" and that current results don't fully cover the topic, generate a refined search query to find missing information.`;
+            refined = signal
+              ? await llm.generate(prompt, accumulatedResults, { signal })
+              : await llm.generate(prompt, accumulatedResults);
             if (refined && typeof refined === 'string' && refined.trim()) {
               refined = refined.trim();
             } else {
               refined = null;
             }
           } catch (_err) {
+            // An abort is not an LLM failure: stop instead of falling back.
+            throwIfAborted();
             // LLM failed -- fall through to heuristic
             refined = null;
           }