@devilsdev/rag-pipeline-utils 2.6.2 → 2.7.0

package/src/retrieval/qdrant-retriever.js

@@ -0,0 +1,361 @@
+'use strict';
+
+/**
+ * Retriever on Qdrant, through its REST API with the built-in fetch: no
+ * client package is needed.
+ *
+ * Qdrant point ids must be integers or UUIDs, so each document id is
+ * mapped to a UUID derived from it, and the original id is kept in the
+ * payload (`rag_id`) with `content` and `metadata`. Writes wait for Qdrant
+ * to apply them (`wait=true`), so a query right after ingest sees them.
+ * Written against the Qdrant 1.20 API (`/points/query`).
+ *
+ * The collection can change without this process knowing, so the
+ * retriever has no cache identity unless you give it a `cacheKey`.
+ */
+
+const crypto = require('crypto');
+const { throwIfAborted } = require('../connectors/connector-utils');
+
+const COLLECTION = /^[A-Za-z0-9_-]{1,255}$/;
+const DEFAULT_COLLECTION = 'rag_documents';
+const PAGE = 256;
+const UPSERT_BATCH = 256;
+
+/** A stable UUID (version 8, RFC 9562) for a document id. */
+function pointId(id) {
+  const h = crypto.createHash('sha256').update(String(id)).digest();
+  h[6] = (h[6] & 0x0f) | 0x80;
+  h[8] = (h[8] & 0x3f) | 0x80;
+  const x = h.subarray(0, 16).toString('hex');
+  return `${x.slice(0, 8)}-${x.slice(8, 12)}-${x.slice(12, 16)}-${x.slice(16, 20)}-${x.slice(20, 32)}`;
+}
+
+function checkVector(vector, id) {
+  if (!Array.isArray(vector) || vector.length === 0) {
+    throw new Error(`QdrantRetriever: document "${id}" has no vector`);
+  }
+  if (!vector.every((x) => typeof x === 'number' && Number.isFinite(x))) {
+    throw new Error(
+      `QdrantRetriever: document "${id}" has a vector with a non-finite value`,
+    );
+  }
+}
+
+// Matches every point this class wrote, and nothing else in the collection.
+const OURS = { must_not: [{ is_empty: { key: 'rag_id' } }] };
+
+/**
+ * Retriever on Qdrant over its REST API, with no client package. Creates
+ * the collection on first store; similarity is cosine.
+ */
+class QdrantRetriever {
+  /**
+   * @param {Object} [options]
+   * @param {string} [options.url='http://localhost:6333'] - Qdrant's REST endpoint
+   * @param {string} [options.apiKey] - Sent as the `api-key` header
+   * @param {string} [options.collection='rag_documents']
+   * @param {number} [options.dimensions] - Vector size for a new collection; defaults to the first stored vector's
+   * @param {boolean} [options.createCollection=true] - Create the collection on first store
+   * @param {string} [options.cacheKey] - Cache identity; change it when the collection changes
+   * @param {Function} [options.fetch] - fetch implementation (defaults to the global)
+   */
+  constructor(options = {}) {
+    const {
+      url = 'http://localhost:6333',
+      apiKey,
+      collection = DEFAULT_COLLECTION,
+      dimensions,
+      createCollection = true,
+      cacheKey,
+      fetch: fetchImpl = globalThis.fetch,
+    } = options;
+    if (!COLLECTION.test(collection)) {
+      throw new Error(
+        `QdrantRetriever: invalid collection name "${collection}" (letters, digits, "_" and "-")`,
+      );
+    }
+    if (
+      dimensions !== undefined &&
+      !(Number.isInteger(dimensions) && dimensions > 0)
+    ) {
+      throw new Error(
+        'QdrantRetriever: `dimensions` must be a positive integer',
+      );
+    }
+    if (typeof fetchImpl !== 'function') {
+      throw new Error('QdrantRetriever needs fetch (Node.js 18 or later)');
+    }
+    this.url = String(url).replace(/\/+$/, '');
+    this.collection = collection;
+    this.dimensions = dimensions;
+    this.createCollection = createCollection;
+    this.cacheKey = cacheKey;
+    this._apiKey = apiKey;
+    this._fetch = fetchImpl;
+    this._ready = null;
+  }
+
+  /**
+   * Cache identity for createRagPipeline({ cache }): none unless you set
+   * `cacheKey`, since other clients can change the collection.
+   * @param {'retriever'} role
+   * @returns {string|null}
+   */
+  getCacheKey(role) {
+    if (role !== 'retriever' || !this.cacheKey) return null;
+    return `qdrant|${this.url}|${this.collection}|${this.cacheKey}`;
+  }
+
+  /**
+   * One REST call to the collection; resolves Qdrant's `result`.
+   * @private
+   * @param {string} method
+   * @param {string} path - After /collections/<name>
+   * @param {Object} [body]
+   * @param {{ signal?: AbortSignal, allow404?: boolean }} [options] - allow404 resolves null for a missing collection
+   * @returns {Promise<any>}
+   */
+  async _request(method, path, body, { signal, allow404 = false } = {}) {
+    throwIfAborted(signal);
+    const headers = {
+      'content-type': 'application/json',
+      // fetch asks for gzip by default, and Qdrant's compressed responses
+      // stall each request by about 40 ms; uncompressed they take ~1 ms.
+      'accept-encoding': 'identity',
+    };
+    if (this._apiKey) headers['api-key'] = this._apiKey;
+    let res;
+    try {
+      res = await this._fetch(
+        `${this.url}/collections/${this.collection}${path}`,
+        {
+          method,
+          headers,
+          ...(body !== undefined && { body: JSON.stringify(body) }),
+          ...(signal && { signal }),
+        },
+      );
+    } catch (err) {
+      throwIfAborted(signal);
+      throw new Error(
+        `QdrantRetriever: cannot reach Qdrant at ${this.url} (${err.message})`,
+      );
+    }
+    if (res.status === 404 && allow404) return null;
+    const text = await res.text();
+    let json;
+    try {
+      json = text ? JSON.parse(text) : {};
+    } catch {
+      json = {};
+    }
+    if (!res.ok) {
+      const detail =
+        (json.status && json.status.error) || text || res.statusText;
+      if (res.status === 401 || res.status === 403) {
+        throw new Error(
+          `QdrantRetriever: Qdrant refused the request (${res.status}); check QDRANT_API_KEY`,
+        );
+      }
+      throw new Error(
+        `QdrantRetriever: ${method} ${path || '/'} failed (${res.status}): ${detail}`,
+      );
+    }
+    return json.result;
+  }
+
+  _ensureCollection(dimensions) {
+    if (!this.createCollection) return Promise.resolve();
+    if (!this._ready) {
+      this._ready = (async () => {
+        const existing = await this._request('GET', '', undefined, {
+          allow404: true,
+        });
+        if (!existing) {
+          await this._request('PUT', '', {
+            vectors: {
+              size: this.dimensions || dimensions,
+              distance: 'Cosine',
+            },
+          });
+        }
+        // removeBySource filters on metadata.source; index it.
+        await this._request('PUT', '/index?wait=true', {
+          field_name: 'metadata.source',
+          field_schema: 'keyword',
+        });
+      })().catch((err) => {
+        this._ready = null;
+        throw err;
+      });
+    }
+    return this._ready;
+  }
+
+  /**
+   * Inserts or replaces documents.
+   * @param {Array<{ id: string, vector: number[], content?: string, metadata?: Object }>} docs
+   * @returns {Promise<void>}
+   */
+  async store(docs) {
+    if (!Array.isArray(docs))
+      throw new Error('QdrantRetriever.store() takes an array');
+    if (docs.length === 0) return;
+    let dimensions = null;
+    const points = docs.map((doc) => {
+      const id = doc && doc.id;
+      if (typeof id !== 'string' || id.length === 0) {
+        throw new Error(
+          'QdrantRetriever.store(): every document needs a string id',
+        );
+      }
+      checkVector(doc.vector, id);
+      if (dimensions === null) dimensions = doc.vector.length;
+      else if (doc.vector.length !== dimensions) {
+        throw new Error(
+          `QdrantRetriever.store(): document "${id}" has ${doc.vector.length} dimensions, the batch has ${dimensions}`,
+        );
+      }
+      return {
+        id: pointId(id),
+        vector: doc.vector,
+        payload: {
+          rag_id: id,
+          content: doc.content || '',
+          metadata: doc.metadata || {},
+        },
+      };
+    });
+    await this._ensureCollection(dimensions);
+    for (let i = 0; i < points.length; i += UPSERT_BATCH) {
+      await this._request('PUT', '/points?wait=true', {
+        points: points.slice(i, i + UPSERT_BATCH),
+      });
+    }
+  }
+
+  /**
+   * The documents most similar to a query vector.
+   * @param {number[]|{ queryVector: number[], topK?: number, signal?: AbortSignal }} request
+   * @param {number} [k=10] - Results when `request` is a vector
+   * @returns {Promise<Array<{ id: string, score: number, content: string, metadata: Object }>>}
+   */
+  async retrieve(request, k = 10) {
+    const {
+      queryVector,
+      topK = k,
+      signal,
+    } = Array.isArray(request) ? { queryVector: request } : request || {};
+    if (!Number.isInteger(topK) || topK < 1) {
+      throw new Error(
+        'QdrantRetriever.retrieve(): topK must be a positive integer',
+      );
+    }
+    checkVector(queryVector, 'query');
+    const result = await this._request(
+      'POST',
+      '/points/query',
+      { query: queryVector, limit: topK, with_payload: true, filter: OURS },
+      { signal, allow404: true },
+    );
+    throwIfAborted(signal);
+    return ((result && result.points) || []).map((p) => ({
+      id: p.payload.rag_id,
+      score: p.score,
+      content: p.payload.content || '',
+      metadata: p.payload.metadata || {},
+    }));
+  }
+
+  /**
+   * Deletes the chunks of one source (`metadata.source`), for re-ingesting.
+   * @param {string} source
+   */
+  async removeBySource(source) {
+    await this._request(
+      'POST',
+      '/points/delete?wait=true',
+      {
+        filter: {
+          must: [{ key: 'metadata.source', match: { value: String(source) } }],
+        },
+      },
+      { allow404: true },
+    );
+  }
+
+  /**
+   * Deletes documents by id.
+   * @param {string[]} ids
+   */
+  async delete(ids) {
+    await this._request(
+      'POST',
+      '/points/delete?wait=true',
+      { points: ids.map(pointId) },
+      { allow404: true },
+    );
+  }
+
+  /** Deletes every document this class stored; the collection stays. */
+  async clear() {
+    await this._request(
+      'POST',
+      '/points/delete?wait=true',
+      { filter: OURS },
+      { allow404: true },
+    );
+  }
+
+  /** @returns {Promise<number>} */
+  async size() {
+    const result = await this._request(
+      'POST',
+      '/points/count',
+      { exact: true, filter: OURS },
+      { allow404: true },
+    );
+    return result ? result.count : 0;
+  }
+
+  /**
+   * Every document, without vectors.
+   * @param {{ limit?: number }} [options]
+   * @returns {Promise<Array<{ id: string, content: string, metadata: Object }>>}
+   */
+  async list({ limit } = {}) {
+    const out = [];
+    let offset;
+    do {
+      const result = await this._request(
+        'POST',
+        '/points/scroll',
+        {
+          limit: limit ? Math.min(PAGE, limit - out.length) : PAGE,
+          with_payload: true,
+          with_vector: false,
+          filter: OURS,
+          ...(offset !== undefined && offset !== null && { offset }),
+        },
+        { allow404: true },
+      );
+      if (!result) break;
+      for (const p of result.points) {
+        out.push({
+          id: p.payload.rag_id,
+          content: p.payload.content || '',
+          metadata: p.payload.metadata || {},
+        });
+      }
+      offset = result.next_page_offset;
+    } while (
+      offset !== undefined &&
+      offset !== null &&
+      (!limit || out.length < limit)
+    );
+    return out;
+  }
+}
+
+module.exports = { QdrantRetriever, pointId };

package/src/security/validator.js

@@ -1,5 +1,13 @@
 'use strict';
 
+/**
+ * Checks input against a maximum length and a fixed list of disallowed
+ * tokens (code fences, `${`, `</script`, `--`, `;`, `||`, `&&`).
+ * @param {*} input - Converted to a string
+ * @param {Object} [opts]
+ * @param {number} [opts.maxLength=20000]
+ * @returns {{valid: boolean, reasons: string[], sanitized: string}} `sanitized` is the input cut to `maxLength`
+ */
 function validateInput(input, opts = {}) {
   const maxLength = opts.maxLength || 20000;
   const inputStr = String(input);

package/src/testing.js

@@ -124,7 +124,9 @@ class EchoLLM {
    *   there are no documents and no `answer`
    */
   constructor({ answer, fallback = 'No information found.' } = {}) {
+    /** The fixed answer, or a function of (query, docs) that returns it. */
     this.answer = answer;
+    /** Returned when there are no documents and no `answer`. */
     this.fallback = fallback;
     /** @type {Array<{query: string, docs: Array<Object>, options: Object}>} */
     this.calls = [];

package/src/utils/batch-processor.js

@@ -111,9 +111,13 @@ class BatchProcessor extends EventEmitter {
   /**
    * Process items in optimized batches
    *
-   * @param {Array<string|Object>} items - Items to process
-   * @param {Function} processFn - Function to process each batch
-   * @param {Object} options - Processing options
+   * @param {Array<string|Object>} items - Items to process; objects are read from `text` or `content`
+   * @param {function(string[], {batchIndex: number, estimatedTokens: number, signal: AbortSignal}): Promise<Array>} processFn - Processes one batch of texts, returning one result per text
+   * @param {Object} [options] - Processing options
+   * @param {function(Object): void} [options.onProgress]
+   * @param {function(Object): void} [options.onBatchComplete]
+   * @param {AbortController} [options.abortController] - Abort to stop between batches
+   * @param {boolean} [options.returnOriginalOrder=true]
    * @returns {Promise<Array>} Processed results
    */
   async processBatches(items, processFn, options = {}) {

package/src/utils/error-formatter.js

@@ -11,7 +11,9 @@
  */
 
 /**
- * Error codes organized by category
+ * Known error codes, grouped by prefix (CONFIG_, PLUGIN_, PIPELINE_,
+ * VALIDATION_, FS_, NETWORK_, HOT_RELOAD_, DAG_, AUTH_). Each value equals
+ * its key.
  */
 const ERROR_CODES = {
   // Configuration errors (CONFIG_*)
@@ -75,7 +77,9 @@ const ERROR_CODES = {
 };
 
 /**
- * Error message templates with suggestions
+ * Message, suggestion and documentation templates keyed by error code.
+ * Placeholders such as {path} are filled from the context passed to
+ * formatError. Codes without an entry fall back to UNKNOWN_ERROR.
  */
 const ERROR_TEMPLATES = {
   [ERROR_CODES.CONFIG_NOT_FOUND]: {
@@ -351,9 +355,15 @@ const ERROR_TEMPLATES = {
 };
 
 /**
- * Enhanced Error class with code and context
+ * Error carrying a code, the context used to build its message, suggested
+ * fixes and a documentation path, all taken from ERROR_TEMPLATES.
  */
 class EnhancedError extends Error {
+  /**
+   * @param {string} code - Error code, normally one of ERROR_CODES
+   * @param {Object<string, *>} [context={}] - Values for the template placeholders
+   * @param {Error|null} [originalError=null] - Underlying error, if any
+   */
   constructor(code, context = {}, originalError = null) {
     const formattedMessage = formatError(code, context);
     super(formattedMessage.message);
@@ -373,14 +383,17 @@ class EnhancedError extends Error {
   }
 
   /**
-   * Get formatted error message with all details
+   * Format this error for display with formatErrorMessage's default options.
+   * @returns {string} Multi-line message with code, suggestions and documentation link
    */
   getFormattedMessage() {
     return formatErrorMessage(this);
   }
 
   /**
-   * Convert to JSON for logging
+   * Plain object for JSON logging, including the stack and the original
+   * error's message and stack (or null).
+   * @returns {{name: string, code: string, message: string, suggestions: string[], documentation: string, context: Object<string, *>, timestamp: string, stack: string, originalError: ({message: string, stack: string}|null)}}
    */
   toJSON() {
     return {
@@ -403,11 +416,13 @@ class EnhancedError extends Error {
 }
 
 /**
- * Format an error message with context
+ * Fill the template for an error code with context values. Unknown codes use
+ * the UNKNOWN_ERROR template. Each placeholder is replaced at its first
+ * occurrence only; placeholders with no context value stay as written.
  *
  * @param {string} code - Error code
- * @param {object} context - Context variables
- * @returns {object} Formatted error details
+ * @param {Object<string, *>} [context={}] - Placeholder values, converted with String()
+ * @returns {{code: string, message: string, suggestions: string[], documentation: string, context: Object<string, *>}} Formatted error details
  */
 function formatError(code, context = {}) {
   const template =
@@ -438,10 +453,17 @@ function formatError(code, context = {}) {
 }
 
 /**
- * Format error message for display
+ * Build a multi-line display string for an error. Fields the error lacks
+ * (code, suggestions, documentation) are skipped.
  *
  * @param {Error|EnhancedError} error - Error to format
- * @param {object} options - Formatting options
+ * @param {Object} [options] - Formatting options
+ * @param {boolean} [options.includeCode=true] - Prefix the message with the error code
+ * @param {boolean} [options.includeSuggestions=true] - List the suggestions
+ * @param {boolean} [options.includeDocumentation=true] - Add the documentation URL
+ * @param {boolean} [options.includeContext=false] - List the context values as JSON
+ * @param {boolean} [options.includeStack=false] - Append the stack trace
+ * @param {boolean} [options.color=false] - Accepted but unused
  * @returns {string} Formatted error message
  */
 function formatErrorMessage(error, options = {}) {
@@ -500,11 +522,11 @@ function formatErrorMessage(error, options = {}) {
 }
 
 /**
- * Create an enhanced error
+ * Create an EnhancedError. Same as calling its constructor.
  *
  * @param {string} code - Error code
- * @param {object} context - Context variables
- * @param {Error} originalError - Original error (optional)
+ * @param {Object<string, *>} [context={}] - Placeholder values
+ * @param {Error|null} [originalError=null] - Underlying error, if any
  * @returns {EnhancedError} Enhanced error instance
  */
 function createError(code, context = {}, originalError = null) {
@@ -512,11 +534,12 @@ function createError(code, context = {}, originalError = null) {
 }
 
 /**
- * Wrap an existing error with enhanced context
+ * Wrap an existing error in an EnhancedError. The original message is added
+ * to the context as originalMessage and the error is kept as originalError.
  *
  * @param {Error} error - Original error
  * @param {string} code - Error code
- * @param {object} context - Additional context
+ * @param {Object<string, *>} [context={}] - Additional placeholder values
  * @returns {EnhancedError} Enhanced error
  */
 function wrapError(error, code, context = {}) {
@@ -531,7 +554,7 @@ function wrapError(error, code, context = {}) {
 }
 
 /**
- * Check if an error is an EnhancedError
+ * Check whether a value is an EnhancedError instance.
  *
  * @param {*} error - Value to check
  * @returns {boolean} True if enhanced error
@@ -541,10 +564,12 @@ function isEnhancedError(error) {
 }
 
 /**
- * Get error severity level
+ * Severity for an error code: 'critical' for PIPELINE_CREATION_FAILED,
+ * CONFIG_NOT_FOUND, FS_PERMISSION_DENIED and AUTH_INVALID_TOKEN, 'warning'
+ * for the HOT_RELOAD_FAILED and HOT_RELOAD_STATE_ERROR codes, 'error' otherwise.
  *
  * @param {string} code - Error code
- * @returns {string} Severity level (critical, error, warning)
+ * @returns {'critical'|'warning'|'error'} Severity level
  */
 function getErrorSeverity(code) {
   const critical = [
@@ -565,10 +590,13 @@ function getErrorSeverity(code) {
 }
 
 /**
- * Log error with appropriate formatting
+ * Format an error and write it to a logger. Warnings go to logger.warn,
+ * everything else to logger.error; critical errors include the stack trace.
+ * Errors that are not EnhancedError are treated as 'error'.
  *
  * @param {Error|EnhancedError} error - Error to log
- * @param {object} logger - Logger instance (optional)
+ * @param {Pick<Console, 'error'|'warn'>} [logger=console] - Logger to write to
+ * @returns {void}
  */
 function logError(error, logger = console) {
   const severity = isEnhancedError(error)

package/src/utils/logger.js

@@ -154,18 +154,38 @@ class StructuredLogger {
     this.logger[level](logData, message);
   }
 
+  /**
+   * Logs at info level.
+   * @param {string} message
+   * @param {*} [meta] - An object, Error or value to attach
+   */
   info(message, meta) {
     this._log('info', message, meta);
   }
 
+  /**
+   * Logs at error level.
+   * @param {string} message
+   * @param {*} [meta] - An object, Error or value to attach
+   */
   error(message, meta) {
     this._log('error', message, meta);
   }
 
+  /**
+   * Logs at warn level.
+   * @param {string} message
+   * @param {*} [meta] - An object, Error or value to attach
+   */
   warn(message, meta) {
     this._log('warn', message, meta);
   }
 
+  /**
+   * Logs at debug level.
+   * @param {string} message
+   * @param {*} [meta] - An object, Error or value to attach
+   */
   debug(message, meta) {
     this._log('debug', message, meta);
   }
@@ -207,7 +227,7 @@ class StructuredLogger {
   }
 }
 
-// Create default logger instance
+/** The shared structured logger (pino), exported as `logger.logger`. */
 const logger = new StructuredLogger(baseLogger);
 
 /**
@@ -238,4 +258,14 @@ module.exports = {
   createLogger,
   correlationMiddleware,
   StructuredLogger,
+  // The package exports this module as `logger`, so `logger.info()` and the
+  // rest log through the shared instance as well as `logger.logger.info()`.
+  /** Logs at info level. Same as `logger.logger.info()`. @param {string} message @param {*} [meta] */
+  info: (message, meta) => logger.info(message, meta),
+  /** Logs at warn level. Same as `logger.logger.warn()`. @param {string} message @param {*} [meta] */
+  warn: (message, meta) => logger.warn(message, meta),
+  /** Logs at error level. Same as `logger.logger.error()`. @param {string} message @param {*} [meta] */
+  error: (message, meta) => logger.error(message, meta),
+  /** Logs at debug level. Same as `logger.logger.debug()`. @param {string} message @param {*} [meta] */
+  debug: (message, meta) => logger.debug(message, meta),
 };

package/src/utils/rate-limiter.js

@@ -11,14 +11,19 @@
 
 const crypto = require('crypto');
 
+/**
+ * Sliding-window limiter: allows up to `maxAttempts` requests per identifier
+ * in `windowMs`, then blocks that identifier for `blockDurationMs`. Check a
+ * request with allowRequest(identifier).
+ */
 class RateLimiter {
   /**
    * Create a rate limiter instance
    * @param {Object} options - Configuration options
-   * @param {number} options.maxAttempts - Maximum attempts allowed (default: 5)
-   * @param {number} options.windowMs - Time window in milliseconds (default: 15 minutes)
-   * @param {number} options.blockDurationMs - How long to block after exceeding limit (default: 1 hour)
-   * @param {number} options.cleanupIntervalMs - How often to cleanup expired entries (default: 5 minutes)
+   * @param {number} [options.maxAttempts=5] - Maximum attempts allowed (default: 5)
+   * @param {number} [options.windowMs=900000] - Time window in milliseconds (default: 15 minutes)
+   * @param {number} [options.blockDurationMs=3600000] - How long to block after exceeding limit (default: 1 hour)
+   * @param {number} [options.cleanupIntervalMs=300000] - How often to cleanup expired entries (default: 5 minutes)
    */
   constructor(options = {}) {
     this.maxAttempts = options.maxAttempts || 5;

package/src/utils/retry.js

@@ -75,7 +75,16 @@ async function retry(fn, opts = {}) {
   }
 }
 
-// Legacy wrapper for backward compatibility
+/**
+ * Older wrapper around retry(): retries `fn` with exponential backoff from
+ * `initialDelay` ms and logs each retry and the final failure under `label`.
+ * @param {function(number): *} fn - The operation; called with the attempt index
+ * @param {Object} [options]
+ * @param {number} [options.retries=3] - Retries after the first attempt
+ * @param {number} [options.initialDelay=300] - First backoff delay in ms
+ * @param {string} [options.label='operation'] - Name used in log messages
+ * @returns {Promise<*>} The operation's result
+ */
 async function withRetry(
   _fn,
   { retries = 3, initialDelay = 300, label = 'operation' } = {},