@devilsdev/rag-pipeline-utils 2.4.5 → 2.6.0

package/src/config/component-factory.js

@@ -0,0 +1,125 @@
+"use strict";
+
+const path = require("path");
+const pluginRegistry = require("../core/plugin-registry.js");
+
+/**
+ * Built-in components a .ragrc.json pipeline entry can name. Each factory
+ * receives the entry's `config` object and the directory of the config file.
+ * Requires are lazy so naming one provider does not load the others.
+ */
+const BUILT_INS = {
+  // "auto" picks a loader per file extension (see loader/document-reader).
+  loader: {
+    auto: () => require("../loader/document-reader"),
+  },
+  embedder: {
+    local: (cfg) =>
+      new (require("../connectors/local-embedder").LocalEmbedder)(cfg),
+    openai: (cfg) =>
+      new (require("../connectors/openai-connector").OpenAIConnector)(cfg),
+    ollama: (cfg) =>
+      new (require("../connectors/ollama-connector").OllamaConnector)(cfg),
+    cohere: (cfg) =>
+      new (require("../connectors/cohere-connector").CohereConnector)(cfg),
+  },
+  retriever: {
+    "file-store": (cfg, configDir) =>
+      new (require("../retrieval/file-vector-store").FileVectorStore)({
+        path: path.resolve(configDir, cfg.path || ".rag/index.json"),
+      }),
+    memory: (cfg) =>
+      new (require("../connectors/memory-retriever").MemoryRetriever)(cfg),
+  },
+  llm: {
+    openai: (cfg) =>
+      new (require("../connectors/openai-connector").OpenAIConnector)(cfg),
+    anthropic: (cfg) =>
+      new (require("../connectors/anthropic-connector").AnthropicConnector)(
+        cfg,
+      ),
+    ollama: (cfg) =>
+      new (require("../connectors/ollama-connector").OllamaConnector)(cfg),
+  },
+  reranker: {
+    scoring: (cfg) =>
+      new (require("../reranker/scoring-reranker").ScoringReranker)(cfg),
+  },
+};
+
+const STAGES = ["loader", "embedder", "retriever", "reranker", "llm"];
+
+/**
+ * Instantiate a custom plugin from `plugins.<stage>.<name>` in the config.
+ * The module may export a class (constructed with the entry config), an
+ * instance, or either of those as `default`.
+ */
+function loadPluginModule(modulePath, entryConfig, configDir) {
+  const resolved = path.resolve(configDir, modulePath);
+  const mod = require(resolved);
+  const exported = mod && mod.default ? mod.default : mod;
+  if (typeof exported === "function") {
+    return /^class\s/.test(Function.prototype.toString.call(exported))
+      ? new exported(entryConfig)
+      : exported(entryConfig);
+  }
+  if (exported && typeof exported === "object") return exported;
+  throw new Error(`Plugin module ${modulePath} must export a class or object`);
+}
+
+/**
+ * Turn a normalized .ragrc.json config into component instances.
+ *
+ * Each `pipeline` entry `{ stage, name, config }` is resolved in order:
+ *   1. a built-in name (see BUILT_INS),
+ *   2. a custom module path under `plugins.<stage>.<name>`,
+ *   3. a plugin registered on the shared pluginRegistry.
+ *
+ * @param {object} config - Output of loadRagConfig()/validateRagrc().normalized
+ * @param {object} [options]
+ * @param {string} [options.configDir=process.cwd()] - Paths resolve relative to this
+ * @returns {{embedder?: object, retriever?: object, llm?: object, reranker?: object, loader?: object, names: object}}
+ */
+function buildComponents(config, options = {}) {
+  const configDir = options.configDir || process.cwd();
+  const components = { names: {} };
+
+  for (const entry of config.pipeline || []) {
+    if (!STAGES.includes(entry.stage)) {
+      throw new Error(
+        `Unknown pipeline stage "${entry.stage}". Expected one of: ${STAGES.join(", ")}`,
+      );
+    }
+    components[entry.stage] = resolveComponent(entry, config, configDir);
+    components.names[entry.stage] = entry.name;
+  }
+  return components;
+}
+
+function resolveComponent(entry, config, configDir) {
+  const { stage, name } = entry;
+  const entryConfig = entry.config || {};
+
+  const builtIn = BUILT_INS[stage]?.[name];
+  if (builtIn) return builtIn(entryConfig, configDir);
+
+  const modulePath = config.plugins?.[stage]?.[name];
+  if (typeof modulePath === "string") {
+    return loadPluginModule(modulePath, entryConfig, configDir);
+  }
+
+  try {
+    return pluginRegistry.get(stage, name);
+  } catch {
+    const builtIns = Object.keys(BUILT_INS[stage] || {});
+    throw new Error(
+      `Unknown ${stage} "${name}". ` +
+        (builtIns.length
+          ? `Built-in ${stage}s: ${builtIns.join(", ")}. `
+          : "") +
+        `For a custom plugin add "plugins": { "${stage}": { "${name}": "./path/to/module.js" } }.`,
+    );
+  }
+}
+
+module.exports = { buildComponents, loadPluginModule, BUILT_INS, STAGES };

package/src/config/enhanced-ragrc-schema.js

@@ -3,8 +3,6 @@
  * Single source of truth for configuration validation
  */
 
-const Ajv = require("ajv"); // eslint-disable-line global-require
-const addFormats = require("ajv-formats"); // eslint-disable-line global-require
 const { normalizeConfig } = require("./normalize-config");
 
 /**

package/src/config/load-config.js

@@ -18,11 +18,18 @@ const CONFIG_FILENAME = ".ragrc.json";
  * @param {string} [cwd=process.cwd()] - Directory to resolve the config from
  * @returns {object} Validated configuration object
  */
-function loadRagConfig(cwd = process.cwd()) {
-  const configPath = path.resolve(cwd, CONFIG_FILENAME);
+/**
+ * Load and validate .ragrc.json.
+ *
+ * @param {string} [location=process.cwd()] - A directory containing
+ *   .ragrc.json, or the path of a config file (any name ending in .json)
+ * @returns {object} Normalized configuration
+ */
+function loadRagConfig(location = process.cwd()) {
+  const configPath = resolveConfigPath(location);
 
+  // Errors are thrown, not logged: callers (e.g. the CLI) report them once.
   if (!fs.existsSync(configPath)) {
-    logger.error(`❌ Config file not found: ${configPath}`);
     throw new Error(`Config file not found: ${configPath}`);
   }
 
@@ -32,23 +39,35 @@ function loadRagConfig(cwd = process.cwd()) {
   try {
     config = JSON.parse(raw);
   } catch (err) {
-    logger.error("❌ Failed to parse JSON configuration.");
-    throw new Error("Invalid JSON in config file.");
+    throw new Error(
+      `Invalid JSON in config file ${configPath}: ${err.message}`,
+    );
   }
 
   const { valid, errors, normalized } = validateRagrc(config);
 
   if (!valid) {
-    logger.error(
-      `❌ Config validation failed:\n${JSON.stringify(errors, null, 2)}`,
-    );
-    throw new Error("Config validation failed");
+    const details = (errors || []).map((e) => e.message).join("; ");
+    throw new Error(`Config validation failed: ${details}`);
   }
 
-  logger.info("✅ Configuration loaded and normalized successfully");
+  logger.debug("Configuration loaded", { configPath });
   return normalized;
 }
 
+/**
+ * @param {string} [location=process.cwd()] - Directory or config file path
+ * @returns {string} Absolute path of the config file
+ */
+function resolveConfigPath(location = process.cwd()) {
+  const absolute = path.resolve(location);
+  const isFile =
+    (fs.existsSync(absolute) && fs.statSync(absolute).isFile()) ||
+    absolute.toLowerCase().endsWith(".json");
+  return isFile ? absolute : path.join(absolute, CONFIG_FILENAME);
+}
+
 module.exports = {
   loadRagConfig,
+  resolveConfigPath,
 };

package/src/config/load-plugin-config.js

@@ -17,6 +17,48 @@ const { validatePluginSchema } = require("./validate-schema.js");
 const { logger } = require("../utils/logger.js");
 // eslint-disable-line global-require
 
+/**
+ * Verify a plugin file against its signed manifest BEFORE it is imported
+ * (importing runs the plugin's code).
+ *
+ * The manifest is read from `<plugin file>.manifest.json`. Its
+ * `contentDigest` (if any) must cover the plugin file plus any extra files
+ * listed in `manifest.files` (paths relative to the plugin file's
+ * directory). Nothing is checked when the registry's signature verification
+ * is disabled; when it is enabled and the registry was created with
+ * `requireSignedPlugins: true`, a missing manifest is an error.
+ *
+ * @param {string} name - Plugin name
+ * @param {string} absPath - Absolute path of the plugin entry file
+ * @returns {Promise<void>}
+ */
+async function verifyPluginBeforeImport(name, absPath) {
+  if (
+    typeof registry.isSignatureVerificationEnabled !== "function" ||
+    !registry.isSignatureVerificationEnabled()
+  ) {
+    return;
+  }
+
+  const manifestPath = `${absPath}.manifest.json`;
+  if (!fs.existsSync(manifestPath)) {
+    if (registry.requiresSignedPlugins()) {
+      throw new Error(
+        `Plugin ${name} has no signed manifest (${manifestPath}) and requireSignedPlugins is enabled`,
+      );
+    }
+    return;
+  }
+
+  const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf-8"));
+  const baseDir = path.dirname(absPath);
+  const extraFiles = Array.isArray(manifest.files) ? manifest.files : [];
+  await registry.verifyPluginContent(name, manifest, {
+    baseDir,
+    files: [absPath, ...extraFiles.map((f) => path.resolve(baseDir, f))],
+  });
+}
+
 /**
  * Loads plugins from validated _config path and registers them.
  * @param {string} configPath
@@ -43,6 +85,7 @@ async function loadPluginsFromJson(configPath) {
       const fileUrl = pathToFileURL(absPath).href;
 
       try {
+        await verifyPluginBeforeImport(name, absPath);
         const mod = await import(fileUrl);
         const PluginClass = mod?.default;
         if (typeof PluginClass !== "function") {
@@ -52,7 +95,8 @@ async function loadPluginsFromJson(configPath) {
         }
 
         const _instance = new PluginClass();
-        registry.register(_type, name, _instance);
+        // register() is async; await it so failures are not unhandled
+        await registry.register(_type, name, _instance);
 
         logger.info({ _type, name }, `Registered plugin: ${_type}:${name}`);
       } catch (err) {
@@ -66,4 +110,4 @@ async function loadPluginsFromJson(configPath) {
   }
 }
 
-module.exports = { loadPluginsFromJson };
+module.exports = { loadPluginsFromJson, verifyPluginBeforeImport };

package/src/config/validate-schema.js

@@ -5,7 +5,8 @@
  * File: /src/_config/validate-schema.js
  */
 
-const Ajv = require("ajv");
+// ajv loads when a schema is validated, not on import.
+const loadAjv = () => require("ajv"); // eslint-disable-line global-require
 // eslint-disable-line global-require
 
 /**
@@ -89,6 +90,7 @@ const pluginSchema = {
  * @returns {{ valid: boolean, errors?: any[] }}
  */
 function validateRagrcSchema(config) {
+  const Ajv = loadAjv();
   const ajv = new Ajv({ allErrors: true, strict: false });
   const validate = ajv.compile(ragrcSchema);
   const valid = validate(config);
@@ -101,6 +103,7 @@ function validateRagrcSchema(config) {
  * @returns {{ valid: boolean, errors?: any[] }}
  */
 function validatePluginSchema(config) {
+  const Ajv = loadAjv();
   const ajv = new Ajv({ allErrors: true, strict: false });
   const validate = ajv.compile(pluginSchema);
   const valid = validate(config);

package/src/connectors/anthropic-connector.js

@@ -1,19 +1,38 @@
 'use strict';
 
-const axios = require('axios');
+// axios loads on first use, so importing the package does not pay for it
+// (about 125 ms) unless this connector is constructed.
+const loadAxios = () => require('axios'); // eslint-disable-line global-require
 const { BaseConnector } = require('./base-connector');
+const {
+  missingApiKeyError,
+  raceAbort,
+  resolveApiKey,
+  signalOptions,
+  throwIfAborted,
+} = require('./connector-utils');
+const { buildRagPrompt } = require('./rag-prompt');
+const { componentCacheKey } = require('../cache/component-keys');
+
+const API_KEY_ENV = 'ANTHROPIC_API_KEY';
 
 /**
  * Default configuration for the Anthropic connector.
+ * `apiKey` is a getter so the environment is read when a connector is
+ * constructed, not when this module is first required.
  * @type {Object}
  */
 const DEFAULT_CONFIG = {
   model: 'claude-3-5-sonnet-20241022',
-  apiKey: process.env.ANTHROPIC_API_KEY || '',
   baseURL: 'https://api.anthropic.com',
   apiVersion: '2023-06-01',
   maxTokens: 4096,
   temperature: 0.7,
+  maxRetries: 3,
+  retryDelay: 1000,
+  get apiKey() {
+    return process.env[API_KEY_ENV] || '';
+  },
 };
 
 /**
@@ -23,19 +42,43 @@ const DEFAULT_CONFIG = {
  * @extends BaseConnector
  */
 class AnthropicConnector extends BaseConnector {
+  /**
+   * Cache identity for createRagPipeline({ cache }).
+   * @param {'llm'} role
+   * @returns {string|null}
+   */
+  getCacheKey(role) {
+    if (role !== 'llm') return null;
+    const c = this.config;
+    return componentCacheKey('anthropic', {
+      baseURL: c.baseURL,
+      model: c.model,
+      temperature: c.temperature,
+      maxTokens: c.maxTokens,
+      systemPrompt: c.systemPrompt,
+    });
+  }
+
   /**
    * @param {Object} [options]
    * @param {string} [options.model='claude-3-5-sonnet-20241022'] - Claude model to use
-   * @param {string} [options.apiKey] - Anthropic API key
+   * @param {string} [options.apiKey] - Anthropic API key (defaults to ANTHROPIC_API_KEY at construction)
    * @param {string} [options.baseURL='https://api.anthropic.com'] - API base URL
    * @param {string} [options.apiVersion='2023-06-01'] - API version header
    * @param {number} [options.maxTokens=4096] - Max tokens in response
    * @param {number} [options.temperature=0.7] - Sampling temperature
+   * @param {number} [options.maxRetries=3] - Retries for transient failures
+   * @param {number} [options.retryDelay=1000] - Base backoff delay in ms
+   * @param {string} [options.systemPrompt] - System prompt used when RAG context is supplied
    */
   constructor(options = {}) {
     super({ name: 'anthropic', ...options });
-    this.config = { ...DEFAULT_CONFIG, ...options };
-    this.client = axios.create({
+    this.config = {
+      ...DEFAULT_CONFIG,
+      ...options,
+      apiKey: resolveApiKey(options, API_KEY_ENV),
+    };
+    this.client = loadAxios().create({
       baseURL: this.config.baseURL,
       headers: {
         'x-api-key': this.config.apiKey,
@@ -45,51 +88,73 @@ class AnthropicConnector extends BaseConnector {
     });
   }
 
+  /**
+   * @throws {Error} code MISSING_API_KEY when no key is configured
+   * @private
+   */
+  _requireApiKey() {
+    if (!this.config.apiKey) {
+      throw missingApiKeyError('Anthropic', API_KEY_ENV);
+    }
+  }
+
   /**
    * Connect to Anthropic. Verifies the API key is present.
    * @returns {Promise<void>}
    * @throws {Error} If no API key is configured
    */
   async connect() {
-    if (!this.config.apiKey) {
-      throw new Error(
-        'Anthropic API key required. Set ANTHROPIC_API_KEY or pass apiKey option.',
-      );
-    }
+    this._requireApiKey();
     await super.connect();
   }
 
+  /**
+   * Build the Messages API request body. Retrieved context goes into the
+   * user message as delimited, untrusted reference data; the system prompt
+   * only carries instructions.
+   * @private
+   */
+  _buildBody(query, context, options) {
+    const prompt = buildRagPrompt(query, context, {
+      systemPrompt: this.config.systemPrompt,
+    });
+    const body = {
+      model: options.model || this.config.model,
+      max_tokens: options.maxTokens ?? this.config.maxTokens,
+      temperature: options.temperature ?? this.config.temperature,
+      messages: [{ role: 'user', content: prompt.user }],
+    };
+    if (prompt.system) body.system = prompt.system;
+    return body;
+  }
+
   /**
    * Generate a response from Claude using the provided query and optional RAG context.
    *
    * @param {string} query - The user query / prompt
    * @param {Array<Object|string>} [context=[]] - Retrieved documents for RAG
    * @param {Object} [options={}] - Override model, temperature, maxTokens per-call
+   * @param {AbortSignal} [options.signal] - Cancels the request
    * @returns {Promise<string>} The generated text
    */
   async generate(query, context = [], options = {}) {
-    return this.withRetry(async () => {
-      const body = {
-        model: options.model || this.config.model,
-        max_tokens: options.maxTokens ?? this.config.maxTokens,
-        temperature: options.temperature ?? this.config.temperature,
-        messages: [
-          { role: 'user', content: this._buildPrompt(query, context) },
-        ],
-      };
-
-      // Add system message when context is provided
-      if (context && context.length > 0) {
-        const contextText = context
-          .map((doc) => doc.content || doc.text || String(doc))
-          .join('\n\n');
-        body.system = `Use the following context to answer the question:\n\n${contextText}`;
-        body.messages = [{ role: 'user', content: query }];
-      }
-
-      const response = await this.client.post('/v1/messages', body);
-      return response.data.content[0].text;
-    });
+    const { signal } = options;
+    throwIfAborted(signal);
+    this._requireApiKey();
+    const body = this._buildBody(query, context, options);
+    return this.withRetry(
+      async () => {
+        const response = await this.client.post(
+          '/v1/messages',
+          body,
+          signal ? { signal } : undefined,
+        );
+        return response.data.content[0].text;
+      },
+      this.config.maxRetries,
+      this.config.retryDelay,
+      { signal },
+    );
   }
 
   /**
@@ -98,32 +163,27 @@ class AnthropicConnector extends BaseConnector {
    * @param {string} query - The user query / prompt
    * @param {Array<Object|string>} [context=[]] - Retrieved documents for RAG
    * @param {Object} [options={}] - Override model, temperature, maxTokens per-call
+   * @param {AbortSignal} [options.signal] - Cancels the stream
    * @yields {{ token: string, done: boolean }}
    */
   async *generateStream(query, context = [], options = {}) {
-    const body = {
-      model: options.model || this.config.model,
-      max_tokens: options.maxTokens ?? this.config.maxTokens,
-      temperature: options.temperature ?? this.config.temperature,
-      messages: [{ role: 'user', content: this._buildPrompt(query, context) }],
-      stream: true,
-    };
-
-    if (context && context.length > 0) {
-      const contextText = context
-        .map((doc) => doc.content || doc.text || String(doc))
-        .join('\n\n');
-      body.system = `Use the following context to answer the question:\n\n${contextText}`;
-      body.messages = [{ role: 'user', content: query }];
-    }
-
-    const response = await this.client.post('/v1/messages', body, {
-      responseType: 'stream',
-    });
+    const { signal } = options;
+    throwIfAborted(signal);
+    this._requireApiKey();
+    const body = { ...this._buildBody(query, context, options), stream: true };
+
+    const response = await raceAbort(
+      this.client.post('/v1/messages', body, {
+        responseType: 'stream',
+        ...signalOptions(signal),
+      }),
+      signal,
+    );
 
     let buffer = '';
 
     for await (const chunk of response.data) {
+      throwIfAborted(signal);
       buffer += chunk.toString();
       const lines = buffer.split('\n');
       // Keep the last incomplete line in the buffer
@@ -160,7 +220,9 @@ class AnthropicConnector extends BaseConnector {
   }
 
   /**
-   * Build a prompt string that includes RAG context when available.
+   * Build a single prompt string that includes RAG context when available.
+   * Kept for backward compatibility; the context is delimited and labelled
+   * as untrusted reference data.
    *
    * @param {string} query - User query
    * @param {Array<Object|string>} context - RAG context documents
@@ -168,11 +230,7 @@ class AnthropicConnector extends BaseConnector {
    * @private
    */
   _buildPrompt(query, context) {
-    if (!context || context.length === 0) return query;
-    const contextText = context
-      .map((doc) => doc.content || doc.text || String(doc))
-      .join('\n\n');
-    return `Context:\n${contextText}\n\nQuestion: ${query}`;
+    return buildRagPrompt(query, context).user;
   }
 }
 

package/src/connectors/base-connector.js

@@ -1,6 +1,16 @@
 'use strict';
 
 const { EventEmitter } = require('events');
+const {
+  abortableSleep,
+  computeRetryDelay,
+  createAbortError,
+  getErrorStatus,
+  isAbortError,
+  isRetryableError,
+  raceAbort,
+  throwIfAborted,
+} = require('./connector-utils');
 
 /**
  * Abstract base class for all connectors.
@@ -49,33 +59,78 @@ class BaseConnector extends EventEmitter {
   }
 
   /**
-   * Retry wrapper that attempts a function up to N times with a delay between retries.
-   * Emits a 'retry' event on each retry attempt.
-   * @param {Function} fn - Async function to execute
+   * Retry wrapper for transient failures.
+   *
+   * Only retryable errors are retried: network errors and HTTP
+   * 408/409/425/429/5xx (see {@link isRetryableError}). Other errors
+   * (400, 401, 403, 404, aborts, errors with no status or network code) are
+   * thrown immediately. Backoff is exponential with full jitter, starting at
+   * `delay` and capped at `opts.maxDelay`; a Retry-After header on the error
+   * is honoured (capped at `opts.maxDelay`). Emits a 'retry' event before
+   * each retry.
+   *
+   * @param {Function} fn - Async function to execute; receives the attempt number
    * @param {number} [retries=3] - Maximum number of retry attempts
-   * @param {number} [delay=1000] - Delay in ms between retries
+   * @param {number} [delay=1000] - Base backoff delay in ms
+   * @param {object} [opts]
+   * @param {AbortSignal} [opts.signal] - Aborts the call and any pending backoff
+   * @param {number} [opts.maxDelay=30000] - Upper bound for a single backoff delay
+   * @param {(err: *) => boolean} [opts.shouldRetry] - Override retry classification
    * @returns {Promise<*>} Result of the function
-   * @throws {Error} Last error if all retries are exhausted
+   * @throws {Error} The first non-retryable error, or the last error once retries are exhausted
    */
-  async withRetry(fn, retries = 3, delay = 1000) {
-    let lastError;
-    for (let attempt = 0; attempt <= retries; attempt++) {
+  async withRetry(fn, retries = 3, delay = 1000, opts = {}) {
+    const signal = opts.signal;
+    const shouldRetry = opts.shouldRetry || isRetryableError;
+    const maxDelay = opts.maxDelay ?? this.options.retryMaxDelay ?? 30000;
+
+    for (let attempt = 0; ; attempt++) {
+      throwIfAborted(signal);
       try {
-        return await fn();
+        return await raceAbort(() => fn(attempt), signal);
       } catch (err) {
-        lastError = err;
-        if (attempt < retries) {
-          this.emit('retry', {
-            name: this.name,
-            attempt: attempt + 1,
-            maxRetries: retries,
-            error: err.message,
-          });
-          await new Promise((resolve) => setTimeout(resolve, delay));
+        if (isAbortError(err) || (signal && signal.aborted)) {
+          throw createAbortError(signal);
+        }
+        if (attempt >= retries || !shouldRetry(err)) {
+          throw err;
         }
+        const delayMs = computeRetryDelay(err, attempt, {
+          baseDelay: delay,
+          maxDelay,
+          random: () => this._random(),
+        });
+        this.emit('retry', {
+          name: this.name,
+          attempt: attempt + 1,
+          maxRetries: retries,
+          error: err.message,
+          status: getErrorStatus(err),
+          delayMs,
+        });
+        await this._sleep(delayMs, signal);
       }
     }
-    throw lastError;
+  }
+
+  /**
+   * Wait between retries. Overridable in tests.
+   * @param {number} ms
+   * @param {AbortSignal} [signal]
+   * @returns {Promise<void>}
+   * @protected
+   */
+  _sleep(ms, signal) {
+    return abortableSleep(ms, signal);
+  }
+
+  /**
+   * Jitter source for retry backoff. Overridable in tests.
+   * @returns {number} Value in [0, 1)
+   * @protected
+   */
+  _random() {
+    return Math.random();
   }
 
   /**