@devilsdev/rag-pipeline-utils 2.1.11 → 2.2.0

package/src/config/load-plugin-config.js

@@ -0,0 +1,69 @@
+/**
+ * Version: 1.1.1
+ * Description: Dynamic plugin loader with direct _type:name registry mapping
+ * Author: Ali Kahwaji
+ */
+
+const fs = require("fs");
+// eslint-disable-line global-require
+const path = require("path");
+// eslint-disable-line global-require
+const { pathToFileURL } = require("url");
+// eslint-disable-line global-require
+const registry = require("../core/plugin-registry.js");
+// eslint-disable-line global-require
+const { validatePluginSchema } = require("./validate-schema.js");
+// eslint-disable-line global-require
+const { logger } = require("../utils/logger.js");
+// eslint-disable-line global-require
+
+/**
+ * Loads plugins from validated _config path and registers them.
+ * @param {string} configPath
+ */
+async function loadPluginsFromJson(configPath) {
+  if (!fs.existsSync(configPath)) {
+    throw new Error(`Plugin _config missing: ${configPath}`);
+  }
+
+  const raw = fs.readFileSync(configPath, "utf-8");
+  const _config = JSON.parse(raw);
+
+  const validation = validatePluginSchema(_config);
+  if (!validation.valid) {
+    logger.error({ errors: validation.errors }, "Invalid plugin _config");
+    throw new Error("Plugin _config validation failed");
+  }
+
+  for (const [_type, namedPlugins] of Object.entries(_config)) {
+    if (!["loader", "embedder", "retriever", "llm"].includes(_type)) continue;
+
+    for (const [name, relPath] of Object.entries(namedPlugins)) {
+      const absPath = path.resolve(path.dirname(configPath), relPath);
+      const fileUrl = pathToFileURL(absPath).href;
+
+      try {
+        const mod = await import(fileUrl);
+        const PluginClass = mod?.default;
+        if (typeof PluginClass !== "function") {
+          throw new Error(
+            `Invalid export in plugin [${_type}:${name}] at ${absPath}`,
+          );
+        }
+
+        const _instance = new PluginClass();
+        registry.register(_type, name, _instance);
+
+        logger.info({ _type, name }, `Registered plugin: ${_type}:${name}`);
+      } catch (err) {
+        logger.error(
+          { _type, name, err },
+          `Failed to register plugin [${_type}:${name}]`,
+        );
+        throw err;
+      }
+    }
+  }
+}
+
+module.exports = { loadPluginsFromJson };

package/src/config/normalize-config.js

@@ -0,0 +1,105 @@
+/**
+ * Configuration Normalization - Unifies config shape and validation
+ * Handles pipeline object/array conversion and default namespace assignment
+ */
+
+function normalizeConfig(cfg) {
+  if (!cfg || typeof cfg !== "object" || Array.isArray(cfg)) {
+    throw new TypeError("Configuration must be a non-null object");
+  }
+
+  // Track validation warnings for invalid entries that get filtered out
+  const warnings = [];
+
+  // Default namespace
+  const namespace = cfg.namespace || "default";
+
+  // Canonical pipeline: array of { stage, name, ...opts }
+  // Accept legacy object form like:
+  // { pipeline: { loader:{name:'x'}, embedder:{name:'y'} } }
+  // { plugins: { loader:'file-loader', embedder:'openai-embedder' } }
+  // and modern array form:
+  // { pipeline: [{stage:'loader', name:'x'}] }
+  const legacy =
+    (cfg.pipeline &&
+      !Array.isArray(cfg.pipeline) &&
+      typeof cfg.pipeline === "object") ||
+    (cfg.plugins && typeof cfg.plugins === "object");
+  const modern = Array.isArray(cfg.pipeline);
+
+  let pipeline = [];
+  if (modern) {
+    pipeline = cfg.pipeline
+      .filter(Boolean)
+      .map((p) => {
+        const entry = { stage: p.stage, name: p.name };
+        if (p.version) entry.version = p.version;
+        if (p.config) entry.config = p.config;
+        if (p.options) Object.assign(entry, p.options);
+        return entry;
+      })
+      .filter((p) => typeof p.stage === "string" && typeof p.name === "string");
+  } else if (legacy) {
+    const order = ["loader", "embedder", "retriever", "reranker", "llm"];
+    const source = cfg.pipeline || cfg.plugins;
+
+    for (const stage of order) {
+      const entry = source[stage];
+      if (entry) {
+        if (typeof entry === "string") {
+          // Simple format: { plugins: { loader: 'file-loader' } }
+          pipeline.push({ stage, name: entry });
+        } else if (
+          entry &&
+          typeof entry === "object" &&
+          typeof entry.name === "string"
+        ) {
+          // Complex format: { pipeline: { loader: { name: 'file-loader', version: '1.0.0', config: {}, options: {} } } }
+          const pipelineEntry = { stage, name: entry.name };
+          if (entry.version) pipelineEntry.version = entry.version;
+          if (entry.config) pipelineEntry.config = entry.config;
+          if (entry.options) Object.assign(pipelineEntry, entry.options);
+          pipeline.push(pipelineEntry);
+        } else if (entry && typeof entry === "object") {
+          // Invalid entry - has object but invalid name
+          if (typeof entry.name !== "string") {
+            warnings.push(
+              `Invalid ${stage} plugin name: expected string, got ${typeof entry.name}`,
+            );
+          }
+        }
+      }
+    }
+  }
+
+  // Ensure at least handles loader/embedder for legacy tests
+  // (empty array is allowed but tests expect 2 when legacy provided)
+  const metadata =
+    cfg.metadata && typeof cfg.metadata === "object" ? cfg.metadata : {};
+
+  const out = {
+    namespace,
+    metadata,
+    pipeline,
+  };
+
+  // Preserve known top-level fields (non-breaking)
+  const known = [
+    "cache",
+    "limits",
+    "plugins",
+    "observability",
+    "storage",
+    "performance",
+  ];
+  for (const k of known) if (k in cfg) out[k] = cfg[k];
+
+  // Add warnings if any invalid entries were encountered
+  if (warnings.length > 0) {
+    out._warnings = warnings;
+  }
+
+  return out;
+}
+
+module.exports = { normalizeConfig };

package/src/config/schema-validator.js

@@ -0,0 +1,315 @@
+/**
+ * Configuration Schema Validator - Enterprise-grade config validation
+ * Provides JSON Schema validation for all configuration files
+ */
+
+const Ajv = require("ajv");
+const addFormats = require("ajv-formats");
+const { logger } = require("../utils/logger");
+
+class ConfigSchemaValidator {
+  constructor() {
+    this.ajv = new Ajv({
+      allErrors: true,
+      verbose: true,
+      strict: false,
+    });
+    addFormats(this.ajv);
+
+    // Register custom formats
+    this.ajv.addFormat("semver", /^\d+\.\d+\.\d+(-[\w.-]+)?(\+[\w.-]+)?$/);
+    this.ajv.addFormat("log-level", /^(trace|debug|info|warn|error|fatal)$/);
+
+    this.schemas = new Map();
+    this._loadSchemas();
+  }
+
+  _loadSchemas() {
+    // Core pipeline configuration schema
+    const pipelineSchema = {
+      type: "object",
+      properties: {
+        version: { type: "string", format: "semver" },
+        name: { type: "string", minLength: 1 },
+        description: { type: "string" },
+        pipeline: {
+          type: "object",
+          properties: {
+            steps: {
+              type: "array",
+              items: {
+                type: "object",
+                properties: {
+                  name: { type: "string", minLength: 1 },
+                  type: {
+                    type: "string",
+                    enum: [
+                      "ingest",
+                      "chunk",
+                      "embed",
+                      "store",
+                      "query",
+                      "evaluate",
+                    ],
+                  },
+                  config: { type: "object" },
+                  enabled: { type: "boolean", default: true },
+                },
+                required: ["name", "type"],
+                additionalProperties: false,
+              },
+              minItems: 1,
+            },
+            concurrency: {
+              type: "integer",
+              minimum: 1,
+              maximum: 100,
+              default: 5,
+            },
+            timeout: { type: "integer", minimum: 1000, default: 30000 },
+            retries: { type: "integer", minimum: 0, maximum: 10, default: 3 },
+          },
+          required: ["steps"],
+          additionalProperties: false,
+        },
+        logging: {
+          type: "object",
+          properties: {
+            level: { type: "string", format: "log-level", default: "info" },
+            structured: { type: "boolean", default: true },
+            correlationId: { type: "boolean", default: true },
+          },
+          additionalProperties: false,
+        },
+      },
+      required: ["version", "name", "pipeline"],
+      additionalProperties: false,
+    };
+
+    // AI/ML configuration schema
+    const aiConfigSchema = {
+      type: "object",
+      properties: {
+        embeddings: {
+          type: "object",
+          properties: {
+            provider: {
+              type: "string",
+              enum: ["openai", "azure", "huggingface", "local"],
+            },
+            model: { type: "string", minLength: 1 },
+            dimensions: { type: "integer", minimum: 1, maximum: 4096 },
+            batchSize: {
+              type: "integer",
+              minimum: 1,
+              maximum: 1000,
+              default: 100,
+            },
+            timeout: { type: "integer", minimum: 1000, default: 30000 },
+          },
+          required: ["provider", "model"],
+          additionalProperties: false,
+        },
+        training: {
+          type: "object",
+          properties: {
+            batchSize: {
+              type: "integer",
+              minimum: 1,
+              maximum: 1024,
+              default: 32,
+            },
+            learningRate: {
+              type: "number",
+              minimum: 0.0001,
+              maximum: 1,
+              default: 0.001,
+            },
+            epochs: { type: "integer", minimum: 1, maximum: 1000, default: 10 },
+            validationSplit: {
+              type: "number",
+              minimum: 0,
+              maximum: 0.5,
+              default: 0.2,
+            },
+          },
+          additionalProperties: false,
+        },
+        federation: {
+          type: "object",
+          properties: {
+            minParticipants: {
+              type: "integer",
+              minimum: 2,
+              maximum: 1000,
+              default: 2,
+            },
+            maxParticipants: {
+              type: "integer",
+              minimum: 2,
+              maximum: 10000,
+              default: 100,
+            },
+            roundDuration: { type: "integer", minimum: 60000, default: 300000 },
+            convergenceThreshold: {
+              type: "number",
+              minimum: 0.0001,
+              maximum: 1,
+              default: 0.001,
+            },
+          },
+          additionalProperties: false,
+        },
+      },
+      additionalProperties: false,
+    };
+
+    // Security configuration schema
+    const securitySchema = {
+      type: "object",
+      properties: {
+        authentication: {
+          type: "object",
+          properties: {
+            enabled: { type: "boolean", default: true },
+            provider: {
+              type: "string",
+              enum: ["jwt", "oauth2", "api-key", "none"],
+            },
+            tokenExpiry: { type: "integer", minimum: 300, default: 3600 },
+          },
+          required: ["enabled", "provider"],
+          additionalProperties: false,
+        },
+        encryption: {
+          type: "object",
+          properties: {
+            algorithm: {
+              type: "string",
+              enum: ["aes-256-gcm", "aes-256-cbc", "chacha20-poly1305"],
+            },
+            keyRotation: { type: "boolean", default: true },
+            keyRotationInterval: {
+              type: "integer",
+              minimum: 86400,
+              default: 2592000,
+            },
+          },
+          additionalProperties: false,
+        },
+        audit: {
+          type: "object",
+          properties: {
+            enabled: { type: "boolean", default: true },
+            retention: { type: "integer", minimum: 86400, default: 7776000 },
+            sensitiveFields: {
+              type: "array",
+              items: { type: "string" },
+              default: ["password", "token", "key", "secret"],
+            },
+          },
+          additionalProperties: false,
+        },
+      },
+      additionalProperties: false,
+    };
+
+    this.schemas.set("pipeline", pipelineSchema);
+    this.schemas.set("ai", aiConfigSchema);
+    this.schemas.set("security", securitySchema);
+  }
+
+  validate(config, schemaName) {
+    const schema = this.schemas.get(schemaName);
+    if (!schema) {
+      throw new Error(`Unknown schema: ${schemaName}`);
+    }
+
+    const validate = this.ajv.compile(schema);
+    const valid = validate(config);
+
+    if (!valid) {
+      const errors = validate.errors.map((error) => ({
+        path: error.instancePath || error.schemaPath,
+        message: error.message,
+        value: error.data,
+        allowedValues: error.schema?.enum,
+      }));
+
+      logger.error("Configuration validation failed", {
+        schema: schemaName,
+        errors,
+        config: this._sanitizeConfig(config),
+      });
+
+      return {
+        valid: false,
+        errors,
+      };
+    }
+
+    logger.info("Configuration validation passed", {
+      schema: schemaName,
+      configKeys: Object.keys(config),
+    });
+
+    return {
+      valid: true,
+      errors: [],
+    };
+  }
+
+  validatePipeline(config) {
+    return this.validate(config, "pipeline");
+  }
+
+  validateAI(config) {
+    return this.validate(config, "ai");
+  }
+
+  validateSecurity(config) {
+    return this.validate(config, "security");
+  }
+
+  addSchema(name, schema) {
+    this.schemas.set(name, schema);
+    logger.info("Custom schema registered", { schemaName: name });
+  }
+
+  getSchema(name) {
+    return this.schemas.get(name);
+  }
+
+  listSchemas() {
+    return Array.from(this.schemas.keys());
+  }
+
+  _sanitizeConfig(config) {
+    // Remove sensitive fields from logs
+    const sensitiveFields = ["password", "token", "key", "secret", "apiKey"];
+    const sanitized = JSON.parse(JSON.stringify(config));
+
+    const sanitizeObject = (obj) => {
+      for (const [key, value] of Object.entries(obj)) {
+        if (
+          sensitiveFields.some((field) => key.toLowerCase().includes(field))
+        ) {
+          obj[key] = "[REDACTED]";
+        } else if (typeof value === "object" && value !== null) {
+          sanitizeObject(value);
+        }
+      }
+    };
+
+    sanitizeObject(sanitized);
+    return sanitized;
+  }
+}
+
+// Create singleton instance
+const validator = new ConfigSchemaValidator();
+
+module.exports = {
+  ConfigSchemaValidator,
+  validator,
+};

package/src/config/validate-plugin-schema.js

@@ -0,0 +1,35 @@
+/**
+ * Version: 0.1.0
+ * Description: Schema validator for plugin._config.json
+ * Author: Ali Kahwaji
+ */
+
+function validatePluginSchema(config) {
+  const validTypes = ["loader", "embedder", "retriever", "llm"];
+  const errors = [];
+
+  for (const _type of Object.keys(config)) {
+    if (!validTypes.includes(_type)) {
+      errors.push(`Unknown plugin _type: ${_type}`);
+      continue;
+    }
+    for (const [name, modulePath] of Object.entries(config[_type])) {
+      if (typeof name !== "string" || typeof modulePath !== "string") {
+        errors.push(
+          `Invalid plugin definition for _type "${_type}" and name "${name}"`,
+        );
+      }
+    }
+  }
+
+  return {
+    valid: errors.length === 0,
+    errors,
+  };
+}
+
+// Default export
+
+module.exports = {
+  validatePluginSchema,
+};

package/src/config/validate-schema.js

@@ -0,0 +1,113 @@
+/**
+ * Version: 1.3.0
+ * Description: AJV-powered validators for .ragrc.json full _config and plugin-only structure
+ * Author: Ali Kahwaji
+ * File: /src/_config/validate-schema.js
+ */
+
+const Ajv = require("ajv");
+// eslint-disable-line global-require
+
+/**
+ * Full .ragrc.json schema (used in load-_config.js)
+ */
+const ragrcSchema = {
+  type: "object",
+  required: ["loader", "embedder", "retriever", "llm", "namespace", "pipeline"],
+  properties: {
+    loader: {
+      type: "object",
+      minProperties: 1,
+      additionalProperties: { type: "string" },
+    },
+    embedder: {
+      type: "object",
+      minProperties: 1,
+      additionalProperties: { type: "string" },
+    },
+    retriever: {
+      type: "object",
+      minProperties: 1,
+      additionalProperties: { type: "string" },
+    },
+    llm: {
+      type: "object",
+      minProperties: 1,
+      additionalProperties: { type: "string" },
+    },
+    namespace: {
+      type: "string",
+      minLength: 1,
+    },
+    pipeline: {
+      type: "array",
+      items: {
+        type: "string",
+        enum: ["loader", "embedder", "retriever"],
+      },
+      minItems: 1,
+      uniqueItems: true,
+    },
+  },
+  additionalProperties: false,
+};
+
+/**
+ * Minimal plugin-only schema (used in load-plugin-_config.js)
+ */
+const pluginSchema = {
+  type: "object",
+  required: ["loader", "embedder", "retriever", "llm"],
+  properties: {
+    loader: {
+      type: "object",
+      minProperties: 1,
+      additionalProperties: { type: "string" },
+    },
+    embedder: {
+      type: "object",
+      minProperties: 1,
+      additionalProperties: { type: "string" },
+    },
+    retriever: {
+      type: "object",
+      minProperties: 1,
+      additionalProperties: { type: "string" },
+    },
+    llm: {
+      type: "object",
+      minProperties: 1,
+      additionalProperties: { type: "string" },
+    },
+  },
+  additionalProperties: true, // allow namespace, pipeline, etc.
+};
+
+/**
+ * Validates the full .ragrc.json _config structure
+ * @param {object} config
+ * @returns {{ valid: boolean, errors?: any[] }}
+ */
+function validateRagrcSchema(config) {
+  const ajv = new Ajv({ allErrors: true, strict: false });
+  const validate = ajv.compile(ragrcSchema);
+  const valid = validate(config);
+  return valid ? { valid: true } : { valid: false, errors: validate.errors };
+}
+
+/**
+ * Validates a plugin-only structure (for runtime plugin loading)
+ * @param {object} config
+ * @returns {{ valid: boolean, errors?: any[] }}
+ */
+function validatePluginSchema(config) {
+  const ajv = new Ajv({ allErrors: true, strict: false });
+  const validate = ajv.compile(pluginSchema);
+  const valid = validate(config);
+  return valid ? { valid: true } : { valid: false, errors: validate.errors };
+}
+
+module.exports = {
+  validateRagrcSchema,
+  validatePluginSchema,
+};

package/src/contracts/plugin-contracts.js

@@ -0,0 +1,33 @@
+/**
+ * Plugin Contract Definitions
+ * Minimal fixture for test compatibility
+ */
+
+const pluginContracts = {
+  loader: {
+    name: 'loader',
+    version: '1.0.0',
+    methods: ['load'],
+    description: 'Document loader contract',
+  },
+  embedder: {
+    name: 'embedder',
+    version: '1.0.0',
+    methods: ['embed', 'embedQuery'],
+    description: 'Text embedder contract',
+  },
+  retriever: {
+    name: 'retriever',
+    version: '1.0.0',
+    methods: ['store', 'retrieve'],
+    description: 'Vector retriever contract',
+  },
+  llm: {
+    name: 'llm',
+    version: '1.0.0',
+    methods: ['generate'],
+    description: 'Language model contract',
+  },
+};
+
+module.exports = { pluginContracts };