@caoscompanybr/merlin 3.5.0

package/.merlin-core/core/config/config-loader.js

@@ -0,0 +1,279 @@
+/**
+ * Merlin Config Loader with Lazy Loading
+ *
+ * Intelligent configuration loader that only loads what each agent needs,
+ * significantly reducing memory footprint and load times.
+ *
+ * @module core/config/config-loader
+ * @version 3.0.0
+ */
+
+const fs = require("fs").promises;
+const path = require("path");
+const yaml = require("js-yaml");
+
+/**
+ * Config cache with TTL.
+ * H4 note: This is the primary cache for the loader's lazy-loading path.
+ * The config-resolver uses its own globalConfigCache for full resolution.
+ */
+const configCache = {
+  full: null,
+  lastLoad: null,
+  TTL: 5 * 60 * 1000, // 5 minutes
+};
+
+/**
+ * Agent requirements mapping -- which config sections each agent needs
+ */
+const agentRequirements = {
+  "merlin-master": [
+    "project",
+    "alkimia",
+    "approval",
+    "skills",
+    "mcp",
+    "git",
+    "decisionLogging",
+    "projectStatus",
+    "agentIdentity",
+    "ideSync",
+  ],
+  dev: [
+    "project",
+    "alkimia",
+    "skills",
+    "git",
+    "devLoadAlwaysFiles",
+    "lazyLoading",
+  ],
+  architect: [
+    "project",
+    "alkimia",
+    "skills",
+    "git",
+    "devLoadAlwaysFiles",
+    "lazyLoading",
+  ],
+  analyst: [
+    "project",
+    "alkimia",
+    "skills",
+    "devLoadAlwaysFiles",
+    "lazyLoading",
+  ],
+  qa: [
+    "project",
+    "alkimia",
+    "skills",
+    "git",
+    "devLoadAlwaysFiles",
+    "lazyLoading",
+  ],
+  devops: [
+    "project",
+    "alkimia",
+    "skills",
+    "git",
+    "mcp",
+    "devLoadAlwaysFiles",
+    "lazyLoading",
+  ],
+  pm: ["project", "alkimia", "devLoadAlwaysFiles", "lazyLoading"],
+  po: ["project", "alkimia", "devLoadAlwaysFiles", "lazyLoading"],
+  sm: ["project", "alkimia", "devLoadAlwaysFiles", "lazyLoading"],
+  ux: ["project", "alkimia", "devLoadAlwaysFiles", "lazyLoading"],
+  "data-engineer": [
+    "project",
+    "alkimia",
+    "skills",
+    "devLoadAlwaysFiles",
+    "lazyLoading",
+  ],
+  researcher: ["project", "alkimia", "devLoadAlwaysFiles", "lazyLoading"],
+  scout: ["project", "devLoadAlwaysFiles", "lazyLoading"],
+  "web-researcher": ["project", "devLoadAlwaysFiles", "lazyLoading"],
+};
+
+/**
+ * Always-loaded sections (lightweight, needed by all)
+ */
+const ALWAYS_LOADED = ["project", "devLoadAlwaysFiles", "lazyLoading"];
+
+/**
+ * Performance tracking
+ */
+const performanceMetrics = {
+  loads: 0,
+  cacheHits: 0,
+  cacheMisses: 0,
+  avgLoadTime: 0,
+  totalLoadTime: 0,
+};
+
+/**
+ * Checks if cache is still valid
+ */
+function isCacheValid() {
+  if (!configCache.lastLoad) return false;
+  return Date.now() - configCache.lastLoad < configCache.TTL;
+}
+
+/**
+ * Loads full config file (used for initial load or cache refresh)
+ */
+async function loadFullConfig() {
+  const configPath = path.join(".merlin-core", "core-config.yaml");
+  const startTime = Date.now();
+
+  try {
+    const content = await fs.readFile(configPath, "utf8");
+    const config = yaml.load(content);
+    const loadTime = Date.now() - startTime;
+
+    performanceMetrics.loads++;
+    performanceMetrics.totalLoadTime += loadTime;
+    performanceMetrics.avgLoadTime =
+      performanceMetrics.totalLoadTime / performanceMetrics.loads;
+
+    configCache.full = config;
+    configCache.lastLoad = Date.now();
+
+    return config;
+  } catch (error) {
+    throw new Error(`Config load failed: ${error.message}`);
+  }
+}
+
+/**
+ * Loads specific config sections on demand
+ *
+ * @param {string[]} sections - Array of section names to load
+ * @returns {Promise<Object>} Config object with requested sections
+ */
+async function loadConfigSections(sections) {
+  if (isCacheValid() && configCache.full) {
+    performanceMetrics.cacheHits++;
+    const config = {};
+    sections.forEach((section) => {
+      if (configCache.full[section] !== undefined) {
+        config[section] = configCache.full[section];
+      }
+    });
+    return config;
+  }
+
+  performanceMetrics.cacheMisses++;
+  const fullConfig = await loadFullConfig();
+
+  const config = {};
+  sections.forEach((section) => {
+    if (fullConfig[section] !== undefined) {
+      config[section] = fullConfig[section];
+    }
+  });
+
+  return config;
+}
+
+/**
+ * Loads config for specific agent with lazy loading
+ *
+ * @param {string} agentId - Agent ID (e.g., 'dev', 'qa', 'merlin-master')
+ * @returns {Promise<Object>} Config object with sections needed by agent
+ */
+async function loadAgentConfig(agentId) {
+  const requiredSections = agentRequirements[agentId] || ALWAYS_LOADED;
+  return await loadConfigSections(requiredSections);
+}
+
+/**
+ * Loads always-loaded sections (minimal config)
+ *
+ * @returns {Promise<Object>} Minimal config with always-loaded sections
+ */
+async function loadMinimalConfig() {
+  return await loadConfigSections(ALWAYS_LOADED);
+}
+
+/**
+ * Preloads config into cache
+ */
+async function preloadConfig() {
+  await loadFullConfig();
+}
+
+/**
+ * Clears config cache
+ */
+function clearCache() {
+  configCache.full = null;
+  configCache.lastLoad = null;
+}
+
+/**
+ * Gets performance metrics
+ *
+ * @returns {Object} Performance statistics
+ */
+function getPerformanceMetrics() {
+  return {
+    ...performanceMetrics,
+    cacheHitRate:
+      performanceMetrics.loads > 0
+        ? (
+            (performanceMetrics.cacheHits / performanceMetrics.loads) *
+            100
+          ).toFixed(1) + "%"
+        : "0%",
+    avgLoadTimeMs: Math.round(performanceMetrics.avgLoadTime),
+  };
+}
+
+/**
+ * Validates that required sections exist in config
+ *
+ * @param {string} agentId - Agent ID to validate
+ * @returns {Promise<Object>} Validation result
+ */
+async function validateAgentConfig(agentId) {
+  const requiredSections = agentRequirements[agentId] || ALWAYS_LOADED;
+  const config = await loadFullConfig();
+
+  const missingSections = requiredSections.filter(
+    (section) => config[section] === undefined,
+  );
+
+  return {
+    valid: missingSections.length === 0,
+    agentId,
+    requiredSections,
+    missingSections,
+    availableSections: Object.keys(config),
+  };
+}
+
+/**
+ * Gets config section on demand
+ *
+ * @param {string} sectionName - Section to load
+ * @returns {Promise<any>} Section content
+ */
+async function getConfigSection(sectionName) {
+  const config = await loadConfigSections([sectionName]);
+  return config[sectionName];
+}
+
+module.exports = {
+  loadAgentConfig,
+  loadConfigSections,
+  loadMinimalConfig,
+  loadFullConfig,
+  preloadConfig,
+  clearCache,
+  getPerformanceMetrics,
+  validateAgentConfig,
+  getConfigSection,
+  agentRequirements,
+  ALWAYS_LOADED,
+};

package/.merlin-core/core/config/config-resolver.js

@@ -0,0 +1,411 @@
+/**
+ * Configuration Resolver -- Layered Config Hierarchy
+ *
+ * Implements Merlin's 4-level configuration hierarchy:
+ *   L0 Core -> L1 Framework -> L2 Project -> L3 Local
+ *
+ * Provides:
+ * - resolveConfig(projectRoot, options) -- main entry point
+ * - loadLayeredConfig(projectRoot, options) -- 4-level loading
+ * - getConfigAtLevel(projectRoot, level) -- raw config from a specific level
+ *
+ * Integrates with:
+ * - ConfigCache (config-cache.js) -- TTL-based caching
+ * - deepMerge (merge-utils.js) -- merge strategy
+ * - interpolateEnvVars (env-interpolator.js) -- ${VAR} resolution
+ *
+ * @module core/config/config-resolver
+ * @version 3.0.0
+ */
+
+const fs = require("fs");
+const path = require("path");
+const yaml = require("js-yaml");
+const { deepMerge } = require("./merge-utils");
+const { interpolateEnvVars, lintEnvPatterns } = require("./env-interpolator");
+const { globalConfigCache } = require("./config-cache");
+
+// ---------------------------------------------------------------------------
+// JSON Schema validation
+// ---------------------------------------------------------------------------
+
+let _ajvInstance = null;
+let _schemaCache = {};
+
+/**
+ * Schema file mapping for each config level.
+ */
+const SCHEMA_FILES = {
+  core: "core-config.schema.json",
+  framework: "framework-config.schema.json",
+  project: "project-config.schema.json",
+  local: "local-config.schema.json",
+};
+
+/**
+ * Get or create the shared Ajv instance (lazy-loaded).
+ *
+ * @returns {Object} Ajv instance
+ */
+function getAjvInstance() {
+  if (!_ajvInstance) {
+    try {
+      const Ajv = require("ajv");
+      const addFormats = require("ajv-formats");
+      _ajvInstance = new Ajv({ allErrors: true, strict: false });
+      addFormats(_ajvInstance);
+    } catch {
+      // ajv/ajv-formats not installed -- skip validation
+      return null;
+    }
+  }
+  return _ajvInstance;
+}
+
+/**
+ * Load a JSON Schema from the schemas/ directory.
+ *
+ * @param {string} schemaFileName - Schema file name
+ * @returns {Object|null} Parsed schema or null if not found
+ */
+function loadSchema(schemaFileName) {
+  if (_schemaCache[schemaFileName]) {
+    return _schemaCache[schemaFileName];
+  }
+
+  const schemaPath = path.join(__dirname, "schemas", schemaFileName);
+
+  try {
+    if (!fs.existsSync(schemaPath)) {
+      return null;
+    }
+    const content = fs.readFileSync(schemaPath, "utf8");
+    const schema = JSON.parse(content);
+    _schemaCache[schemaFileName] = schema;
+    return schema;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Validate config data against a JSON Schema for the given level.
+ * Returns warnings (does not throw) for graceful degradation.
+ *
+ * @param {string} level - Config level: 'core' | 'framework' | 'project' | 'local'
+ * @param {Object} data - Config data to validate
+ * @param {string} filePath - Source file path (for error messages)
+ * @returns {string[]} Validation warnings (empty if valid)
+ */
+function validateConfig(level, data, filePath) {
+  const warnings = [];
+  const schemaFile = SCHEMA_FILES[level];
+
+  if (!schemaFile) return warnings;
+
+  const schema = loadSchema(schemaFile);
+  if (!schema) return warnings;
+
+  try {
+    const ajv = getAjvInstance();
+    if (!ajv) return warnings;
+
+    const validate = ajv.compile(schema);
+    const isValid = validate(data);
+
+    if (!isValid && validate.errors) {
+      for (const err of validate.errors) {
+        const field = err.instancePath
+          ? err.instancePath.replace(/^\//, "")
+          : err.params?.missingProperty || "unknown";
+        warnings.push(`${filePath} invalid: field '${field}' ${err.message}`);
+      }
+    }
+  } catch {
+    // Graceful: if ajv fails, skip validation
+  }
+
+  return warnings;
+}
+
+/**
+ * Clear the schema cache (useful for testing).
+ */
+function clearSchemaCache() {
+  _schemaCache = {};
+  _ajvInstance = null;
+}
+
+/**
+ * Standard config file paths relative to project root.
+ */
+const CONFIG_FILES = {
+  core: ".merlin-core/core-config.yaml",
+  framework: ".merlin-core/framework-config.yaml",
+  project: ".merlin-core/project-config.yaml",
+  local: ".merlin-core/local-config.yaml",
+};
+
+/**
+ * Level identifiers for debug/tracing.
+ */
+const LEVELS = {
+  core: "L0",
+  framework: "L1",
+  project: "L2",
+  local: "L3",
+};
+
+// ---------------------------------------------------------------------------
+// YAML loading helpers
+// ---------------------------------------------------------------------------
+
+/**
+ * Load and parse a YAML file. Returns null if file doesn't exist.
+ *
+ * @param {string} projectRoot - Project root directory
+ * @param {string} relativePath - Path relative to projectRoot
+ * @returns {{ data: Object|null, path: string }} Parsed YAML or null
+ */
+function loadYaml(projectRoot, relativePath) {
+  const fullPath = path.join(projectRoot, relativePath);
+
+  try {
+    if (!fs.existsSync(fullPath)) {
+      return { data: null, path: fullPath };
+    }
+
+    const content = fs.readFileSync(fullPath, "utf8");
+    const data = yaml.load(content) || {};
+    return { data, path: fullPath };
+  } catch (error) {
+    throw new Error(`Failed to parse YAML at ${fullPath}: ${error.message}`);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Layered config loading
+// ---------------------------------------------------------------------------
+
+/**
+ * Load configuration using the 4-level hierarchy.
+ *
+ * Order: L0 Core -> L1 Framework -> L2 Project -> L3 Local
+ * Each level deep-merges onto the previous result.
+ *
+ * @param {string} projectRoot - Project root directory
+ * @param {Object} options - Load options
+ * @param {boolean} [options.debug] - Collect source-tracking metadata
+ * @returns {Object} result
+ * @returns {Object} result.config - Merged configuration
+ * @returns {Object} [result.sources] - Per-key source tracking (when debug=true)
+ * @returns {string[]} result.warnings - Lint/interpolation warnings
+ */
+function loadLayeredConfig(projectRoot, options = {}) {
+  const warnings = [];
+  const sources = options.debug ? {} : null;
+
+  // L0: Core (read-only, ships with framework)
+  const l0 = loadYaml(projectRoot, CONFIG_FILES.core);
+  let config = l0.data || {};
+
+  if (options.debug && l0.data) {
+    trackSources(sources, l0.data, LEVELS.core, CONFIG_FILES.core);
+  }
+
+  // Lint L0 for env patterns (should not contain ${...})
+  if (l0.data) {
+    const l0Lint = lintEnvPatterns(l0.data, CONFIG_FILES.core);
+    if (l0Lint.length > 0) {
+      warnings.push(...l0Lint.map((w) => `[LINT] ${w}`));
+    }
+    const l0Validation = validateConfig("core", l0.data, CONFIG_FILES.core);
+    if (l0Validation.length > 0) {
+      warnings.push(...l0Validation.map((w) => `[SCHEMA] ${w}`));
+    }
+  }
+
+  // L1: Framework (ships with framework, customizable)
+  const l1 = loadYaml(projectRoot, CONFIG_FILES.framework);
+  if (l1.data) {
+    config = deepMerge(config, l1.data);
+    if (options.debug) {
+      trackSources(sources, l1.data, LEVELS.framework, CONFIG_FILES.framework);
+    }
+    const l1Lint = lintEnvPatterns(l1.data, CONFIG_FILES.framework);
+    if (l1Lint.length > 0) {
+      warnings.push(...l1Lint.map((w) => `[LINT] ${w}`));
+    }
+    const l1Validation = validateConfig(
+      "framework",
+      l1.data,
+      CONFIG_FILES.framework,
+    );
+    if (l1Validation.length > 0) {
+      warnings.push(...l1Validation.map((w) => `[SCHEMA] ${w}`));
+    }
+  }
+
+  // L2: Project (team-shared, project-specific)
+  const l2 = loadYaml(projectRoot, CONFIG_FILES.project);
+  if (l2.data) {
+    config = deepMerge(config, l2.data);
+    if (options.debug) {
+      trackSources(sources, l2.data, LEVELS.project, CONFIG_FILES.project);
+    }
+    // H5 fix: Lint L2 for env patterns (committed config should not contain ${...})
+    const l2Lint = lintEnvPatterns(l2.data, CONFIG_FILES.project);
+    if (l2Lint.length > 0) {
+      warnings.push(...l2Lint.map((w) => `[LINT] ${w}`));
+    }
+    const l2Validation = validateConfig(
+      "project",
+      l2.data,
+      CONFIG_FILES.project,
+    );
+    if (l2Validation.length > 0) {
+      warnings.push(...l2Validation.map((w) => `[SCHEMA] ${w}`));
+    }
+  }
+
+  // L3: Local (machine-specific, gitignored, may contain ${ENV_VAR})
+  const l3 = loadYaml(projectRoot, CONFIG_FILES.local);
+  if (l3.data) {
+    config = deepMerge(config, l3.data);
+    if (options.debug) {
+      trackSources(sources, l3.data, LEVELS.local, CONFIG_FILES.local);
+    }
+    const l3Validation = validateConfig("local", l3.data, CONFIG_FILES.local);
+    if (l3Validation.length > 0) {
+      warnings.push(...l3Validation.map((w) => `[SCHEMA] ${w}`));
+    }
+  }
+
+  return { config, sources, warnings };
+}
+
+// ---------------------------------------------------------------------------
+// Source tracking (debug mode)
+// ---------------------------------------------------------------------------
+
+/**
+ * Track which level each config key came from (for --debug output).
+ *
+ * @param {Object} sources - Sources accumulator { 'key.path': { level, file } }
+ * @param {Object} data - Config data from a level
+ * @param {string} level - Level label (L0, L1, L2, L3)
+ * @param {string} file - Source file path
+ * @param {string} [prefix] - Key prefix for nested tracking
+ */
+function trackSources(sources, data, level, file, prefix = "") {
+  if (!sources || !data) return;
+
+  for (const [key, value] of Object.entries(data)) {
+    const fullKey = prefix ? `${prefix}.${key}` : key;
+
+    if (value !== null && typeof value === "object" && !Array.isArray(value)) {
+      sources[fullKey] = { level, file };
+      trackSources(sources, value, level, file, fullKey);
+    } else {
+      sources[fullKey] = { level, file };
+    }
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Main entry point
+// ---------------------------------------------------------------------------
+
+/**
+ * Resolve the final configuration for the project.
+ *
+ * Loads all 4 config levels, merges them, and interpolates env vars.
+ * Caches the resolved config via ConfigCache (TTL-based).
+ *
+ * @param {string} projectRoot - Project root directory
+ * @param {Object} [options] - Options
+ * @param {boolean} [options.debug] - Enable source tracking
+ * @param {boolean} [options.skipCache] - Bypass cache
+ * @returns {Object} result
+ * @returns {Object} result.config - Final resolved config
+ * @returns {Object} [result.sources] - Source tracking (debug only)
+ * @returns {string[]} result.warnings - All warnings
+ */
+function resolveConfig(projectRoot, options = {}) {
+  const cacheKey = `resolved:${projectRoot}:${options.debug ? "debug" : "std"}`;
+
+  // Check cache (unless explicitly skipped)
+  if (!options.skipCache) {
+    const cached = globalConfigCache.get(cacheKey);
+    if (cached) return cached;
+  }
+
+  const result = loadLayeredConfig(projectRoot, options);
+
+  // Interpolate environment variables
+  const envWarnings = [];
+  result.config = interpolateEnvVars(result.config, { warnings: envWarnings });
+
+  if (envWarnings.length > 0) {
+    result.warnings.push(...envWarnings.map((w) => `[ENV] ${w}`));
+  }
+
+  // Cache the result
+  globalConfigCache.set(cacheKey, result);
+
+  return result;
+}
+
+/**
+ * Get the raw config from a specific level (no merge, no interpolation).
+ *
+ * @param {string} projectRoot - Project root directory
+ * @param {string} level - Level: 'core' | 'framework' | 'project' | 'local' | 'L0' | 'L1' | 'L2' | 'L3'
+ * @returns {Object|null} Raw config or null if file doesn't exist
+ */
+function getConfigAtLevel(projectRoot, level) {
+  let relativePath;
+
+  switch (level) {
+    case "core":
+    case "0":
+    case "L0":
+      relativePath = CONFIG_FILES.core;
+      break;
+    case "framework":
+    case "1":
+    case "L1":
+      relativePath = CONFIG_FILES.framework;
+      break;
+    case "project":
+    case "2":
+    case "L2":
+      relativePath = CONFIG_FILES.project;
+      break;
+    case "local":
+    case "3":
+    case "L3":
+      relativePath = CONFIG_FILES.local;
+      break;
+    default:
+      throw new Error(`Unknown config level: ${level}`);
+  }
+
+  const { data } = loadYaml(projectRoot, relativePath);
+  return data;
+}
+
+// ---------------------------------------------------------------------------
+// Exports
+// ---------------------------------------------------------------------------
+
+module.exports = {
+  resolveConfig,
+  loadLayeredConfig,
+  getConfigAtLevel,
+  validateConfig,
+  clearSchemaCache,
+  CONFIG_FILES,
+  LEVELS,
+  SCHEMA_FILES,
+};