aios-core 3.1.0 → 3.3.0

package/scripts/validate-manifest.js

@@ -0,0 +1,265 @@
+#!/usr/bin/env node
+/**
+ * Validate Install Manifest
+ * Ensures install-manifest.yaml is up-to-date with actual files
+ *
+ * @script scripts/validate-manifest.js
+ * @story 6.18 - Dynamic Manifest & Brownfield Upgrade System
+ *
+ * Usage:
+ *   node scripts/validate-manifest.js
+ *   npm run validate:manifest
+ *
+ * Exit codes:
+ *   0 - Manifest is valid and up-to-date
+ *   1 - Manifest is outdated or has issues
+ */
+
+const fs = require('fs-extra');
+const path = require('path');
+const yaml = require('js-yaml');
+const { hashFile } = require('../src/installer/file-hasher');
+const {
+  scanDirectory,
+  FOLDERS_TO_COPY,
+  ROOT_FILES_TO_COPY,
+  getFileType,
+} = require('./generate-install-manifest');
+
+/**
+ * Validation result object
+ * @typedef {Object} ValidationResult
+ * @property {boolean} valid - Whether manifest is valid
+ * @property {string[]} newFiles - Files in filesystem but not in manifest
+ * @property {string[]} removedFiles - Files in manifest but not in filesystem
+ * @property {string[]} modifiedFiles - Files with different hashes
+ * @property {string[]} errors - Error messages
+ */
+
+/**
+ * Load and parse the install manifest
+ * @returns {Object|null} - Parsed manifest or null if not found
+ */
+function loadManifest() {
+  const manifestPath = path.join(__dirname, '..', '.aios-core', 'install-manifest.yaml');
+
+  if (!fs.existsSync(manifestPath)) {
+    return null;
+  }
+
+  const content = fs.readFileSync(manifestPath, 'utf8');
+  return yaml.load(content);
+}
+
+/**
+ * Get current files from filesystem
+ * @returns {Map<string, Object>} - Map of relativePath -> file metadata
+ */
+function getCurrentFiles() {
+  const aiosCoreDir = path.join(__dirname, '..', '.aios-core');
+  const filesMap = new Map();
+
+  // Scan folders
+  const allFiles = [];
+  for (const folder of FOLDERS_TO_COPY) {
+    const folderPath = path.join(aiosCoreDir, folder);
+    if (fs.existsSync(folderPath)) {
+      scanDirectory(folderPath, aiosCoreDir, allFiles);
+    }
+  }
+
+  // Add root files
+  for (const file of ROOT_FILES_TO_COPY) {
+    const filePath = path.join(aiosCoreDir, file);
+    if (fs.existsSync(filePath)) {
+      allFiles.push(filePath);
+    }
+  }
+
+  // Build map
+  for (const fullPath of allFiles) {
+    const relativePath = path.relative(aiosCoreDir, fullPath).replace(/\\/g, '/');
+    try {
+      const hash = hashFile(fullPath);
+      filesMap.set(relativePath, {
+        path: relativePath,
+        hash: `sha256:${hash}`,
+        type: getFileType(relativePath),
+      });
+    } catch (error) {
+      console.warn(`Warning: Could not hash ${relativePath}: ${error.message}`);
+    }
+  }
+
+  return filesMap;
+}
+
+/**
+ * Validate manifest against current filesystem
+ * @returns {ValidationResult} - Validation results
+ */
+function validateManifest() {
+  const result = {
+    valid: true,
+    newFiles: [],
+    removedFiles: [],
+    modifiedFiles: [],
+    errors: [],
+  };
+
+  // Load manifest
+  const manifest = loadManifest();
+  if (!manifest) {
+    result.valid = false;
+    result.errors.push('install-manifest.yaml not found');
+    return result;
+  }
+
+  if (!manifest.files || !Array.isArray(manifest.files)) {
+    result.valid = false;
+    result.errors.push('Manifest has no files array');
+    return result;
+  }
+
+  // Get current files
+  const currentFiles = getCurrentFiles();
+
+  // Build set of manifest paths
+  const manifestPaths = new Set();
+  const manifestMap = new Map();
+
+  for (const entry of manifest.files) {
+    const normalizedPath = entry.path.replace(/\\/g, '/');
+    manifestPaths.add(normalizedPath);
+    manifestMap.set(normalizedPath, entry);
+  }
+
+  // Check for new files (in filesystem but not in manifest)
+  for (const [filePath, _fileData] of currentFiles) {
+    if (!manifestPaths.has(filePath)) {
+      result.newFiles.push(filePath);
+      result.valid = false;
+    }
+  }
+
+  // Check for removed files and hash mismatches
+  for (const [manifestPath, manifestEntry] of manifestMap) {
+    const currentFile = currentFiles.get(manifestPath);
+
+    if (!currentFile) {
+      // File in manifest but not in filesystem
+      result.removedFiles.push(manifestPath);
+      result.valid = false;
+    } else if (currentFile.hash !== manifestEntry.hash) {
+      // Hash mismatch
+      result.modifiedFiles.push({
+        path: manifestPath,
+        manifestHash: manifestEntry.hash,
+        currentHash: currentFile.hash,
+      });
+      result.valid = false;
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Print validation report
+ * @param {ValidationResult} result - Validation results
+ */
+function printReport(result) {
+  console.log('='.repeat(60));
+  console.log('AIOS-Core Manifest Validation Report');
+  console.log('='.repeat(60));
+  console.log('');
+
+  if (result.errors.length > 0) {
+    console.log('❌ ERRORS:');
+    for (const error of result.errors) {
+      console.log(`   - ${error}`);
+    }
+    console.log('');
+  }
+
+  if (result.newFiles.length > 0) {
+    console.log(`📁 NEW FILES (${result.newFiles.length}) - not in manifest:`);
+    for (const file of result.newFiles.slice(0, 20)) {
+      console.log(`   + ${file}`);
+    }
+    if (result.newFiles.length > 20) {
+      console.log(`   ... and ${result.newFiles.length - 20} more`);
+    }
+    console.log('');
+  }
+
+  if (result.removedFiles.length > 0) {
+    console.log(`🗑️  REMOVED FILES (${result.removedFiles.length}) - in manifest but missing:`);
+    for (const file of result.removedFiles.slice(0, 20)) {
+      console.log(`   - ${file}`);
+    }
+    if (result.removedFiles.length > 20) {
+      console.log(`   ... and ${result.removedFiles.length - 20} more`);
+    }
+    console.log('');
+  }
+
+  if (result.modifiedFiles.length > 0) {
+    console.log(`📝 MODIFIED FILES (${result.modifiedFiles.length}) - hash mismatch:`);
+    for (const file of result.modifiedFiles.slice(0, 20)) {
+      console.log(`   ~ ${file.path}`);
+      if (process.env.VERBOSE) {
+        console.log(`     manifest: ${file.manifestHash}`);
+        console.log(`     current:  ${file.currentHash}`);
+      }
+    }
+    if (result.modifiedFiles.length > 20) {
+      console.log(`   ... and ${result.modifiedFiles.length - 20} more`);
+    }
+    console.log('');
+  }
+
+  // Summary
+  console.log('-'.repeat(60));
+  if (result.valid) {
+    console.log('✅ Manifest is VALID and up-to-date');
+  } else {
+    console.log('❌ Manifest is OUTDATED');
+    console.log('');
+    console.log('To fix, run: npm run generate:manifest');
+  }
+  console.log('-'.repeat(60));
+}
+
+/**
+ * Main execution
+ */
+async function main() {
+  try {
+    const result = validateManifest();
+    printReport(result);
+
+    if (!result.valid) {
+      process.exit(1);
+    }
+    process.exit(0);
+  } catch (error) {
+    console.error('\n❌ Error validating manifest:', error.message);
+    if (process.env.DEBUG) {
+      console.error(error.stack);
+    }
+    process.exit(1);
+  }
+}
+
+// Run if called directly
+if (require.main === module) {
+  main();
+}
+
+module.exports = {
+  validateManifest,
+  loadManifest,
+  getCurrentFiles,
+  printReport,
+};

package/src/installer/brownfield-upgrader.js

@@ -0,0 +1,438 @@
+/**
+ * Brownfield Upgrader
+ * Handles incremental upgrades for existing AIOS-Core installations
+ *
+ * @module src/installer/brownfield-upgrader
+ * @story 6.18 - Dynamic Manifest & Brownfield Upgrade System
+ */
+
+const fs = require('fs-extra');
+const path = require('path');
+const yaml = require('js-yaml');
+const semver = require('semver');
+const { hashFile, hashesMatch } = require('./file-hasher');
+
+/**
+ * Upgrade report structure
+ * @typedef {Object} UpgradeReport
+ * @property {string} sourceVersion - Version being upgraded to
+ * @property {string} installedVersion - Currently installed version
+ * @property {Object[]} newFiles - Files to be added
+ * @property {Object[]} modifiedFiles - Files that changed (framework side)
+ * @property {Object[]} userModifiedFiles - Files modified by user (won't overwrite)
+ * @property {Object[]} deletedFiles - Files removed from framework
+ * @property {boolean} upgradeAvailable - Whether an upgrade is available
+ */
+
+/**
+ * Load manifest from a given base path
+ * @param {string} basePath - Base path to look for manifest
+ * @param {string} manifestName - Name of manifest file
+ * @returns {Object|null} - Parsed manifest or null
+ */
+function loadManifest(basePath, manifestName = 'install-manifest.yaml') {
+  const manifestPath = path.join(basePath, manifestName);
+
+  if (!fs.existsSync(manifestPath)) {
+    return null;
+  }
+
+  try {
+    const content = fs.readFileSync(manifestPath, 'utf8');
+    return yaml.load(content);
+  } catch (error) {
+    console.error(`Error loading manifest from ${manifestPath}:`, error.message);
+    return null;
+  }
+}
+
+/**
+ * Load installed manifest from target project
+ * @param {string} targetDir - Target project directory
+ * @returns {Object|null} - Installed manifest or null if not found
+ */
+function loadInstalledManifest(targetDir) {
+  return loadManifest(
+    path.join(targetDir, '.aios-core'),
+    '.installed-manifest.yaml',
+  );
+}
+
+/**
+ * Load source manifest from package
+ * @param {string} sourceDir - Source package directory (.aios-core from npm)
+ * @returns {Object|null} - Source manifest
+ */
+function loadSourceManifest(sourceDir) {
+  return loadManifest(sourceDir, 'install-manifest.yaml');
+}
+
+/**
+ * Build file map from manifest for quick lookup
+ * @param {Object} manifest - Manifest object
+ * @returns {Map<string, Object>} - Map of path -> entry
+ */
+function buildFileMap(manifest) {
+  const map = new Map();
+  if (manifest && manifest.files) {
+    for (const entry of manifest.files) {
+      const normalizedPath = entry.path.replace(/\\/g, '/');
+      map.set(normalizedPath, entry);
+    }
+  }
+  return map;
+}
+
+/**
+ * Check if a file has been modified by the user
+ * Compares current file hash against installed manifest hash
+ *
+ * @param {string} filePath - Absolute path to file
+ * @param {string} expectedHash - Hash from installed manifest (with sha256: prefix)
+ * @returns {boolean} - True if file was modified by user
+ */
+function isUserModified(filePath, expectedHash) {
+  if (!fs.existsSync(filePath)) {
+    return false;
+  }
+
+  try {
+    const currentHash = `sha256:${hashFile(filePath)}`;
+    return !hashesMatch(currentHash, expectedHash);
+  } catch (_error) {
+    // If we can't hash, assume it's modified
+    return true;
+  }
+}
+
+/**
+ * Generate upgrade report comparing source and installed manifests
+ *
+ * @param {Object} sourceManifest - Manifest from source (npm package)
+ * @param {Object} installedManifest - Manifest from target installation
+ * @param {string} targetDir - Target project directory
+ * @returns {UpgradeReport} - Detailed upgrade report
+ */
+function generateUpgradeReport(sourceManifest, installedManifest, targetDir) {
+  const report = {
+    sourceVersion: sourceManifest?.version || 'unknown',
+    installedVersion: installedManifest?.installed_version || installedManifest?.version || 'unknown',
+    newFiles: [],
+    modifiedFiles: [],
+    userModifiedFiles: [],
+    deletedFiles: [],
+    unchangedFiles: 0,
+    upgradeAvailable: false,
+  };
+
+  // Check if upgrade is available via semver
+  if (sourceManifest?.version && installedManifest?.installed_version) {
+    const sourceVer = semver.coerce(sourceManifest.version);
+    const installedVer = semver.coerce(installedManifest.installed_version);
+
+    if (sourceVer && installedVer) {
+      report.upgradeAvailable = semver.gt(sourceVer, installedVer);
+    }
+  } else if (sourceManifest?.version && !installedManifest) {
+    // No installed manifest means this is a fresh install scenario
+    report.upgradeAvailable = false;
+  }
+
+  const sourceMap = buildFileMap(sourceManifest);
+  const installedMap = buildFileMap(installedManifest);
+
+  const aiosCoreDir = path.join(targetDir, '.aios-core');
+
+  // Check source files against installed
+  for (const [filePath, sourceEntry] of sourceMap) {
+    const installedEntry = installedMap.get(filePath);
+    const absolutePath = path.join(aiosCoreDir, filePath);
+
+    if (!installedEntry) {
+      // New file in source
+      report.newFiles.push({
+        path: filePath,
+        type: sourceEntry.type,
+        hash: sourceEntry.hash,
+        size: sourceEntry.size,
+      });
+    } else if (!hashesMatch(sourceEntry.hash, installedEntry.hash)) {
+      // File changed in source
+      // Check if user modified the local copy
+      if (isUserModified(absolutePath, installedEntry.hash)) {
+        report.userModifiedFiles.push({
+          path: filePath,
+          type: sourceEntry.type,
+          sourceHash: sourceEntry.hash,
+          installedHash: installedEntry.hash,
+          reason: 'User modified local file',
+        });
+      } else {
+        report.modifiedFiles.push({
+          path: filePath,
+          type: sourceEntry.type,
+          sourceHash: sourceEntry.hash,
+          installedHash: installedEntry.hash,
+        });
+      }
+    } else {
+      report.unchangedFiles++;
+    }
+  }
+
+  // Check for deleted files (in installed but not in source)
+  for (const [filePath, installedEntry] of installedMap) {
+    if (!sourceMap.has(filePath)) {
+      report.deletedFiles.push({
+        path: filePath,
+        type: installedEntry.type,
+        hash: installedEntry.hash,
+      });
+    }
+  }
+
+  return report;
+}
+
+/**
+ * Apply upgrade to target directory
+ *
+ * @param {UpgradeReport} report - Upgrade report
+ * @param {string} sourceDir - Source directory (npm package .aios-core)
+ * @param {string} targetDir - Target project directory
+ * @param {Object} options - Upgrade options
+ * @param {boolean} options.dryRun - If true, don't actually copy files
+ * @param {boolean} options.includeModified - If true, also update modified files
+ * @returns {Object} - Result of upgrade operation
+ */
+async function applyUpgrade(report, sourceDir, targetDir, options = {}) {
+  const { dryRun = false, includeModified = true } = options;
+  const result = {
+    success: true,
+    filesInstalled: [],
+    filesSkipped: [],
+    errors: [],
+  };
+
+  const aiosCoreDir = path.join(targetDir, '.aios-core');
+
+  // Ensure .aios-core directory exists
+  if (!dryRun) {
+    fs.ensureDirSync(aiosCoreDir);
+  }
+
+  // Install new files
+  for (const file of report.newFiles) {
+    const sourcePath = path.join(sourceDir, file.path);
+    const targetPath = path.join(aiosCoreDir, file.path);
+
+    try {
+      if (!dryRun) {
+        fs.ensureDirSync(path.dirname(targetPath));
+        fs.copyFileSync(sourcePath, targetPath);
+      }
+      result.filesInstalled.push({ path: file.path, action: 'new' });
+    } catch (error) {
+      result.errors.push({ path: file.path, error: error.message });
+      result.success = false;
+    }
+  }
+
+  // Update modified files (if option enabled)
+  if (includeModified) {
+    for (const file of report.modifiedFiles) {
+      const sourcePath = path.join(sourceDir, file.path);
+      const targetPath = path.join(aiosCoreDir, file.path);
+
+      try {
+        if (!dryRun) {
+          fs.ensureDirSync(path.dirname(targetPath));
+          fs.copyFileSync(sourcePath, targetPath);
+        }
+        result.filesInstalled.push({ path: file.path, action: 'updated' });
+      } catch (error) {
+        result.errors.push({ path: file.path, error: error.message });
+        result.success = false;
+      }
+    }
+  }
+
+  // Skip user-modified files
+  for (const file of report.userModifiedFiles) {
+    result.filesSkipped.push({
+      path: file.path,
+      reason: 'User modified - preserving local changes',
+    });
+  }
+
+  // Note: We don't delete files that were removed from source
+  // This is intentional to preserve user additions
+  for (const file of report.deletedFiles) {
+    result.filesSkipped.push({
+      path: file.path,
+      reason: 'Removed from source - keeping local copy',
+    });
+  }
+
+  return result;
+}
+
+/**
+ * Create or update the installed manifest after upgrade
+ *
+ * @param {string} targetDir - Target project directory
+ * @param {Object} sourceManifest - Source manifest that was installed
+ * @param {string} sourcePackage - Name and version of source package
+ */
+function updateInstalledManifest(targetDir, sourceManifest, sourcePackage) {
+  const installedManifestPath = path.join(targetDir, '.aios-core', '.installed-manifest.yaml');
+
+  const installedManifest = {
+    installed_at: new Date().toISOString(),
+    installed_from: sourcePackage,
+    installed_version: sourceManifest.version,
+    source_manifest_hash: `sha256:${require('./file-hasher').hashString(
+      JSON.stringify(sourceManifest.files),
+    )}`,
+    file_count: sourceManifest.files.length,
+    files: sourceManifest.files.map(f => ({
+      path: f.path,
+      hash: f.hash,
+      modified_by_user: false,
+    })),
+  };
+
+  const yamlContent = yaml.dump(installedManifest, {
+    indent: 2,
+    lineWidth: 120,
+    noRefs: true,
+  });
+
+  const header = `# AIOS-Core Installed Manifest
+# This file tracks what was installed from the npm package
+# Used for brownfield upgrades to detect changes
+# DO NOT EDIT MANUALLY
+#
+`;
+
+  fs.ensureDirSync(path.dirname(installedManifestPath));
+  fs.writeFileSync(installedManifestPath, header + yamlContent, 'utf8');
+
+  return installedManifestPath;
+}
+
+/**
+ * Check if an upgrade is available
+ *
+ * @param {string} sourceDir - Source package directory
+ * @param {string} targetDir - Target project directory
+ * @returns {Object} - { available: boolean, from: string, to: string }
+ */
+function checkUpgradeAvailable(sourceDir, targetDir) {
+  const sourceManifest = loadSourceManifest(sourceDir);
+  const installedManifest = loadInstalledManifest(targetDir);
+
+  if (!sourceManifest) {
+    return { available: false, error: 'Source manifest not found' };
+  }
+
+  if (!installedManifest) {
+    return { available: false, reason: 'No installed manifest (fresh install)' };
+  }
+
+  const sourceVer = semver.coerce(sourceManifest.version);
+  const installedVer = semver.coerce(installedManifest.installed_version);
+
+  if (!sourceVer || !installedVer) {
+    return { available: false, error: 'Invalid version numbers' };
+  }
+
+  return {
+    available: semver.gt(sourceVer, installedVer),
+    from: installedManifest.installed_version,
+    to: sourceManifest.version,
+  };
+}
+
+/**
+ * Format upgrade report for display
+ * @param {UpgradeReport} report - Upgrade report
+ * @returns {string} - Formatted report string
+ */
+function formatUpgradeReport(report) {
+  const lines = [];
+
+  lines.push('═'.repeat(60));
+  lines.push('AIOS-Core Upgrade Report');
+  lines.push('═'.repeat(60));
+  lines.push('');
+  lines.push(`Current Version: ${report.installedVersion}`);
+  lines.push(`Available Version: ${report.sourceVersion}`);
+  lines.push(`Upgrade Available: ${report.upgradeAvailable ? 'Yes ✅' : 'No'}`);
+  lines.push('');
+
+  if (report.newFiles.length > 0) {
+    lines.push(`📁 New Files (${report.newFiles.length}):`);
+    for (const file of report.newFiles.slice(0, 10)) {
+      lines.push(`   + ${file.path} [${file.type}]`);
+    }
+    if (report.newFiles.length > 10) {
+      lines.push(`   ... and ${report.newFiles.length - 10} more`);
+    }
+    lines.push('');
+  }
+
+  if (report.modifiedFiles.length > 0) {
+    lines.push(`📝 Modified Files (${report.modifiedFiles.length}):`);
+    for (const file of report.modifiedFiles.slice(0, 10)) {
+      lines.push(`   ~ ${file.path} [${file.type}]`);
+    }
+    if (report.modifiedFiles.length > 10) {
+      lines.push(`   ... and ${report.modifiedFiles.length - 10} more`);
+    }
+    lines.push('');
+  }
+
+  if (report.userModifiedFiles.length > 0) {
+    lines.push(`⚠️  User Modified (${report.userModifiedFiles.length}) - will be preserved:`);
+    for (const file of report.userModifiedFiles.slice(0, 10)) {
+      lines.push(`   ⊘ ${file.path}`);
+    }
+    if (report.userModifiedFiles.length > 10) {
+      lines.push(`   ... and ${report.userModifiedFiles.length - 10} more`);
+    }
+    lines.push('');
+  }
+
+  if (report.deletedFiles.length > 0) {
+    lines.push(`🗑️  Removed from Source (${report.deletedFiles.length}):`);
+    for (const file of report.deletedFiles.slice(0, 5)) {
+      lines.push(`   - ${file.path}`);
+    }
+    if (report.deletedFiles.length > 5) {
+      lines.push(`   ... and ${report.deletedFiles.length - 5} more`);
+    }
+    lines.push('');
+  }
+
+  lines.push('─'.repeat(60));
+  const totalChanges = report.newFiles.length + report.modifiedFiles.length;
+  const totalSkipped = report.userModifiedFiles.length + report.deletedFiles.length;
+  lines.push(`Summary: ${totalChanges} files to update, ${totalSkipped} files preserved`);
+  lines.push('─'.repeat(60));
+
+  return lines.join('\n');
+}
+
+module.exports = {
+  loadManifest,
+  loadInstalledManifest,
+  loadSourceManifest,
+  generateUpgradeReport,
+  applyUpgrade,
+  updateInstalledManifest,
+  checkUpgradeAvailable,
+  formatUpgradeReport,
+  buildFileMap,
+  isUserModified,
+};