bmad-method 4.37.0 → 4.39.1

package/tools/flattener/main.js

@@ -1,20 +1,14 @@
-#!/usr/bin/env node
-
-const { Command } = require("commander");
-const fs = require("fs-extra");
-const path = require("node:path");
-const process = require("node:process");
+const { Command } = require('commander');
+const fs = require('fs-extra');
+const path = require('node:path');
+const process = require('node:process');
 
 // Modularized components
-const { findProjectRoot } = require("./projectRoot.js");
-const { promptYesNo, promptPath } = require("./prompts.js");
-const {
-  discoverFiles,
-  filterFiles,
-  aggregateFileContents,
-} = require("./files.js");
-const { generateXMLOutput } = require("./xml.js");
-const { calculateStatistics } = require("./stats.js");
+const { findProjectRoot } = require('./projectRoot.js');
+const { promptYesNo, promptPath } = require('./prompts.js');
+const { discoverFiles, filterFiles, aggregateFileContents } = require('./files.js');
+const { generateXMLOutput } = require('./xml.js');
+const { calculateStatistics } = require('./stats.js');
 
 /**
  * Recursively discover all files in a directory
@@ -73,30 +67,30 @@ const { calculateStatistics } = require("./stats.js");
 const program = new Command();
 
 program
-  .name("bmad-flatten")
-  .description("BMad-Method codebase flattener tool")
-  .version("1.0.0")
-  .option("-i, --input <path>", "Input directory to flatten", process.cwd())
-  .option("-o, --output <path>", "Output file path", "flattened-codebase.xml")
+  .name('bmad-flatten')
+  .description('BMad-Method codebase flattener tool')
+  .version('1.0.0')
+  .option('-i, --input <path>', 'Input directory to flatten', process.cwd())
+  .option('-o, --output <path>', 'Output file path', 'flattened-codebase.xml')
   .action(async (options) => {
     let inputDir = path.resolve(options.input);
     let outputPath = path.resolve(options.output);
 
     // Detect if user explicitly provided -i/--input or -o/--output
     const argv = process.argv.slice(2);
-    const userSpecifiedInput = argv.some((a) =>
-      a === "-i" || a === "--input" || a.startsWith("--input=")
+    const userSpecifiedInput = argv.some(
+      (a) => a === '-i' || a === '--input' || a.startsWith('--input='),
     );
-    const userSpecifiedOutput = argv.some((a) =>
-      a === "-o" || a === "--output" || a.startsWith("--output=")
+    const userSpecifiedOutput = argv.some(
+      (a) => a === '-o' || a === '--output' || a.startsWith('--output='),
     );
-    const noPathArgs = !userSpecifiedInput && !userSpecifiedOutput;
+    const noPathArguments = !userSpecifiedInput && !userSpecifiedOutput;
 
-    if (noPathArgs) {
+    if (noPathArguments) {
       const detectedRoot = await findProjectRoot(process.cwd());
       const suggestedOutput = detectedRoot
-        ? path.join(detectedRoot, "flattened-codebase.xml")
-        : path.resolve("flattened-codebase.xml");
+        ? path.join(detectedRoot, 'flattened-codebase.xml')
+        : path.resolve('flattened-codebase.xml');
 
       if (detectedRoot) {
         const useDefaults = await promptYesNo(
@@ -107,29 +101,23 @@ program
           inputDir = detectedRoot;
           outputPath = suggestedOutput;
         } else {
-          inputDir = await promptPath(
-            "Enter input directory path",
-            process.cwd(),
-          );
+          inputDir = await promptPath('Enter input directory path', process.cwd());
           outputPath = await promptPath(
-            "Enter output file path",
-            path.join(inputDir, "flattened-codebase.xml"),
+            'Enter output file path',
+            path.join(inputDir, 'flattened-codebase.xml'),
           );
         }
       } else {
-        console.log("Could not auto-detect a project root.");
-        inputDir = await promptPath(
-          "Enter input directory path",
-          process.cwd(),
-        );
+        console.log('Could not auto-detect a project root.');
+        inputDir = await promptPath('Enter input directory path', process.cwd());
         outputPath = await promptPath(
-          "Enter output file path",
-          path.join(inputDir, "flattened-codebase.xml"),
+          'Enter output file path',
+          path.join(inputDir, 'flattened-codebase.xml'),
         );
       }
     } else {
       console.error(
-        "Could not auto-detect a project root and no arguments were provided. Please specify -i/--input and -o/--output.",
+        'Could not auto-detect a project root and no arguments were provided. Please specify -i/--input and -o/--output.',
       );
       process.exit(1);
     }
@@ -137,30 +125,25 @@ program
     // Ensure output directory exists
     await fs.ensureDir(path.dirname(outputPath));
 
-    console.log(`Flattening codebase from: ${inputDir}`);
-    console.log(`Output file: ${outputPath}`);
-
     try {
       // Verify input directory exists
-      if (!await fs.pathExists(inputDir)) {
+      if (!(await fs.pathExists(inputDir))) {
         console.error(`❌ Error: Input directory does not exist: ${inputDir}`);
         process.exit(1);
       }
 
       // Import ora dynamically
-      const { default: ora } = await import("ora");
+      const { default: ora } = await import('ora');
 
       // Start file discovery with spinner
-      const discoverySpinner = ora("🔍 Discovering files...").start();
+      const discoverySpinner = ora('🔍 Discovering files...').start();
       const files = await discoverFiles(inputDir);
       const filteredFiles = await filterFiles(files, inputDir);
-      discoverySpinner.succeed(
-        `📁 Found ${filteredFiles.length} files to include`,
-      );
+      discoverySpinner.succeed(`📁 Found ${filteredFiles.length} files to include`);
 
       // Process files with progress tracking
-      console.log("Reading file contents");
-      const processingSpinner = ora("📄 Processing files...").start();
+      console.log('Reading file contents');
+      const processingSpinner = ora('📄 Processing files...').start();
       const aggregatedContent = await aggregateFileContents(
         filteredFiles,
         inputDir,
@@ -172,40 +155,413 @@ program
       if (aggregatedContent.errors.length > 0) {
         console.log(`Errors: ${aggregatedContent.errors.length}`);
       }
-      console.log(`Text files: ${aggregatedContent.textFiles.length}`);
-      if (aggregatedContent.binaryFiles.length > 0) {
-        console.log(`Binary files: ${aggregatedContent.binaryFiles.length}`);
-      }
 
       // Generate XML output using streaming
-      const xmlSpinner = ora("🔧 Generating XML output...").start();
+      const xmlSpinner = ora('🔧 Generating XML output...').start();
       await generateXMLOutput(aggregatedContent, outputPath);
-      xmlSpinner.succeed("📝 XML generation completed");
+      xmlSpinner.succeed('📝 XML generation completed');
 
       // Calculate and display statistics
       const outputStats = await fs.stat(outputPath);
-      const stats = calculateStatistics(aggregatedContent, outputStats.size);
+      const stats = await calculateStatistics(aggregatedContent, outputStats.size, inputDir);
 
       // Display completion summary
-      console.log("\n📊 Completion Summary:");
+      console.log('\n📊 Completion Summary:');
       console.log(
-        `✅ Successfully processed ${filteredFiles.length} files into ${
-          path.basename(outputPath)
-        }`,
+        `✅ Successfully processed ${filteredFiles.length} files into ${path.basename(outputPath)}`,
       );
       console.log(`📁 Output file: ${outputPath}`);
       console.log(`📏 Total source size: ${stats.totalSize}`);
       console.log(`📄 Generated XML size: ${stats.xmlSize}`);
-      console.log(
-        `📝 Total lines of code: ${stats.totalLines.toLocaleString()}`,
-      );
+      console.log(`📝 Total lines of code: ${stats.totalLines.toLocaleString()}`);
       console.log(`🔢 Estimated tokens: ${stats.estimatedTokens}`);
       console.log(
-        `📊 File breakdown: ${stats.textFiles} text, ${stats.binaryFiles} binary, ${stats.errorFiles} errors`,
+        `📊 File breakdown: ${stats.textFiles} text, ${stats.binaryFiles} binary, ${stats.errorFiles} errors\n`,
+      );
+
+      // Ask user if they want detailed stats + markdown report
+      const generateDetailed = await promptYesNo(
+        'Generate detailed stats (console + markdown) now?',
+        true,
       );
+
+      if (generateDetailed) {
+        // Additional detailed stats
+        console.log('\n📈 Size Percentiles:');
+        console.log(
+          `   Avg: ${Math.round(stats.avgFileSize).toLocaleString()} B, Median: ${Math.round(
+            stats.medianFileSize,
+          ).toLocaleString()} B, p90: ${stats.p90.toLocaleString()} B, p95: ${stats.p95.toLocaleString()} B, p99: ${stats.p99.toLocaleString()} B`,
+        );
+
+        if (Array.isArray(stats.histogram) && stats.histogram.length > 0) {
+          console.log('\n🧮 Size Histogram:');
+          for (const b of stats.histogram.slice(0, 2)) {
+            console.log(`   ${b.label}: ${b.count} files, ${b.bytes.toLocaleString()} bytes`);
+          }
+          if (stats.histogram.length > 2) {
+            console.log(`   … and ${stats.histogram.length - 2} more buckets`);
+          }
+        }
+
+        if (Array.isArray(stats.byExtension) && stats.byExtension.length > 0) {
+          const topExt = stats.byExtension.slice(0, 2);
+          console.log('\n📦 Top Extensions:');
+          for (const e of topExt) {
+            const pct = stats.totalBytes ? (e.bytes / stats.totalBytes) * 100 : 0;
+            console.log(
+              `   ${e.ext}: ${e.count} files, ${e.bytes.toLocaleString()} bytes (${pct.toFixed(
+                2,
+              )}%)`,
+            );
+          }
+          if (stats.byExtension.length > 2) {
+            console.log(`   … and ${stats.byExtension.length - 2} more extensions`);
+          }
+        }
+
+        if (Array.isArray(stats.byDirectory) && stats.byDirectory.length > 0) {
+          const topDir = stats.byDirectory.slice(0, 2);
+          console.log('\n📂 Top Directories:');
+          for (const d of topDir) {
+            const pct = stats.totalBytes ? (d.bytes / stats.totalBytes) * 100 : 0;
+            console.log(
+              `   ${d.dir}: ${d.count} files, ${d.bytes.toLocaleString()} bytes (${pct.toFixed(
+                2,
+              )}%)`,
+            );
+          }
+          if (stats.byDirectory.length > 2) {
+            console.log(`   … and ${stats.byDirectory.length - 2} more directories`);
+          }
+        }
+
+        if (Array.isArray(stats.depthDistribution) && stats.depthDistribution.length > 0) {
+          console.log('\n🌳 Depth Distribution:');
+          const dd = stats.depthDistribution.slice(0, 2);
+          let line = '   ' + dd.map((d) => `${d.depth}:${d.count}`).join('  ');
+          if (stats.depthDistribution.length > 2) {
+            line += `  … +${stats.depthDistribution.length - 2} more`;
+          }
+          console.log(line);
+        }
+
+        if (Array.isArray(stats.longestPaths) && stats.longestPaths.length > 0) {
+          console.log('\n🧵 Longest Paths:');
+          for (const p of stats.longestPaths.slice(0, 2)) {
+            console.log(`   ${p.path} (${p.length} chars, ${p.size.toLocaleString()} bytes)`);
+          }
+          if (stats.longestPaths.length > 2) {
+            console.log(`   … and ${stats.longestPaths.length - 2} more paths`);
+          }
+        }
+
+        if (stats.temporal) {
+          console.log('\n⏱️ Temporal:');
+          if (stats.temporal.oldest) {
+            console.log(
+              `   Oldest: ${stats.temporal.oldest.path} (${stats.temporal.oldest.mtime})`,
+            );
+          }
+          if (stats.temporal.newest) {
+            console.log(
+              `   Newest: ${stats.temporal.newest.path} (${stats.temporal.newest.mtime})`,
+            );
+          }
+          if (Array.isArray(stats.temporal.ageBuckets)) {
+            console.log('   Age buckets:');
+            for (const b of stats.temporal.ageBuckets.slice(0, 2)) {
+              console.log(`     ${b.label}: ${b.count} files, ${b.bytes.toLocaleString()} bytes`);
+            }
+            if (stats.temporal.ageBuckets.length > 2) {
+              console.log(`     … and ${stats.temporal.ageBuckets.length - 2} more buckets`);
+            }
+          }
+        }
+
+        if (stats.quality) {
+          console.log('\n✅ Quality Signals:');
+          console.log(`   Zero-byte files: ${stats.quality.zeroByteFiles}`);
+          console.log(`   Empty text files: ${stats.quality.emptyTextFiles}`);
+          console.log(`   Hidden files: ${stats.quality.hiddenFiles}`);
+          console.log(`   Symlinks: ${stats.quality.symlinks}`);
+          console.log(
+            `   Large files (>= ${(stats.quality.largeThreshold / (1024 * 1024)).toFixed(
+              0,
+            )} MB): ${stats.quality.largeFilesCount}`,
+          );
+          console.log(
+            `   Suspiciously large files (>= 100 MB): ${stats.quality.suspiciousLargeFilesCount}`,
+          );
+        }
+
+        if (Array.isArray(stats.duplicateCandidates) && stats.duplicateCandidates.length > 0) {
+          console.log('\n🧬 Duplicate Candidates:');
+          for (const d of stats.duplicateCandidates.slice(0, 2)) {
+            console.log(`   ${d.reason}: ${d.count} files @ ${d.size.toLocaleString()} bytes`);
+          }
+          if (stats.duplicateCandidates.length > 2) {
+            console.log(`   … and ${stats.duplicateCandidates.length - 2} more groups`);
+          }
+        }
+
+        if (typeof stats.compressibilityRatio === 'number') {
+          console.log(
+            `\n🗜️ Compressibility ratio (sampled): ${(stats.compressibilityRatio * 100).toFixed(
+              2,
+            )}%`,
+          );
+        }
+
+        if (stats.git && stats.git.isRepo) {
+          console.log('\n🔧 Git:');
+          console.log(
+            `   Tracked: ${stats.git.trackedCount} files, ${stats.git.trackedBytes.toLocaleString()} bytes`,
+          );
+          console.log(
+            `   Untracked: ${stats.git.untrackedCount} files, ${stats.git.untrackedBytes.toLocaleString()} bytes`,
+          );
+          if (Array.isArray(stats.git.lfsCandidates) && stats.git.lfsCandidates.length > 0) {
+            console.log('   LFS candidates (top 2):');
+            for (const f of stats.git.lfsCandidates.slice(0, 2)) {
+              console.log(`     ${f.path} (${f.size.toLocaleString()} bytes)`);
+            }
+            if (stats.git.lfsCandidates.length > 2) {
+              console.log(`     … and ${stats.git.lfsCandidates.length - 2} more`);
+            }
+          }
+        }
+
+        if (Array.isArray(stats.largestFiles) && stats.largestFiles.length > 0) {
+          console.log('\n📚 Largest Files (top 2):');
+          for (const f of stats.largestFiles.slice(0, 2)) {
+            // Show LOC for text files when available; omit ext and mtime
+            let locStr = '';
+            if (!f.isBinary && Array.isArray(aggregatedContent?.textFiles)) {
+              const tf = aggregatedContent.textFiles.find((t) => t.path === f.path);
+              if (tf && typeof tf.lines === 'number') {
+                locStr = `, LOC: ${tf.lines.toLocaleString()}`;
+              }
+            }
+            console.log(
+              `   ${f.path} – ${f.sizeFormatted} (${f.percentOfTotal.toFixed(2)}%)${locStr}`,
+            );
+          }
+          if (stats.largestFiles.length > 2) {
+            console.log(`   … and ${stats.largestFiles.length - 2} more files`);
+          }
+        }
+
+        // Write a comprehensive markdown report next to the XML
+        {
+          const mdPath = outputPath.endsWith('.xml')
+            ? outputPath.replace(/\.xml$/i, '.stats.md')
+            : outputPath + '.stats.md';
+          try {
+            const pct = (num, den) => (den ? (num / den) * 100 : 0);
+            const md = [];
+            md.push(
+              `# 🧾 Flatten Stats for ${path.basename(outputPath)}`,
+              '',
+              '## 📊 Summary',
+              `- Total source size: ${stats.totalSize}`,
+              `- Generated XML size: ${stats.xmlSize}`,
+              `- Total lines of code: ${stats.totalLines.toLocaleString()}`,
+              `- Estimated tokens: ${stats.estimatedTokens}`,
+              `- File breakdown: ${stats.textFiles} text, ${stats.binaryFiles} binary, ${stats.errorFiles} errors`,
+              '',
+              '## 📈 Size Percentiles',
+              `Avg: ${Math.round(stats.avgFileSize).toLocaleString()} B, Median: ${Math.round(
+                stats.medianFileSize,
+              ).toLocaleString()} B, p90: ${stats.p90.toLocaleString()} B, p95: ${stats.p95.toLocaleString()} B, p99: ${stats.p99.toLocaleString()} B`,
+              '',
+            );
+
+            // Histogram
+            if (Array.isArray(stats.histogram) && stats.histogram.length > 0) {
+              md.push(
+                '## 🧮 Size Histogram',
+                '| Bucket | Files | Bytes |',
+                '| --- | ---: | ---: |',
+              );
+              for (const b of stats.histogram) {
+                md.push(`| ${b.label} | ${b.count} | ${b.bytes.toLocaleString()} |`);
+              }
+              md.push('');
+            }
+
+            // Top Extensions
+            if (Array.isArray(stats.byExtension) && stats.byExtension.length > 0) {
+              md.push(
+                '## 📦 Top Extensions by Bytes (Top 20)',
+                '| Ext | Files | Bytes | % of total |',
+                '| --- | ---: | ---: | ---: |',
+              );
+              for (const e of stats.byExtension.slice(0, 20)) {
+                const p = pct(e.bytes, stats.totalBytes);
+                md.push(
+                  `| ${e.ext} | ${e.count} | ${e.bytes.toLocaleString()} | ${p.toFixed(2)}% |`,
+                );
+              }
+              md.push('');
+            }
+
+            // Top Directories
+            if (Array.isArray(stats.byDirectory) && stats.byDirectory.length > 0) {
+              md.push(
+                '## 📂 Top Directories by Bytes (Top 20)',
+                '| Directory | Files | Bytes | % of total |',
+                '| --- | ---: | ---: | ---: |',
+              );
+              for (const d of stats.byDirectory.slice(0, 20)) {
+                const p = pct(d.bytes, stats.totalBytes);
+                md.push(
+                  `| ${d.dir} | ${d.count} | ${d.bytes.toLocaleString()} | ${p.toFixed(2)}% |`,
+                );
+              }
+              md.push('');
+            }
+
+            // Depth distribution
+            if (Array.isArray(stats.depthDistribution) && stats.depthDistribution.length > 0) {
+              md.push('## 🌳 Depth Distribution', '| Depth | Count |', '| ---: | ---: |');
+              for (const d of stats.depthDistribution) {
+                md.push(`| ${d.depth} | ${d.count} |`);
+              }
+              md.push('');
+            }
+
+            // Longest paths
+            if (Array.isArray(stats.longestPaths) && stats.longestPaths.length > 0) {
+              md.push(
+                '## 🧵 Longest Paths (Top 25)',
+                '| Path | Length | Bytes |',
+                '| --- | ---: | ---: |',
+              );
+              for (const pth of stats.longestPaths) {
+                md.push(`| ${pth.path} | ${pth.length} | ${pth.size.toLocaleString()} |`);
+              }
+              md.push('');
+            }
+
+            // Temporal
+            if (stats.temporal) {
+              md.push('## ⏱️ Temporal');
+              if (stats.temporal.oldest) {
+                md.push(`- Oldest: ${stats.temporal.oldest.path} (${stats.temporal.oldest.mtime})`);
+              }
+              if (stats.temporal.newest) {
+                md.push(`- Newest: ${stats.temporal.newest.path} (${stats.temporal.newest.mtime})`);
+              }
+              if (Array.isArray(stats.temporal.ageBuckets)) {
+                md.push('', '| Age | Files | Bytes |', '| --- | ---: | ---: |');
+                for (const b of stats.temporal.ageBuckets) {
+                  md.push(`| ${b.label} | ${b.count} | ${b.bytes.toLocaleString()} |`);
+                }
+              }
+              md.push('');
+            }
+
+            // Quality signals
+            if (stats.quality) {
+              md.push(
+                '## ✅ Quality Signals',
+                `- Zero-byte files: ${stats.quality.zeroByteFiles}`,
+                `- Empty text files: ${stats.quality.emptyTextFiles}`,
+                `- Hidden files: ${stats.quality.hiddenFiles}`,
+                `- Symlinks: ${stats.quality.symlinks}`,
+                `- Large files (>= ${(stats.quality.largeThreshold / (1024 * 1024)).toFixed(0)} MB): ${stats.quality.largeFilesCount}`,
+                `- Suspiciously large files (>= 100 MB): ${stats.quality.suspiciousLargeFilesCount}`,
+                '',
+              );
+            }
+
+            // Duplicates
+            if (Array.isArray(stats.duplicateCandidates) && stats.duplicateCandidates.length > 0) {
+              md.push(
+                '## 🧬 Duplicate Candidates',
+                '| Reason | Files | Size (bytes) |',
+                '| --- | ---: | ---: |',
+              );
+              for (const d of stats.duplicateCandidates) {
+                md.push(`| ${d.reason} | ${d.count} | ${d.size.toLocaleString()} |`);
+              }
+              md.push('', '### 🧬 Duplicate Groups Details');
+              let dupIndex = 1;
+              for (const d of stats.duplicateCandidates) {
+                md.push(
+                  `#### Group ${dupIndex}: ${d.count} files @ ${d.size.toLocaleString()} bytes (${d.reason})`,
+                );
+                if (Array.isArray(d.files) && d.files.length > 0) {
+                  for (const fp of d.files) {
+                    md.push(`- ${fp}`);
+                  }
+                } else {
+                  md.push('- (file list unavailable)');
+                }
+                md.push('');
+                dupIndex++;
+              }
+              md.push('');
+            }
+
+            // Compressibility
+            if (typeof stats.compressibilityRatio === 'number') {
+              md.push(
+                '## 🗜️ Compressibility',
+                `Sampled compressibility ratio: ${(stats.compressibilityRatio * 100).toFixed(2)}%`,
+                '',
+              );
+            }
+
+            // Git
+            if (stats.git && stats.git.isRepo) {
+              md.push(
+                '## 🔧 Git',
+                `- Tracked: ${stats.git.trackedCount} files, ${stats.git.trackedBytes.toLocaleString()} bytes`,
+                `- Untracked: ${stats.git.untrackedCount} files, ${stats.git.untrackedBytes.toLocaleString()} bytes`,
+              );
+              if (Array.isArray(stats.git.lfsCandidates) && stats.git.lfsCandidates.length > 0) {
+                md.push('', '### 📦 LFS Candidates (Top 20)', '| Path | Bytes |', '| --- | ---: |');
+                for (const f of stats.git.lfsCandidates.slice(0, 20)) {
+                  md.push(`| ${f.path} | ${f.size.toLocaleString()} |`);
+                }
+              }
+              md.push('');
+            }
+
+            // Largest Files
+            if (Array.isArray(stats.largestFiles) && stats.largestFiles.length > 0) {
+              md.push(
+                '## 📚 Largest Files (Top 50)',
+                '| Path | Size | % of total | LOC |',
+                '| --- | ---: | ---: | ---: |',
+              );
+              for (const f of stats.largestFiles) {
+                let loc = '';
+                if (!f.isBinary && Array.isArray(aggregatedContent?.textFiles)) {
+                  const tf = aggregatedContent.textFiles.find((t) => t.path === f.path);
+                  if (tf && typeof tf.lines === 'number') {
+                    loc = tf.lines.toLocaleString();
+                  }
+                }
+                md.push(
+                  `| ${f.path} | ${f.sizeFormatted} | ${f.percentOfTotal.toFixed(2)}% | ${loc} |`,
+                );
+              }
+              md.push('');
+            }
+
+            await fs.writeFile(mdPath, md.join('\n'));
+            console.log(`\n🧾 Detailed stats report written to: ${mdPath}`);
+          } catch (error) {
+            console.warn(`⚠️ Failed to write stats markdown: ${error.message}`);
+          }
+        }
+      }
     } catch (error) {
-      console.error("❌ Critical error:", error.message);
-      console.error("An unexpected error occurred.");
+      console.error('❌ Critical error:', error.message);
+      console.error('An unexpected error occurred.');
       process.exit(1);
     }
   });