bmad-fh 6.0.0-alpha.052779ef

package/tools/flattener/aggregate.js

@@ -0,0 +1,76 @@
+const fs = require('fs-extra');
+const path = require('node:path');
+const os = require('node:os');
+const { isBinaryFile } = require('./binary.js');
+
+/**
+ * Aggregate file contents with bounded concurrency.
+ * Returns text files, binary files (with size), and errors.
+ * @param {string[]} files absolute file paths
+ * @param {string} rootDir
+ * @param {{ text?: string, warn?: (msg: string) => void } | null} spinner
+ */
+async function aggregateFileContents(files, rootDir, spinner = null) {
+  const results = {
+    textFiles: [],
+    binaryFiles: [],
+    errors: [],
+    totalFiles: files.length,
+    processedFiles: 0,
+  };
+
+  // Automatic concurrency selection based on CPU count and workload size.
+  // - Base on 2x logical CPUs, clamped to [2, 64]
+  // - For very small workloads, avoid excessive parallelism
+  const cpuCount = os.cpus && Array.isArray(os.cpus()) ? os.cpus().length : os.cpus?.length || 4;
+  let concurrency = Math.min(64, Math.max(2, (Number(cpuCount) || 4) * 2));
+  if (files.length > 0 && files.length < concurrency) {
+    concurrency = Math.max(1, Math.min(concurrency, Math.ceil(files.length / 2)));
+  }
+
+  async function processOne(filePath) {
+    try {
+      const relativePath = path.relative(rootDir, filePath);
+      if (spinner) {
+        spinner.text = `Processing: ${relativePath} (${results.processedFiles + 1}/${results.totalFiles})`;
+      }
+
+      const binary = await isBinaryFile(filePath);
+      if (binary) {
+        const { size } = await fs.stat(filePath);
+        results.binaryFiles.push({ path: relativePath, absolutePath: filePath, size });
+      } else {
+        const content = await fs.readFile(filePath, 'utf8');
+        results.textFiles.push({
+          path: relativePath,
+          absolutePath: filePath,
+          content,
+          size: content.length,
+          lines: content.split('\n').length,
+        });
+      }
+    } catch (error) {
+      const relativePath = path.relative(rootDir, filePath);
+      const errorInfo = { path: relativePath, absolutePath: filePath, error: error.message };
+      results.errors.push(errorInfo);
+      if (spinner) {
+        spinner.warn(`Warning: Could not read file ${relativePath}: ${error.message}`);
+      } else {
+        console.warn(`Warning: Could not read file ${relativePath}: ${error.message}`);
+      }
+    } finally {
+      results.processedFiles++;
+    }
+  }
+
+  for (let index = 0; index < files.length; index += concurrency) {
+    const slice = files.slice(index, index + concurrency);
+    await Promise.all(slice.map(processOne));
+  }
+
+  return results;
+}
+
+module.exports = {
+  aggregateFileContents,
+};

package/tools/flattener/binary.js

@@ -0,0 +1,80 @@
+const fsp = require('node:fs/promises');
+const path = require('node:path');
+const { Buffer } = require('node:buffer');
+
+/**
+ * Efficiently determine if a file is binary without reading the whole file.
+ * - Fast path by extension for common binaries
+ * - Otherwise read a small prefix and check for NUL bytes
+ * @param {string} filePath
+ * @returns {Promise<boolean>}
+ */
+async function isBinaryFile(filePath) {
+  try {
+    const stats = await fsp.stat(filePath);
+    if (stats.isDirectory()) {
+      throw new Error('EISDIR: illegal operation on a directory');
+    }
+
+    const binaryExtensions = new Set([
+      '.jpg',
+      '.jpeg',
+      '.png',
+      '.gif',
+      '.bmp',
+      '.ico',
+      '.svg',
+      '.pdf',
+      '.doc',
+      '.docx',
+      '.xls',
+      '.xlsx',
+      '.ppt',
+      '.pptx',
+      '.zip',
+      '.tar',
+      '.gz',
+      '.rar',
+      '.7z',
+      '.exe',
+      '.dll',
+      '.so',
+      '.dylib',
+      '.mp3',
+      '.mp4',
+      '.avi',
+      '.mov',
+      '.wav',
+      '.ttf',
+      '.otf',
+      '.woff',
+      '.woff2',
+      '.bin',
+      '.dat',
+      '.db',
+      '.sqlite',
+    ]);
+
+    const extension = path.extname(filePath).toLowerCase();
+    if (binaryExtensions.has(extension)) return true;
+    if (stats.size === 0) return false;
+
+    const sampleSize = Math.min(4096, stats.size);
+    const fd = await fsp.open(filePath, 'r');
+    try {
+      const buffer = Buffer.allocUnsafe(sampleSize);
+      const { bytesRead } = await fd.read(buffer, 0, sampleSize, 0);
+      const slice = bytesRead === sampleSize ? buffer : buffer.subarray(0, bytesRead);
+      return slice.includes(0);
+    } finally {
+      await fd.close();
+    }
+  } catch (error) {
+    console.warn(`Warning: Could not determine if file is binary: ${filePath} - ${error.message}`);
+    return false;
+  }
+}
+
+module.exports = {
+  isBinaryFile,
+};

package/tools/flattener/discovery.js

@@ -0,0 +1,71 @@
+const path = require('node:path');
+const { execFile } = require('node:child_process');
+const { promisify } = require('node:util');
+const { glob } = require('glob');
+const { loadIgnore } = require('./ignoreRules.js');
+
+const pExecFile = promisify(execFile);
+
+async function isGitRepo(rootDir) {
+  try {
+    const { stdout } = await pExecFile('git', ['rev-parse', '--is-inside-work-tree'], {
+      cwd: rootDir,
+    });
+    return (
+      String(stdout || '')
+        .toString()
+        .trim() === 'true'
+    );
+  } catch {
+    return false;
+  }
+}
+
+async function gitListFiles(rootDir) {
+  try {
+    const { stdout } = await pExecFile('git', ['ls-files', '-co', '--exclude-standard'], {
+      cwd: rootDir,
+    });
+    return String(stdout || '')
+      .split(/\r?\n/)
+      .map((s) => s.trim())
+      .filter(Boolean);
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * Discover files under rootDir.
+ * - Prefer git ls-files when available for speed/correctness
+ * - Fallback to glob and apply unified ignore rules
+ * @param {string} rootDir
+ * @param {object} [options]
+ * @param {boolean} [options.preferGit=true]
+ * @returns {Promise<string[]>} absolute file paths
+ */
+async function discoverFiles(rootDir, options = {}) {
+  const { preferGit = true } = options;
+  const { filter } = await loadIgnore(rootDir);
+
+  // Try git first
+  if (preferGit && (await isGitRepo(rootDir))) {
+    const relFiles = await gitListFiles(rootDir);
+    const filteredRel = relFiles.filter((p) => filter(p));
+    return filteredRel.map((p) => path.resolve(rootDir, p));
+  }
+
+  // Glob fallback
+  const globbed = await glob('**/*', {
+    cwd: rootDir,
+    nodir: true,
+    dot: true,
+    follow: false,
+  });
+  const filteredRel = globbed.filter((p) => filter(p));
+  return filteredRel.map((p) => path.resolve(rootDir, p));
+}
+
+module.exports = {
+  discoverFiles,
+};

package/tools/flattener/files.js

@@ -0,0 +1,35 @@
+const path = require('node:path');
+const discovery = require('./discovery.js');
+const ignoreRules = require('./ignoreRules.js');
+const { isBinaryFile } = require('./binary.js');
+const { aggregateFileContents } = require('./aggregate.js');
+
+// Backward-compatible signature; delegate to central loader
+async function parseGitignore(gitignorePath) {
+  return await ignoreRules.parseGitignore(gitignorePath);
+}
+
+async function discoverFiles(rootDir) {
+  try {
+    // Delegate to discovery module which respects .gitignore and defaults
+    return await discovery.discoverFiles(rootDir, { preferGit: true });
+  } catch (error) {
+    console.error('Error discovering files:', error.message);
+    return [];
+  }
+}
+
+async function filterFiles(files, rootDir) {
+  const { filter } = await ignoreRules.loadIgnore(rootDir);
+  const relativeFiles = files.map((f) => path.relative(rootDir, f));
+  const filteredRelative = relativeFiles.filter((p) => filter(p));
+  return filteredRelative.map((p) => path.resolve(rootDir, p));
+}
+
+module.exports = {
+  parseGitignore,
+  discoverFiles,
+  isBinaryFile,
+  aggregateFileContents,
+  filterFiles,
+};

package/tools/flattener/ignoreRules.js

@@ -0,0 +1,172 @@
+const fs = require('fs-extra');
+const path = require('node:path');
+const ignore = require('ignore');
+
+// Central default ignore patterns for discovery and filtering.
+// These complement .gitignore and are applied regardless of VCS presence.
+const DEFAULT_PATTERNS = [
+  // Project/VCS
+  '**/_bmad/**',
+  '**/.git/**',
+  '**/.svn/**',
+  '**/.hg/**',
+  '**/.bzr/**',
+  // Package/build outputs
+  '**/node_modules/**',
+  '**/bower_components/**',
+  '**/vendor/**',
+  '**/packages/**',
+  '**/build/**',
+  '**/dist/**',
+  '**/out/**',
+  '**/target/**',
+  '**/bin/**',
+  '**/obj/**',
+  '**/release/**',
+  '**/debug/**',
+  // Environments
+  '**/.venv/**',
+  '**/venv/**',
+  '**/.virtualenv/**',
+  '**/virtualenv/**',
+  '**/env/**',
+  // Logs & coverage
+  '**/*.log',
+  '**/npm-debug.log*',
+  '**/yarn-debug.log*',
+  '**/yarn-error.log*',
+  '**/lerna-debug.log*',
+  '**/coverage/**',
+  '**/.nyc_output/**',
+  '**/.coverage/**',
+  '**/test-results/**',
+  // Caches & temp
+  '**/.cache/**',
+  '**/.tmp/**',
+  '**/.temp/**',
+  '**/tmp/**',
+  '**/temp/**',
+  '**/.sass-cache/**',
+  // IDE/editor
+  '**/.vscode/**',
+  '**/.idea/**',
+  '**/*.swp',
+  '**/*.swo',
+  '**/*~',
+  '**/.project',
+  '**/.classpath',
+  '**/.settings/**',
+  '**/*.sublime-project',
+  '**/*.sublime-workspace',
+  // Lockfiles
+  '**/package-lock.json',
+  '**/yarn.lock',
+  '**/pnpm-lock.yaml',
+  '**/composer.lock',
+  '**/Pipfile.lock',
+  // Python/Java/compiled artifacts
+  '**/*.pyc',
+  '**/*.pyo',
+  '**/*.pyd',
+  '**/__pycache__/**',
+  '**/*.class',
+  '**/*.jar',
+  '**/*.war',
+  '**/*.ear',
+  '**/*.o',
+  '**/*.so',
+  '**/*.dll',
+  '**/*.exe',
+  // System junk
+  '**/lib64/**',
+  '**/.venv/lib64/**',
+  '**/venv/lib64/**',
+  '**/_site/**',
+  '**/.jekyll-cache/**',
+  '**/.jekyll-metadata',
+  '**/.DS_Store',
+  '**/.DS_Store?',
+  '**/._*',
+  '**/.Spotlight-V100/**',
+  '**/.Trashes/**',
+  '**/ehthumbs.db',
+  '**/Thumbs.db',
+  '**/desktop.ini',
+  // XML outputs
+  '**/flattened-codebase.xml',
+  '**/repomix-output.xml',
+  // Images, media, fonts, archives, docs, dylibs
+  '**/*.jpg',
+  '**/*.jpeg',
+  '**/*.png',
+  '**/*.gif',
+  '**/*.bmp',
+  '**/*.ico',
+  '**/*.svg',
+  '**/*.pdf',
+  '**/*.doc',
+  '**/*.docx',
+  '**/*.xls',
+  '**/*.xlsx',
+  '**/*.ppt',
+  '**/*.pptx',
+  '**/*.zip',
+  '**/*.tar',
+  '**/*.gz',
+  '**/*.rar',
+  '**/*.7z',
+  '**/*.dylib',
+  '**/*.mp3',
+  '**/*.mp4',
+  '**/*.avi',
+  '**/*.mov',
+  '**/*.wav',
+  '**/*.ttf',
+  '**/*.otf',
+  '**/*.woff',
+  '**/*.woff2',
+  // Env files
+  '**/.env',
+  '**/.env.*',
+  '**/*.env',
+  // Misc
+  '**/junit.xml',
+];
+
+async function readIgnoreFile(filePath) {
+  try {
+    if (!(await fs.pathExists(filePath))) return [];
+    const content = await fs.readFile(filePath, 'utf8');
+    return content
+      .split('\n')
+      .map((l) => l.trim())
+      .filter((l) => l && !l.startsWith('#'));
+  } catch {
+    return [];
+  }
+}
+
+// Backward compatible export matching previous signature
+async function parseGitignore(gitignorePath) {
+  return readIgnoreFile(gitignorePath);
+}
+
+async function loadIgnore(rootDir, extraPatterns = []) {
+  const ig = ignore();
+  const gitignorePath = path.join(rootDir, '.gitignore');
+  const patterns = [...(await readIgnoreFile(gitignorePath)), ...DEFAULT_PATTERNS, ...extraPatterns];
+  // De-duplicate
+  const unique = [...new Set(patterns.map(String))];
+  ig.add(unique);
+
+  // Include-only filter: return true if path should be included
+  const filter = (relativePath) => !ig.ignores(relativePath.replaceAll('\\', '/'));
+
+  return { ig, filter, patterns: unique };
+}
+
+module.exports = {
+  DEFAULT_PATTERNS,
+  parseGitignore,
+  loadIgnore,
+};