@zeyue0329/xiaoma-cli 1.0.7 → 1.0.10

package/tools/flattener/projectRoot.js

@@ -0,0 +1,206 @@
+const fs = require('fs-extra');
+const path = require('node:path');
+
+// Deno/Node compatibility: explicitly import process
+const process = require('node:process');
+const { execFile } = require('node:child_process');
+const { promisify } = require('node:util');
+const execFileAsync = promisify(execFile);
+
+// Simple memoization across calls (keyed by realpath of startDir)
+const _cache = new Map();
+
+async function _tryRun(cmd, args, cwd, timeoutMs = 500) {
+  try {
+    const { stdout } = await execFileAsync(cmd, args, {
+      cwd,
+      timeout: timeoutMs,
+      windowsHide: true,
+      maxBuffer: 1024 * 1024,
+    });
+    const out = String(stdout || '').trim();
+    return out || null;
+  } catch {
+    return null;
+  }
+}
+
+async function _detectVcsTopLevel(startDir) {
+  // Run common VCS root queries in parallel; ignore failures
+  const gitP = _tryRun('git', ['rev-parse', '--show-toplevel'], startDir);
+  const hgP = _tryRun('hg', ['root'], startDir);
+  const svnP = (async () => {
+    const show = await _tryRun('svn', ['info', '--show-item', 'wc-root'], startDir);
+    if (show) return show;
+    const info = await _tryRun('svn', ['info'], startDir);
+    if (info) {
+      const line = info
+        .split(/\r?\n/)
+        .find((l) => l.toLowerCase().startsWith('working copy root path:'));
+      if (line) return line.split(':').slice(1).join(':').trim();
+    }
+    return null;
+  })();
+  const [git, hg, svn] = await Promise.all([gitP, hgP, svnP]);
+  return git || hg || svn || null;
+}
+
+/**
+ * Attempt to find the project root by walking up from startDir.
+ * Uses a robust, prioritized set of ecosystem markers (VCS > workspaces/monorepo > lock/build > language config).
+ * Also recognizes package.json with "workspaces" as a workspace root.
+ * You can augment markers via env PROJECT_ROOT_MARKERS as a comma-separated list of file/dir names.
+ * @param {string} startDir
+ * @returns {Promise<string|null>} project root directory or null if not found
+ */
+async function findProjectRoot(startDir) {
+  try {
+    // Resolve symlinks for robustness (e.g., when invoked from a symlinked path)
+    let dir = path.resolve(startDir);
+    try {
+      dir = await fs.realpath(dir);
+    } catch {
+      // ignore if realpath fails; continue with resolved path
+    }
+    const startKey = dir; // preserve starting point for caching
+    if (_cache.has(startKey)) return _cache.get(startKey);
+    const fsRoot = path.parse(dir).root;
+
+    // Helper to safely check for existence
+    const exists = (p) => fs.pathExists(p);
+
+    // Build checks: an array of { makePath: (dir) => string, weight }
+    const checks = [];
+
+    const add = (rel, weight) => {
+      const makePath = (d) => (Array.isArray(rel) ? path.join(d, ...rel) : path.join(d, rel));
+      checks.push({ makePath, weight });
+    };
+
+    // Highest priority: explicit sentinel markers
+    add('.project-root', 110);
+    add('.workspace-root', 110);
+    add('.repo-root', 110);
+
+    // Highest priority: VCS roots
+    add('.git', 100);
+    add('.hg', 95);
+    add('.svn', 95);
+
+    // Monorepo/workspace indicators
+    add('pnpm-workspace.yaml', 90);
+    add('lerna.json', 90);
+    add('turbo.json', 90);
+    add('nx.json', 90);
+    add('rush.json', 90);
+    add('go.work', 90);
+    add('WORKSPACE', 90);
+    add('WORKSPACE.bazel', 90);
+    add('MODULE.bazel', 90);
+    add('pants.toml', 90);
+
+    // Lockfiles and package-manager/top-level locks
+    add('yarn.lock', 85);
+    add('pnpm-lock.yaml', 85);
+    add('package-lock.json', 85);
+    add('bun.lockb', 85);
+    add('Cargo.lock', 85);
+    add('composer.lock', 85);
+    add('poetry.lock', 85);
+    add('Pipfile.lock', 85);
+    add('Gemfile.lock', 85);
+
+    // Build-system root indicators
+    add('settings.gradle', 80);
+    add('settings.gradle.kts', 80);
+    add('gradlew', 80);
+    add('pom.xml', 80);
+    add('build.sbt', 80);
+    add(['project', 'build.properties'], 80);
+
+    // Language/project config markers
+    add('deno.json', 75);
+    add('deno.jsonc', 75);
+    add('pyproject.toml', 75);
+    add('Pipfile', 75);
+    add('requirements.txt', 75);
+    add('go.mod', 75);
+    add('Cargo.toml', 75);
+    add('composer.json', 75);
+    add('mix.exs', 75);
+    add('Gemfile', 75);
+    add('CMakeLists.txt', 75);
+    add('stack.yaml', 75);
+    add('cabal.project', 75);
+    add('rebar.config', 75);
+    add('pubspec.yaml', 75);
+    add('flake.nix', 75);
+    add('shell.nix', 75);
+    add('default.nix', 75);
+    add('.tool-versions', 75);
+    add('package.json', 74); // generic Node project (lower than lockfiles/workspaces)
+
+    // Changesets
+    add(['.changeset', 'config.json'], 70);
+    add('.changeset', 70);
+
+    // Custom markers via env (comma-separated names)
+    if (process.env.PROJECT_ROOT_MARKERS) {
+      for (const name of process.env.PROJECT_ROOT_MARKERS.split(',')
+        .map((s) => s.trim())
+        .filter(Boolean)) {
+        add(name, 72);
+      }
+    }
+
+    /** Check for package.json with "workspaces" */
+    const hasWorkspacePackageJson = async (d) => {
+      const pkgPath = path.join(d, 'package.json');
+      if (!(await exists(pkgPath))) return false;
+      try {
+        const raw = await fs.readFile(pkgPath, 'utf8');
+        const pkg = JSON.parse(raw);
+        return Boolean(pkg && pkg.workspaces);
+      } catch {
+        return false;
+      }
+    };
+
+    let best = null; // { dir, weight }
+
+    // Try to detect VCS toplevel once up-front; treat as authoritative slightly above .git marker
+    const vcsTop = await _detectVcsTopLevel(dir);
+    if (vcsTop) {
+      best = { dir: vcsTop, weight: 101 };
+    }
+
+    while (true) {
+      // Special check: package.json with "workspaces"
+      if ((await hasWorkspacePackageJson(dir)) && (!best || 90 >= best.weight))
+        best = { dir, weight: 90 };
+
+      // Evaluate all other checks in parallel
+      const results = await Promise.all(
+        checks.map(async (c) => ({ c, ok: await exists(c.makePath(dir)) })),
+      );
+
+      for (const { c, ok } of results) {
+        if (!ok) continue;
+        if (!best || c.weight >= best.weight) {
+          best = { dir, weight: c.weight };
+        }
+      }
+
+      if (dir === fsRoot) break;
+      dir = path.dirname(dir);
+    }
+
+    const out = best ? best.dir : null;
+    _cache.set(startKey, out);
+    return out;
+  } catch {
+    return null;
+  }
+}
+
+module.exports = { findProjectRoot };

package/tools/flattener/prompts.js

@@ -0,0 +1,44 @@
+const os = require('node:os');
+const path = require('node:path');
+const readline = require('node:readline');
+const process = require('node:process');
+
+function expandHome(p) {
+  if (!p) return p;
+  if (p.startsWith('~')) return path.join(os.homedir(), p.slice(1));
+  return p;
+}
+
+function createRl() {
+  return readline.createInterface({
+    input: process.stdin,
+    output: process.stdout,
+  });
+}
+
+function promptQuestion(question) {
+  return new Promise((resolve) => {
+    const rl = createRl();
+    rl.question(question, (answer) => {
+      rl.close();
+      resolve(answer);
+    });
+  });
+}
+
+async function promptYesNo(question, defaultYes = true) {
+  const suffix = defaultYes ? ' [Y/n] ' : ' [y/N] ';
+  const ans = (await promptQuestion(`${question}${suffix}`)).trim().toLowerCase();
+  if (!ans) return defaultYes;
+  if (['y', 'yes'].includes(ans)) return true;
+  if (['n', 'no'].includes(ans)) return false;
+  return promptYesNo(question, defaultYes);
+}
+
+async function promptPath(question, defaultValue) {
+  const prompt = `${question}${defaultValue ? ` (default: ${defaultValue})` : ''}: `;
+  const ans = (await promptQuestion(prompt)).trim();
+  return expandHome(ans || defaultValue);
+}
+
+module.exports = { promptYesNo, promptPath, promptQuestion, expandHome };

package/tools/flattener/stats.helpers.js

@@ -0,0 +1,395 @@
+'use strict';
+
+const fs = require('node:fs/promises');
+const path = require('node:path');
+const zlib = require('node:zlib');
+const { Buffer } = require('node:buffer');
+const crypto = require('node:crypto');
+const cp = require('node:child_process');
+
+const KB = 1024;
+const MB = 1024 * KB;
+
+const formatSize = (bytes) => {
+  if (bytes < 1024) return `${bytes} B`;
+  if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
+  if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+  return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`;
+};
+
+const percentile = (sorted, p) => {
+  if (sorted.length === 0) return 0;
+  const idx = Math.min(sorted.length - 1, Math.max(0, Math.ceil((p / 100) * sorted.length) - 1));
+  return sorted[idx];
+};
+
+async function processWithLimit(items, fn, concurrency = 64) {
+  for (let i = 0; i < items.length; i += concurrency) {
+    await Promise.all(items.slice(i, i + concurrency).map(fn));
+  }
+}
+
+async function enrichAllFiles(textFiles, binaryFiles) {
+  /** @type {Array<{ path: string; absolutePath: string; size: number; lines?: number; isBinary: boolean; ext: string; dir: string; depth: number; hidden: boolean; mtimeMs: number; isSymlink: boolean; }>} */
+  const allFiles = [];
+
+  async function enrich(file, isBinary) {
+    const ext = (path.extname(file.path) || '').toLowerCase();
+    const dir = path.dirname(file.path) || '.';
+    const depth = file.path.split(path.sep).filter(Boolean).length;
+    const hidden = file.path.split(path.sep).some((seg) => seg.startsWith('.'));
+    let mtimeMs = 0;
+    let isSymlink = false;
+    try {
+      const lst = await fs.lstat(file.absolutePath);
+      mtimeMs = lst.mtimeMs;
+      isSymlink = lst.isSymbolicLink();
+    } catch {
+      /* ignore lstat errors during enrichment */
+    }
+    allFiles.push({
+      path: file.path,
+      absolutePath: file.absolutePath,
+      size: file.size || 0,
+      lines: file.lines,
+      isBinary,
+      ext,
+      dir,
+      depth,
+      hidden,
+      mtimeMs,
+      isSymlink,
+    });
+  }
+
+  await processWithLimit(textFiles, (f) => enrich(f, false));
+  await processWithLimit(binaryFiles, (f) => enrich(f, true));
+  return allFiles;
+}
+
+function buildHistogram(allFiles) {
+  const buckets = [
+    [1 * KB, '0–1KB'],
+    [10 * KB, '1–10KB'],
+    [100 * KB, '10–100KB'],
+    [1 * MB, '100KB–1MB'],
+    [10 * MB, '1–10MB'],
+    [100 * MB, '10–100MB'],
+    [Infinity, '>=100MB'],
+  ];
+  const histogram = buckets.map(([_, label]) => ({ label, count: 0, bytes: 0 }));
+  for (const f of allFiles) {
+    for (const [i, bucket] of buckets.entries()) {
+      if (f.size < bucket[0]) {
+        histogram[i].count++;
+        histogram[i].bytes += f.size;
+        break;
+      }
+    }
+  }
+  return histogram;
+}
+
+function aggregateByExtension(allFiles) {
+  const byExtension = new Map();
+  for (const f of allFiles) {
+    const key = f.ext || '<none>';
+    const v = byExtension.get(key) || { ext: key, count: 0, bytes: 0 };
+    v.count++;
+    v.bytes += f.size;
+    byExtension.set(key, v);
+  }
+  return [...byExtension.values()].sort((a, b) => b.bytes - a.bytes);
+}
+
+function aggregateByDirectory(allFiles) {
+  const byDirectory = new Map();
+  function addDirBytes(dir, bytes) {
+    const v = byDirectory.get(dir) || { dir, count: 0, bytes: 0 };
+    v.count++;
+    v.bytes += bytes;
+    byDirectory.set(dir, v);
+  }
+  for (const f of allFiles) {
+    const parts = f.dir === '.' ? [] : f.dir.split(path.sep);
+    let acc = '';
+    for (let i = 0; i < parts.length; i++) {
+      acc = i === 0 ? parts[0] : acc + path.sep + parts[i];
+      addDirBytes(acc, f.size);
+    }
+    if (parts.length === 0) addDirBytes('.', f.size);
+  }
+  return [...byDirectory.values()].sort((a, b) => b.bytes - a.bytes);
+}
+
+function computeDepthAndLongest(allFiles) {
+  const depthDistribution = new Map();
+  for (const f of allFiles) {
+    depthDistribution.set(f.depth, (depthDistribution.get(f.depth) || 0) + 1);
+  }
+  const longestPaths = [...allFiles]
+    .sort((a, b) => b.path.length - a.path.length)
+    .slice(0, 25)
+    .map((f) => ({ path: f.path, length: f.path.length, size: f.size }));
+  const depthDist = [...depthDistribution.entries()]
+    .sort((a, b) => a[0] - b[0])
+    .map(([depth, count]) => ({ depth, count }));
+  return { depthDist, longestPaths };
+}
+
+function computeTemporal(allFiles, nowMs) {
+  let oldest = null,
+    newest = null;
+  const ageBuckets = [
+    { label: '> 1 year', minDays: 365, maxDays: Infinity, count: 0, bytes: 0 },
+    { label: '6–12 months', minDays: 180, maxDays: 365, count: 0, bytes: 0 },
+    { label: '1–6 months', minDays: 30, maxDays: 180, count: 0, bytes: 0 },
+    { label: '7–30 days', minDays: 7, maxDays: 30, count: 0, bytes: 0 },
+    { label: '1–7 days', minDays: 1, maxDays: 7, count: 0, bytes: 0 },
+    { label: '< 1 day', minDays: 0, maxDays: 1, count: 0, bytes: 0 },
+  ];
+  for (const f of allFiles) {
+    const ageDays = Math.max(0, (nowMs - (f.mtimeMs || nowMs)) / (24 * 60 * 60 * 1000));
+    for (const b of ageBuckets) {
+      if (ageDays >= b.minDays && ageDays < b.maxDays) {
+        b.count++;
+        b.bytes += f.size;
+        break;
+      }
+    }
+    if (!oldest || f.mtimeMs < oldest.mtimeMs) oldest = f;
+    if (!newest || f.mtimeMs > newest.mtimeMs) newest = f;
+  }
+  return {
+    oldest: oldest
+      ? { path: oldest.path, mtime: oldest.mtimeMs ? new Date(oldest.mtimeMs).toISOString() : null }
+      : null,
+    newest: newest
+      ? { path: newest.path, mtime: newest.mtimeMs ? new Date(newest.mtimeMs).toISOString() : null }
+      : null,
+    ageBuckets,
+  };
+}
+
+function computeQuality(allFiles, textFiles) {
+  const zeroByteFiles = allFiles.filter((f) => f.size === 0).length;
+  const emptyTextFiles = textFiles.filter(
+    (f) => (f.size || 0) === 0 || (f.lines || 0) === 0,
+  ).length;
+  const hiddenFiles = allFiles.filter((f) => f.hidden).length;
+  const symlinks = allFiles.filter((f) => f.isSymlink).length;
+  const largeThreshold = 50 * MB;
+  const suspiciousThreshold = 100 * MB;
+  const largeFilesCount = allFiles.filter((f) => f.size >= largeThreshold).length;
+  const suspiciousLargeFilesCount = allFiles.filter((f) => f.size >= suspiciousThreshold).length;
+  return {
+    zeroByteFiles,
+    emptyTextFiles,
+    hiddenFiles,
+    symlinks,
+    largeFilesCount,
+    suspiciousLargeFilesCount,
+    largeThreshold,
+  };
+}
+
+function computeDuplicates(allFiles, textFiles) {
+  const duplicatesBySize = new Map();
+  for (const f of allFiles) {
+    const key = String(f.size);
+    const arr = duplicatesBySize.get(key) || [];
+    arr.push(f);
+    duplicatesBySize.set(key, arr);
+  }
+  const duplicateCandidates = [];
+  for (const [sizeKey, arr] of duplicatesBySize.entries()) {
+    if (arr.length < 2) continue;
+    const textGroup = arr.filter((f) => !f.isBinary);
+    const otherGroup = arr.filter((f) => f.isBinary);
+    const contentHashGroups = new Map();
+    for (const tf of textGroup) {
+      try {
+        const src = textFiles.find((x) => x.absolutePath === tf.absolutePath);
+        const content = src ? src.content : '';
+        const h = crypto.createHash('sha1').update(content).digest('hex');
+        const g = contentHashGroups.get(h) || [];
+        g.push(tf);
+        contentHashGroups.set(h, g);
+      } catch {
+        /* ignore hashing errors for duplicate detection */
+      }
+    }
+    for (const [_h, g] of contentHashGroups.entries()) {
+      if (g.length > 1)
+        duplicateCandidates.push({
+          reason: 'same-size+text-hash',
+          size: Number(sizeKey),
+          count: g.length,
+          files: g.map((f) => f.path),
+        });
+    }
+    if (otherGroup.length > 1) {
+      duplicateCandidates.push({
+        reason: 'same-size',
+        size: Number(sizeKey),
+        count: otherGroup.length,
+        files: otherGroup.map((f) => f.path),
+      });
+    }
+  }
+  return duplicateCandidates;
+}
+
+function estimateCompressibility(textFiles) {
+  let compSampleBytes = 0;
+  let compCompressedBytes = 0;
+  for (const tf of textFiles) {
+    try {
+      const sampleLen = Math.min(256 * 1024, tf.size || 0);
+      if (sampleLen <= 0) continue;
+      const sample = tf.content.slice(0, sampleLen);
+      const gz = zlib.gzipSync(Buffer.from(sample, 'utf8'));
+      compSampleBytes += sampleLen;
+      compCompressedBytes += gz.length;
+    } catch {
+      /* ignore compression errors during sampling */
+    }
+  }
+  return compSampleBytes > 0 ? compCompressedBytes / compSampleBytes : null;
+}
+
+function computeGitInfo(allFiles, rootDir, largeThreshold) {
+  const info = {
+    isRepo: false,
+    trackedCount: 0,
+    trackedBytes: 0,
+    untrackedCount: 0,
+    untrackedBytes: 0,
+    lfsCandidates: [],
+  };
+  try {
+    if (!rootDir) return info;
+    const top = cp
+      .execFileSync('git', ['rev-parse', '--show-toplevel'], {
+        cwd: rootDir,
+        stdio: ['ignore', 'pipe', 'ignore'],
+      })
+      .toString()
+      .trim();
+    if (!top) return info;
+    info.isRepo = true;
+    const out = cp.execFileSync('git', ['ls-files', '-z'], {
+      cwd: rootDir,
+      stdio: ['ignore', 'pipe', 'ignore'],
+    });
+    const tracked = new Set(out.toString().split('\0').filter(Boolean));
+    let trackedBytes = 0,
+      trackedCount = 0,
+      untrackedBytes = 0,
+      untrackedCount = 0;
+    const lfsCandidates = [];
+    for (const f of allFiles) {
+      const isTracked = tracked.has(f.path);
+      if (isTracked) {
+        trackedCount++;
+        trackedBytes += f.size;
+        if (f.size >= largeThreshold) lfsCandidates.push({ path: f.path, size: f.size });
+      } else {
+        untrackedCount++;
+        untrackedBytes += f.size;
+      }
+    }
+    info.trackedCount = trackedCount;
+    info.trackedBytes = trackedBytes;
+    info.untrackedCount = untrackedCount;
+    info.untrackedBytes = untrackedBytes;
+    info.lfsCandidates = lfsCandidates.sort((a, b) => b.size - a.size).slice(0, 50);
+  } catch {
+    /* git not available or not a repo, ignore */
+  }
+  return info;
+}
+
+function computeLargestFiles(allFiles, totalBytes) {
+  const toPct = (num, den) => (den === 0 ? 0 : (num / den) * 100);
+  return [...allFiles]
+    .sort((a, b) => b.size - a.size)
+    .slice(0, 50)
+    .map((f) => ({
+      path: f.path,
+      size: f.size,
+      sizeFormatted: formatSize(f.size),
+      percentOfTotal: toPct(f.size, totalBytes),
+      ext: f.ext || '',
+      isBinary: f.isBinary,
+      mtime: f.mtimeMs ? new Date(f.mtimeMs).toISOString() : null,
+    }));
+}
+
+function mdTable(rows, headers) {
+  const header = `| ${headers.join(' | ')} |`;
+  const sep = `| ${headers.map(() => '---').join(' | ')} |`;
+  const body = rows.map((r) => `| ${r.join(' | ')} |`).join('\n');
+  return `${header}\n${sep}\n${body}`;
+}
+
+function buildMarkdownReport(largestFiles, byExtensionArr, byDirectoryArr, totalBytes) {
+  const toPct = (num, den) => (den === 0 ? 0 : (num / den) * 100);
+  const md = [];
+  md.push(
+    '\n### Top Largest Files (Top 50)\n',
+    mdTable(
+      largestFiles.map((f) => [
+        f.path,
+        f.sizeFormatted,
+        `${f.percentOfTotal.toFixed(2)}%`,
+        f.ext || '',
+        f.isBinary ? 'binary' : 'text',
+      ]),
+      ['Path', 'Size', '% of total', 'Ext', 'Type'],
+    ),
+    '\n\n### Top Extensions by Bytes (Top 20)\n',
+  );
+  const topExtRows = byExtensionArr
+    .slice(0, 20)
+    .map((e) => [
+      e.ext,
+      String(e.count),
+      formatSize(e.bytes),
+      `${toPct(e.bytes, totalBytes).toFixed(2)}%`,
+    ]);
+  md.push(
+    mdTable(topExtRows, ['Ext', 'Count', 'Bytes', '% of total']),
+    '\n\n### Top Directories by Bytes (Top 20)\n',
+  );
+  const topDirRows = byDirectoryArr
+    .slice(0, 20)
+    .map((d) => [
+      d.dir,
+      String(d.count),
+      formatSize(d.bytes),
+      `${toPct(d.bytes, totalBytes).toFixed(2)}%`,
+    ]);
+  md.push(mdTable(topDirRows, ['Directory', 'Files', 'Bytes', '% of total']));
+  return md.join('\n');
+}
+
+module.exports = {
+  KB,
+  MB,
+  formatSize,
+  percentile,
+  processWithLimit,
+  enrichAllFiles,
+  buildHistogram,
+  aggregateByExtension,
+  aggregateByDirectory,
+  computeDepthAndLongest,
+  computeTemporal,
+  computeQuality,
+  computeDuplicates,
+  estimateCompressibility,
+  computeGitInfo,
+  computeLargestFiles,
+  buildMarkdownReport,
+};