avorelo 0.1.0

package/scripts/lib/context-budget-scanner.js

@@ -0,0 +1,257 @@
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+
+const SKIP_DIRS = new Set([
+  ".git", "node_modules", "dist", "build", "coverage", ".next",
+  "vendor", "target", "tmp", "temp", "out", ".turbo", ".cache",
+]);
+
+const SENSITIVE_NAMES = [
+  ".env", "credentials", "secret", "private_key", "privatekey",
+  "api_key", "apikey", "api-key", "token", "password", "passwd",
+  "auth_token", "authtoken", "access_token",
+];
+
+const LOCKFILE_NAMES = new Set([
+  "package-lock.json", "yarn.lock", "pnpm-lock.yaml",
+  "composer.lock", "Gemfile.lock", "poetry.lock",
+  "Cargo.lock", "go.sum",
+]);
+
+const BINARY_EXTS = new Set([
+  ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".ico", ".webp",
+  ".woff", ".woff2", ".ttf", ".eot", ".otf",
+  ".mp3", ".mp4", ".wav", ".avi", ".mov", ".mkv", ".webm",
+  ".zip", ".tar", ".gz", ".7z", ".rar", ".tgz",
+  ".pdf", ".exe", ".dll", ".so", ".dylib", ".bin", ".dat",
+  ".db", ".sqlite", ".sqlite3",
+  ".class", ".jar", ".war", ".ear",
+  ".pyc", ".pyo",
+]);
+
+const SOURCE_EXTS = new Set([
+  ".js", ".ts", ".jsx", ".tsx", ".mjs", ".cjs",
+  ".py", ".rb", ".go", ".rs", ".java",
+  ".c", ".cpp", ".cc", ".h", ".hpp", ".hh",
+  ".cs", ".php", ".swift", ".kt", ".scala",
+  ".sh", ".bash", ".zsh", ".fish", ".ps1",
+  ".vue", ".svelte",
+]);
+
+const CONFIG_EXTS = new Set([
+  ".json", ".yaml", ".yml", ".toml", ".ini",
+  ".cfg", ".conf", ".xml", ".env",
+]);
+
+const DOC_EXTS = new Set([
+  ".md", ".txt", ".rst", ".adoc", ".asciidoc", ".rtf",
+]);
+
+const TEST_NAME_PATTERNS = [
+  /\.test\.[jt]sx?$/,
+  /\.spec\.[jt]sx?$/,
+  /\.test\.py$/,
+  /\.spec\.py$/,
+];
+
+const TEST_PATH_PATTERNS = [
+  /(?:^|\/)tests?\//,
+  /(?:^|\/)__tests__\//,
+  /(?:^|\/)test\//,
+  /(?:^|\/)spec\//,
+];
+
+const LARGE_FILE_TOKEN_THRESHOLD = 10000; // ~40 KB
+const LARGE_FOLDER_TOKEN_THRESHOLD = 20000;
+
+function estimateTokens(byteCount) {
+  return Math.ceil(byteCount / 4);
+}
+
+function isSensitivePath(relPath) {
+  const base = path.basename(relPath).toLowerCase();
+  return SENSITIVE_NAMES.some((s) => base.includes(s));
+}
+
+function isLockfile(relPath) {
+  return LOCKFILE_NAMES.has(path.basename(relPath));
+}
+
+function isLogFile(relPath) {
+  const base = path.basename(relPath).toLowerCase();
+  return base.endsWith(".log") || base.endsWith(".log.txt");
+}
+
+function isBinaryExt(relPath) {
+  const ext = path.extname(relPath).toLowerCase();
+  return BINARY_EXTS.has(ext);
+}
+
+function isTestFile(relPath) {
+  const normalized = relPath.replace(/\\/g, "/");
+  return (
+    TEST_NAME_PATTERNS.some((p) => p.test(normalized)) ||
+    TEST_PATH_PATTERNS.some((p) => p.test(normalized))
+  );
+}
+
+function classifyFile(relPath, byteCount, maxFileBytes) {
+  if (isSensitivePath(relPath)) {
+    return { kind: "sensitive", reasonCodes: ["sensitive_context"] };
+  }
+  if (isBinaryExt(relPath)) {
+    return { kind: "binary", reasonCodes: ["binary_or_unknown"] };
+  }
+  if (isLockfile(relPath)) {
+    return { kind: "lockfile", reasonCodes: ["lockfile"] };
+  }
+  if (isLogFile(relPath)) {
+    return { kind: "log", reasonCodes: ["log_file"] };
+  }
+  if (byteCount > maxFileBytes) {
+    return { kind: "too_large", reasonCodes: ["too_large_to_read"] };
+  }
+
+  const ext = path.extname(relPath).toLowerCase();
+  const codes = [];
+
+  if (SOURCE_EXTS.has(ext)) codes.push("source_file");
+  else if (CONFIG_EXTS.has(ext)) codes.push("config_file");
+  else if (DOC_EXTS.has(ext)) codes.push("documentation_file");
+  else codes.push("binary_or_unknown");
+
+  if (isTestFile(relPath)) codes.push("test_file");
+
+  const tokens = estimateTokens(byteCount);
+  if (tokens >= LARGE_FILE_TOKEN_THRESHOLD) {
+    codes.push("large_file");
+  } else {
+    codes.push("small_enough");
+  }
+
+  return { kind: "regular", reasonCodes: codes };
+}
+
+function getSkipDirInfo(dirName) {
+  if (dirName === "node_modules" || dirName === "vendor") {
+    return { reasonCodes: ["dependency_folder"], recommendation: "skip" };
+  }
+  return { reasonCodes: ["generated_or_build_artifact"], recommendation: "ignore_generated" };
+}
+
+function scanRepo(cwd, config = {}) {
+  const maxFilesScanned = Number.isFinite(Number(config.maxFilesScanned))
+    ? Math.max(1, Number(config.maxFilesScanned)) : 2000;
+  const maxFileBytes = Number.isFinite(Number(config.maxFileBytes))
+    ? Math.max(1024, Number(config.maxFileBytes)) : 1000000;
+  const maxDepth = Number.isFinite(Number(config.maxDepth))
+    ? Math.max(1, Number(config.maxDepth)) : 12;
+
+  const fileEntries = [];
+  const skippedDirs = [];
+  let scannedFiles = 0;
+  let skippedFiles = 0;
+
+  function walk(dir, depth) {
+    if (depth > maxDepth) return;
+    if (scannedFiles >= maxFilesScanned) return;
+
+    let entries;
+    try {
+      entries = fs.readdirSync(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+
+    for (const entry of entries) {
+      if (scannedFiles >= maxFilesScanned) break;
+
+      const absPath = path.join(dir, entry.name);
+      const relPath = path.relative(cwd, absPath).replace(/\\/g, "/");
+
+      if (entry.isDirectory()) {
+        if (SKIP_DIRS.has(entry.name)) {
+          const info = getSkipDirInfo(entry.name);
+          skippedDirs.push({
+            path: relPath,
+            type: "folder",
+            estimatedTokens: 0,
+            reasonCodes: info.reasonCodes,
+            recommendation: info.recommendation,
+          });
+          skippedFiles++;
+          continue;
+        }
+        walk(absPath, depth + 1);
+        continue;
+      }
+
+      if (!entry.isFile()) continue;
+
+      let byteCount = 0;
+      try {
+        byteCount = fs.statSync(absPath).size;
+      } catch {
+        skippedFiles++;
+        continue;
+      }
+
+      const classification = classifyFile(relPath, byteCount, maxFileBytes);
+      const estimatedTokens = estimateTokens(byteCount);
+
+      fileEntries.push({
+        path: relPath,
+        type: "file",
+        byteCount,
+        estimatedTokens,
+        reasonCodes: classification.reasonCodes,
+        kind: classification.kind,
+      });
+      scannedFiles++;
+    }
+  }
+
+  walk(cwd, 0);
+
+  const folderAggregates = {};
+  for (const f of fileEntries) {
+    const parts = f.path.split("/");
+    for (let i = 1; i < parts.length; i++) {
+      const folderPath = parts.slice(0, i).join("/");
+      if (!folderAggregates[folderPath]) {
+        folderAggregates[folderPath] = { path: folderPath, estimatedTokens: 0, fileCount: 0 };
+      }
+      folderAggregates[folderPath].estimatedTokens += f.estimatedTokens;
+      folderAggregates[folderPath].fileCount++;
+    }
+  }
+
+  const largeFolders = Object.values(folderAggregates)
+    .filter((f) => f.estimatedTokens >= LARGE_FOLDER_TOKEN_THRESHOLD)
+    .sort((a, b) => b.estimatedTokens - a.estimatedTokens);
+
+  return {
+    fileEntries,
+    skippedDirs,
+    folderAggregates,
+    largeFolders,
+    scannedFiles,
+    skippedFiles: skippedFiles + skippedDirs.length,
+  };
+}
+
+module.exports = {
+  scanRepo,
+  estimateTokens,
+  classifyFile,
+  isSensitivePath,
+  isLockfile,
+  isLogFile,
+  isTestFile,
+  isBinaryExt,
+  SKIP_DIRS,
+  LARGE_FILE_TOKEN_THRESHOLD,
+  LARGE_FOLDER_TOKEN_THRESHOLD,
+};