@rely-ai/caliber 1.15.3 → 1.17.0

package/dist/bin.js

@@ -503,47 +503,187 @@ var SKIP_PATTERNS = [
   /\.generated\./,
   /\.snap$/
 ];
+var COMMENT_LINE_PATTERNS = {
+  "c-style": /^\s*\/\//,
+  // ts, js, go, rs, java, etc.
+  "hash": /^\s*#/,
+  // py, sh, yaml, tf, etc.
+  "html": /^\s*<!--.*-->\s*$/
+};
+var EXT_TO_COMMENT_STYLE = {
+  ".ts": "c-style",
+  ".tsx": "c-style",
+  ".js": "c-style",
+  ".jsx": "c-style",
+  ".mjs": "c-style",
+  ".cjs": "c-style",
+  ".go": "c-style",
+  ".rs": "c-style",
+  ".java": "c-style",
+  ".kt": "c-style",
+  ".scala": "c-style",
+  ".cs": "c-style",
+  ".c": "c-style",
+  ".cpp": "c-style",
+  ".h": "c-style",
+  ".hpp": "c-style",
+  ".swift": "c-style",
+  ".php": "c-style",
+  ".py": "hash",
+  ".pyw": "hash",
+  ".rb": "hash",
+  ".sh": "hash",
+  ".bash": "hash",
+  ".zsh": "hash",
+  ".fish": "hash",
+  ".r": "hash",
+  ".tf": "hash",
+  ".hcl": "hash",
+  ".yaml": "hash",
+  ".yml": "hash",
+  ".toml": "hash",
+  ".ini": "hash",
+  ".cfg": "hash",
+  ".env": "hash",
+  ".html": "html",
+  ".xml": "html",
+  ".svg": "html",
+  ".vue": "html",
+  ".svelte": "html"
+};
 var TOKEN_BUDGET = 18e4;
 var CHAR_BUDGET = TOKEN_BUDGET * 4;
+function compressContent(content, ext) {
+  const commentStyle = EXT_TO_COMMENT_STYLE[ext];
+  const commentPattern = commentStyle ? COMMENT_LINE_PATTERNS[commentStyle] : null;
+  const lines = content.split("\n");
+  const result = [];
+  let prevBlank = false;
+  let inBlockComment = false;
+  for (const line of lines) {
+    const trimmed = line.trimEnd();
+    if (!inBlockComment && /^\s*\/\*/.test(trimmed) && !trimmed.includes("*/")) {
+      inBlockComment = true;
+      continue;
+    }
+    if (inBlockComment) {
+      if (trimmed.includes("*/")) inBlockComment = false;
+      continue;
+    }
+    if (/^\s*\/\*.*\*\/\s*$/.test(trimmed)) continue;
+    if (trimmed.length === 0) {
+      if (!prevBlank) result.push("");
+      prevBlank = true;
+      continue;
+    }
+    prevBlank = false;
+    if (commentPattern && commentPattern.test(trimmed)) continue;
+    const leadingMatch = line.match(/^(\s*)/);
+    if (leadingMatch) {
+      const spaces = leadingMatch[1].replace(/\t/g, "    ").length;
+      const normalizedIndent = " ".repeat(Math.floor(spaces / 2) * 2);
+      result.push(normalizedIndent + line.trimStart().trimEnd());
+    } else {
+      result.push(trimmed);
+    }
+  }
+  while (result.length > 0 && result[result.length - 1] === "") result.pop();
+  return result.join("\n");
+}
+function structuralFingerprint(content, ext) {
+  const lines = content.split("\n").filter((l) => l.trim().length > 0);
+  const lineCount = lines.length;
+  const sizeBucket = Math.floor(lineCount / 10) * 10;
+  const firstLine = lines[0]?.trim().slice(0, 60) || "";
+  const imports = lines.filter((l) => /^\s*(import |from |require\(|use )/.test(l)).length;
+  const exports = lines.filter((l) => /^\s*export /.test(l)).length;
+  const functions = lines.filter((l) => /^\s*(function |def |func |fn |pub fn )/.test(l)).length;
+  return `${ext}:${sizeBucket}:${imports}:${exports}:${functions}:${firstLine}`;
+}
+function deduplicateFiles(files) {
+  const groups = /* @__PURE__ */ new Map();
+  for (const f of files) {
+    const fp = structuralFingerprint(f.content, f.ext);
+    const group = groups.get(fp) || [];
+    group.push({ path: f.path, content: f.content });
+    groups.set(fp, group);
+  }
+  const result = [];
+  for (const [, group] of groups) {
+    const representative = group[0];
+    result.push({
+      path: representative.path,
+      content: representative.content,
+      size: representative.content.length
+    });
+    if (group.length > 1) {
+      const similarPaths = group.slice(1).map((f) => f.path);
+      const summary = `(${similarPaths.length} similar file${similarPaths.length === 1 ? "" : "s"}: ${similarPaths.join(", ")})`;
+      result.push({
+        path: `[similar to ${representative.path}]`,
+        content: summary,
+        size: summary.length
+      });
+    }
+  }
+  return result;
+}
 function analyzeCode(dir) {
   const allFiles = [];
   walkDir(dir, "", 0, 10, allFiles);
   sortByPriority(allFiles);
   let totalChars = 0;
-  let includedChars = 0;
-  let truncated = false;
-  const files = [];
-  const fileSizes = /* @__PURE__ */ new Map();
   for (const relPath of allFiles) {
     try {
       const stat = fs3.statSync(path3.join(dir, relPath));
-      fileSizes.set(relPath, stat.size);
       totalChars += stat.size;
     } catch {
     }
   }
+  const readFiles = [];
+  let compressedChars = 0;
   for (const relPath of allFiles) {
     const fullPath = path3.join(dir, relPath);
-    let content;
+    let rawContent;
     try {
-      content = fs3.readFileSync(fullPath, "utf-8");
+      rawContent = fs3.readFileSync(fullPath, "utf-8");
     } catch {
       continue;
     }
-    if (content.split("\n").length > 500) continue;
-    const entrySize = relPath.length + content.length + 10;
+    if (rawContent.split("\n").length > 500) continue;
+    const ext = path3.extname(relPath).toLowerCase();
+    const compressed = compressContent(rawContent, ext);
+    compressedChars += compressed.length;
+    readFiles.push({
+      path: relPath,
+      content: compressed,
+      ext,
+      rawSize: rawContent.length
+    });
+  }
+  const deduped = deduplicateFiles(readFiles);
+  const dupGroups = deduped.filter((f) => f.path.startsWith("[similar")).length;
+  let includedChars = 0;
+  let truncated = false;
+  const files = [];
+  for (const file of deduped) {
+    const entrySize = file.path.length + file.content.length + 10;
     if (includedChars + entrySize > CHAR_BUDGET) {
       truncated = true;
       continue;
     }
-    files.push({ path: relPath, content, size: content.length });
+    files.push(file);
     includedChars += entrySize;
   }
   return {
     files,
     truncated,
     totalProjectTokens: Math.ceil(totalChars / 4),
-    includedTokens: Math.ceil(includedChars / 4)
+    compressedTokens: Math.ceil(compressedChars / 4),
+    includedTokens: Math.ceil(includedChars / 4),
+    filesAnalyzed: readFiles.length,
+    filesIncluded: files.length,
+    duplicateGroups: dupGroups
   };
 }
 function walkDir(base, rel, depth, maxDepth, files) {
@@ -6036,15 +6176,25 @@ async function initCommand(options) {
   spinner.succeed("Project analyzed");
   log(options.verbose, `Fingerprint: ${fingerprint.languages.length} languages, ${fingerprint.frameworks.length} frameworks, ${fingerprint.fileTree.length} files`);
   if (options.verbose && fingerprint.codeAnalysis) {
-    log(options.verbose, `Code analysis: ${fingerprint.codeAnalysis.files.length} files, ~${fingerprint.codeAnalysis.includedTokens.toLocaleString()} tokens${fingerprint.codeAnalysis.truncated ? ` (trimmed from ~${fingerprint.codeAnalysis.totalProjectTokens.toLocaleString()})` : ""}`);
+    log(options.verbose, `Code analysis: ${fingerprint.codeAnalysis.filesIncluded}/${fingerprint.codeAnalysis.filesAnalyzed} files, ~${fingerprint.codeAnalysis.includedTokens.toLocaleString()} tokens, ${fingerprint.codeAnalysis.duplicateGroups} dedup groups`);
   }
   trackInitProjectDiscovered(fingerprint.languages.length, fingerprint.frameworks.length, fingerprint.fileTree.length);
   console.log(chalk8.dim(`  Languages: ${fingerprint.languages.join(", ") || "none detected"}`));
   console.log(chalk8.dim(`  Files: ${fingerprint.fileTree.length} found`));
   if (fingerprint.codeAnalysis) {
     const ca = fingerprint.codeAnalysis;
-    const contextInfo = ca.truncated ? `Context: ~${ca.includedTokens.toLocaleString()} tokens (${Math.round(ca.includedTokens / ca.totalProjectTokens * 100)}% of ${ca.totalProjectTokens.toLocaleString()} total)` : `Context: ~${ca.includedTokens.toLocaleString()} tokens`;
-    console.log(chalk8.dim(`  ${contextInfo}`));
+    const compressionPct = ca.totalProjectTokens > 0 ? Math.round((1 - ca.compressedTokens / ca.totalProjectTokens) * 100) : 0;
+    const parts = [`Context: ~${ca.includedTokens.toLocaleString()} tokens sent`];
+    if (ca.truncated) {
+      parts.push(`(${Math.round(ca.includedTokens / ca.totalProjectTokens * 100)}% of ${ca.totalProjectTokens.toLocaleString()} total)`);
+    }
+    if (compressionPct > 5) {
+      parts.push(`compressed ${compressionPct}%`);
+    }
+    if (ca.duplicateGroups > 0) {
+      parts.push(`${ca.duplicateGroups} duplicate group${ca.duplicateGroups === 1 ? "" : "s"} merged`);
+    }
+    console.log(chalk8.dim(`  ${parts.join(" \xB7 ")}`));
   }
   console.log("");
   if (report) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rely-ai/caliber",
-  "version": "1.15.3",
+  "version": "1.17.0",
   "description": "Analyze your codebase and generate optimized AI agent configs (CLAUDE.md, .cursorrules, skills) — no API key needed",
   "type": "module",
   "bin": {