@rely-ai/caliber 1.20.0-dev.1773691540 → 1.20.0-dev.1773694290

package/dist/bin.js

@@ -228,7 +228,7 @@ import { fileURLToPath } from "url";
 // src/commands/init.ts
 import path20 from "path";
 import chalk11 from "chalk";
-import ora2 from "ora";
+import ora3 from "ora";
 import select5 from "@inquirer/select";
 import checkbox from "@inquirer/checkbox";
 import fs25 from "fs";
@@ -4362,7 +4362,7 @@ function checkExistence(dir) {
 import { join as join4 } from "path";
 
 // src/scoring/utils.ts
-import { readFileSync as readFileSync3, readdirSync as readdirSync2 } from "fs";
+import { existsSync as existsSync3, readFileSync as readFileSync3, readdirSync as readdirSync2 } from "fs";
 import { join as join3, relative } from "path";
 function readFileOrNull2(filePath) {
   try {
@@ -4545,6 +4545,58 @@ function extractReferences(content) {
   }
   return Array.from(refs);
 }
+function validateFileReferences(content, dir, checkExists = existsSync3) {
+  const refs = extractReferences(content);
+  const valid = [];
+  const invalid = [];
+  for (const ref of refs) {
+    if (/^https?:\/\//.test(ref)) continue;
+    if (/^\d+\.\d+/.test(ref)) continue;
+    if (ref.startsWith("#") || ref.startsWith("@")) continue;
+    if (ref.includes("*") || ref.includes("..")) continue;
+    if (!ref.includes("/") && !ref.includes(".")) continue;
+    const fullPath = join3(dir, ref);
+    if (checkExists(fullPath)) {
+      valid.push(ref);
+    } else {
+      const withoutTrailing = ref.replace(/\/+$/, "");
+      if (withoutTrailing !== ref && checkExists(join3(dir, withoutTrailing))) {
+        valid.push(ref);
+      } else {
+        invalid.push(ref);
+      }
+    }
+  }
+  return { valid, invalid, total: valid.length + invalid.length };
+}
+function countConcreteness(content) {
+  let concrete = 0;
+  let abstract = 0;
+  let inCodeBlock = false;
+  for (const line of content.split("\n")) {
+    if (line.trim().startsWith("```")) {
+      inCodeBlock = !inCodeBlock;
+      continue;
+    }
+    const classification = classifyLine(line, inCodeBlock);
+    if (classification === "concrete") concrete++;
+    else if (classification === "abstract") abstract++;
+  }
+  return { concrete, abstract };
+}
+function countTreeLines(content) {
+  const treeLinePattern = /[├└│─┬]/;
+  let count = 0;
+  let inCodeBlock = false;
+  for (const line of content.split("\n")) {
+    if (line.trim().startsWith("```")) {
+      inCodeBlock = !inCodeBlock;
+      continue;
+    }
+    if (inCodeBlock && treeLinePattern.test(line)) count++;
+  }
+  return count;
+}
 function classifyLine(line, inCodeBlock) {
   if (inCodeBlock) return "concrete";
   const trimmed = line.trim();
@@ -4604,25 +4656,17 @@ function checkQuality(dir) {
       instruction: `Reduce total config from ~${totalTokens} tokens to under 5000.`
     } : void 0
   });
-  let concreteCount = 0;
-  let abstractCount = 0;
+  const { concrete: concreteCount, abstract: abstractCount } = primaryInstructions ? countConcreteness(primaryInstructions) : { concrete: 0, abstract: 0 };
   const abstractExamples = [];
-  if (primaryInstructions) {
-    let inCodeBlock2 = false;
+  if (primaryInstructions && abstractCount > 0) {
+    let inCb = false;
     for (const line of primaryInstructions.split("\n")) {
       if (line.trim().startsWith("```")) {
-        inCodeBlock2 = !inCodeBlock2;
+        inCb = !inCb;
         continue;
       }
-      const classification = classifyLine(line, inCodeBlock2);
-      if (classification === "neutral") continue;
-      if (classification === "concrete") {
-        concreteCount++;
-      } else {
-        abstractCount++;
-        if (abstractExamples.length < 3) {
-          abstractExamples.push(line.trim().slice(0, 80));
-        }
+      if (!inCb && classifyLine(line, false) === "abstract" && abstractExamples.length < 3) {
+        abstractExamples.push(line.trim().slice(0, 80));
       }
     }
   }
@@ -4645,20 +4689,7 @@ function checkQuality(dir) {
       instruction: `Replace generic prose with specific references. Examples of vague lines: ${abstractExamples.join("; ")}`
     } : void 0
   });
-  const treeLinePattern = /[├└│─┬]/;
-  let treeLineCount = 0;
-  let inCodeBlock = false;
-  if (combinedContent) {
-    for (const line of combinedContent.split("\n")) {
-      if (line.trim().startsWith("```")) {
-        inCodeBlock = !inCodeBlock;
-        continue;
-      }
-      if (inCodeBlock && treeLinePattern.test(line)) {
-        treeLineCount++;
-      }
-    }
-  }
+  const treeLineCount = combinedContent ? countTreeLines(combinedContent) : 0;
   const hasLargeTree = treeLineCount > 10;
   checks.push({
     id: "no_directory_tree",
@@ -4823,30 +4854,7 @@ import { join as join5 } from "path";
 function validateReferences(dir) {
   const configContent = collectPrimaryConfigContent(dir);
   if (!configContent) return { valid: [], invalid: [], total: 0 };
-  const refs = extractReferences(configContent);
-  const valid = [];
-  const invalid = [];
-  for (const ref of refs) {
-    if (/^https?:\/\//.test(ref)) continue;
-    if (/^\d+\.\d+/.test(ref)) continue;
-    if (ref.startsWith("#")) continue;
-    if (ref.startsWith("@")) continue;
-    if (ref.includes("*")) continue;
-    if (ref.includes("..")) continue;
-    if (!ref.includes("/") && !ref.includes(".")) continue;
-    const fullPath = join5(dir, ref);
-    if (existsSync4(fullPath)) {
-      valid.push(ref);
-    } else {
-      const withoutTrailing = ref.replace(/\/+$/, "");
-      if (withoutTrailing !== ref && existsSync4(join5(dir, withoutTrailing))) {
-        valid.push(ref);
-      } else {
-        invalid.push(ref);
-      }
-    }
-  }
-  return { valid, invalid, total: valid.length + invalid.length };
+  return validateFileReferences(configContent, dir);
 }
 function detectGitDrift(dir) {
   try {
@@ -6428,6 +6436,184 @@ function printSkills(recs) {
   console.log("");
 }
 
+// src/ai/score-refine.ts
+import { existsSync as existsSync9 } from "fs";
+import ora2 from "ora";
+var MAX_REFINE_ITERATIONS = 2;
+function extractConfigContent(setup) {
+  const claude = setup.claude;
+  const codex = setup.codex;
+  return {
+    claudeMd: claude?.claudeMd ?? null,
+    agentsMd: codex?.agentsMd ?? null
+  };
+}
+function validateSetup(setup, dir, checkExists = existsSync9) {
+  const issues = [];
+  const { claudeMd, agentsMd } = extractConfigContent(setup);
+  const primaryContent = [claudeMd, agentsMd].filter(Boolean).join("\n");
+  if (!primaryContent) return issues;
+  const refs = validateFileReferences(primaryContent, dir, checkExists);
+  if (refs.invalid.length > 0 && refs.total > 0) {
+    const ratio = refs.valid.length / refs.total;
+    const earnedPoints = Math.round(ratio * POINTS_REFERENCES_VALID);
+    const lost = POINTS_REFERENCES_VALID - earnedPoints;
+    if (lost > 0) {
+      issues.push({
+        check: "References valid",
+        detail: `${refs.valid.length}/${refs.total} references verified, ${refs.invalid.length} invalid`,
+        fixInstruction: `Remove these non-existent paths from the config: ${refs.invalid.map((r) => `\`${r}\``).join(", ")}. Do NOT guess replacements \u2014 just delete them.`,
+        pointsLost: lost
+      });
+    }
+  }
+  const totalTokens = estimateTokens2(primaryContent);
+  const tokenThreshold = TOKEN_BUDGET_THRESHOLDS.find((t) => totalTokens <= t.maxTokens);
+  const tokenPoints = tokenThreshold?.points ?? 0;
+  const maxTokenPoints = TOKEN_BUDGET_THRESHOLDS[0].points;
+  if (tokenPoints < maxTokenPoints) {
+    issues.push({
+      check: "Token budget",
+      detail: `~${totalTokens} tokens (target: \u2264${TOKEN_BUDGET_THRESHOLDS[0].maxTokens} for full points)`,
+      fixInstruction: `Config is ~${totalTokens} tokens. Remove the least important lines to get under ${TOKEN_BUDGET_THRESHOLDS[0].maxTokens} tokens. Prioritize removing verbose prose over code blocks or path references.`,
+      pointsLost: maxTokenPoints - tokenPoints
+    });
+  }
+  const content = claudeMd ?? agentsMd ?? "";
+  if (content) {
+    const structure = analyzeMarkdownStructure(content);
+    const blockThreshold = CODE_BLOCK_THRESHOLDS.find((t) => structure.codeBlockCount >= t.minBlocks);
+    const blockPoints = blockThreshold?.points ?? 0;
+    const maxBlockPoints = CODE_BLOCK_THRESHOLDS[0].points;
+    if (blockPoints < maxBlockPoints && structure.codeBlockCount < CODE_BLOCK_THRESHOLDS[0].minBlocks) {
+      issues.push({
+        check: "Executable content",
+        detail: `${structure.codeBlockCount} code block${structure.codeBlockCount === 1 ? "" : "s"} (need \u2265${CODE_BLOCK_THRESHOLDS[0].minBlocks} for full points)`,
+        fixInstruction: `Add ${CODE_BLOCK_THRESHOLDS[0].minBlocks - structure.codeBlockCount} more code blocks with actual project commands (build, test, lint, deploy).`,
+        pointsLost: maxBlockPoints - blockPoints
+      });
+    }
+    const { concrete: concreteCount, abstract: abstractCount } = countConcreteness(content);
+    const totalMeaningful = concreteCount + abstractCount;
+    const concreteRatio = totalMeaningful > 0 ? concreteCount / totalMeaningful : 1;
+    const concThreshold = CONCRETENESS_THRESHOLDS.find((t) => concreteRatio >= t.minRatio);
+    const concPoints = totalMeaningful === 0 ? 0 : concThreshold?.points ?? 0;
+    const maxConcPoints = CONCRETENESS_THRESHOLDS[0].points;
+    if (concPoints < maxConcPoints && totalMeaningful > 0 && concreteRatio < CONCRETENESS_THRESHOLDS[0].minRatio) {
+      issues.push({
+        check: "Concrete instructions",
+        detail: `${Math.round(concreteRatio * 100)}% concrete (need \u2265${Math.round(CONCRETENESS_THRESHOLDS[0].minRatio * 100)}%)`,
+        fixInstruction: `${abstractCount} lines are generic prose. Replace vague instructions with specific ones that reference project files, paths, or commands in backticks.`,
+        pointsLost: maxConcPoints - concPoints
+      });
+    }
+    const treeLineCount = countTreeLines(content);
+    if (treeLineCount > 10) {
+      issues.push({
+        check: "No directory tree listings",
+        detail: `${treeLineCount}-line directory tree found in code blocks`,
+        fixInstruction: "Remove directory tree listings from code blocks. Reference key directories inline with backticks instead.",
+        pointsLost: POINTS_NO_DIR_TREE
+      });
+    }
+    if (structure.h2Count < 3 || structure.listItemCount < 3) {
+      const parts = [];
+      if (structure.h2Count < 3) parts.push(`add ${3 - structure.h2Count} more ## sections`);
+      if (structure.listItemCount < 3) parts.push("use bullet lists for multi-item instructions");
+      issues.push({
+        check: "Structured with headings",
+        detail: `${structure.h2Count} sections, ${structure.listItemCount} list items`,
+        fixInstruction: `Improve structure: ${parts.join(" and ")}.`,
+        pointsLost: POINTS_HAS_STRUCTURE - ((structure.h2Count >= 3 ? 1 : 0) + (structure.listItemCount >= 3 ? 1 : 0))
+      });
+    }
+  }
+  return issues.sort((a, b) => b.pointsLost - a.pointsLost);
+}
+function buildFeedbackMessage(issues) {
+  const lines = [
+    "Your generated config has these scoring issues. Fix ONLY these \u2014 do not rewrite, restructure, or make cosmetic changes to anything else:\n"
+  ];
+  for (let i = 0; i < issues.length; i++) {
+    const issue = issues[i];
+    lines.push(`${i + 1}. ${issue.check.toUpperCase()} (-${issue.pointsLost} pts): ${issue.detail}`);
+    lines.push(`   Action: ${issue.fixInstruction}
+`);
+  }
+  lines.push("Return the complete updated AgentSetup JSON with only these fixes applied.");
+  return lines.join("\n");
+}
+function countIssuePoints(issues) {
+  return issues.reduce((sum, i) => sum + i.pointsLost, 0);
+}
+async function scoreAndRefine(setup, dir, sessionHistory, callbacks) {
+  const existsCache = /* @__PURE__ */ new Map();
+  const cachedExists = (path29) => {
+    const cached = existsCache.get(path29);
+    if (cached !== void 0) return cached;
+    const result = existsSync9(path29);
+    existsCache.set(path29, result);
+    return result;
+  };
+  let currentSetup = setup;
+  let bestSetup = setup;
+  let bestLostPoints = Infinity;
+  for (let iteration = 0; iteration < MAX_REFINE_ITERATIONS; iteration++) {
+    const issues = validateSetup(currentSetup, dir, cachedExists);
+    const lostPoints = countIssuePoints(issues);
+    if (lostPoints < bestLostPoints) {
+      bestSetup = currentSetup;
+      bestLostPoints = lostPoints;
+    }
+    if (issues.length === 0) {
+      if (callbacks?.onStatus) callbacks.onStatus("Setup passes all scoring checks");
+      return bestSetup;
+    }
+    if (callbacks?.onStatus) {
+      const issueNames = issues.map((i) => i.check).join(", ");
+      callbacks.onStatus(`Fixing ${issues.length} scoring issue${issues.length === 1 ? "" : "s"}: ${issueNames}...`);
+    }
+    const feedbackMessage = buildFeedbackMessage(issues);
+    const refined = await refineSetup(currentSetup, feedbackMessage, sessionHistory);
+    if (!refined) {
+      if (callbacks?.onStatus) callbacks.onStatus("Refinement failed, keeping current setup");
+      return bestSetup;
+    }
+    sessionHistory.push({ role: "user", content: feedbackMessage });
+    sessionHistory.push({
+      role: "assistant",
+      content: `Applied scoring fixes for: ${issues.map((i) => i.check).join(", ")}`
+    });
+    currentSetup = refined;
+  }
+  const finalIssues = validateSetup(currentSetup, dir, cachedExists);
+  const finalLostPoints = countIssuePoints(finalIssues);
+  if (finalLostPoints < bestLostPoints) {
+    bestSetup = currentSetup;
+  }
+  return bestSetup;
+}
+async function runScoreRefineWithSpinner(setup, dir, sessionHistory) {
+  const spinner = ora2("Validating setup against scoring criteria...").start();
+  try {
+    const refined = await scoreAndRefine(setup, dir, sessionHistory, {
+      onStatus: (msg) => {
+        spinner.text = msg;
+      }
+    });
+    if (refined !== setup) {
+      spinner.succeed("Setup refined based on scoring feedback");
+    } else {
+      spinner.succeed("Setup passes scoring validation");
+    }
+    return refined;
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : "unknown error";
+    spinner.warn(`Scoring validation skipped: ${msg}`);
+    return setup;
+  }
+}
+
 // src/lib/debug-report.ts
 import fs24 from "fs";
 import path19 from "path";
@@ -7074,12 +7260,12 @@ async function initCommand(options) {
     report.addJson("Generation: Parsed Setup", generatedSetup);
   }
   log(options.verbose, `Generation completed: ${elapsedMs}ms, stopReason: ${genStopReason || "end_turn"}`);
-  printSetupSummary(generatedSetup);
   const sessionHistory = [];
   sessionHistory.push({
     role: "assistant",
     content: summarizeSetup("Initial generation", generatedSetup)
   });
+  generatedSetup = await runScoreRefineWithSpinner(generatedSetup, process.cwd(), sessionHistory);
   console.log(title.bold("  Step 3/4 \u2014 Review\n"));
   const setupFiles = collectSetupFiles(generatedSetup, targetAgent);
   const staged = stageFiles(setupFiles, process.cwd());
@@ -7149,7 +7335,7 @@ async function initCommand(options) {
     console.log(JSON.stringify(generatedSetup, null, 2));
     return;
   }
-  const writeSpinner = ora2("Writing config files...").start();
+  const writeSpinner = ora3("Writing config files...").start();
   try {
     if (targetAgent.includes("codex") && !fs25.existsSync("AGENTS.md") && !generatedSetup.codex) {
       const claude = generatedSetup.claude;
@@ -7314,7 +7500,7 @@ async function refineLoop(currentSetup, _targetAgent, sessionHistory) {
       console.log(chalk11.dim('  Type "done" to accept the current setup.\n'));
       continue;
     }
-    const refineSpinner = ora2("Refining setup...").start();
+    const refineSpinner = ora3("Refining setup...").start();
     const refineMessages = new SpinnerMessages(refineSpinner, REFINE_MESSAGES);
     refineMessages.start();
     const refined = await refineSetup(
@@ -7640,9 +7826,9 @@ function writeErrorLog(config, rawOutput, error, stopReason) {
 
 // src/commands/undo.ts
 import chalk12 from "chalk";
-import ora3 from "ora";
+import ora4 from "ora";
 function undoCommand() {
-  const spinner = ora3("Reverting setup...").start();
+  const spinner = ora4("Reverting setup...").start();
   try {
     const { restored, removed } = undoSetup();
     if (restored.length === 0 && removed.length === 0) {
@@ -7708,7 +7894,7 @@ async function statusCommand(options) {
 
 // src/commands/regenerate.ts
 import chalk14 from "chalk";
-import ora4 from "ora";
+import ora5 from "ora";
 import select6 from "@inquirer/select";
 init_config();
 async function regenerateCommand(options) {
@@ -7724,7 +7910,7 @@ async function regenerateCommand(options) {
   }
   const targetAgent = readState()?.targetAgent ?? ["claude", "cursor"];
   await validateModel({ fast: true });
-  const spinner = ora4("Analyzing project...").start();
+  const spinner = ora5("Analyzing project...").start();
   const fingerprint = await collectFingerprint(process.cwd());
   spinner.succeed("Project analyzed");
   const baselineScore = computeLocalScore(process.cwd(), targetAgent);
@@ -7733,7 +7919,7 @@ async function regenerateCommand(options) {
     console.log(chalk14.green("  Your setup is already at 100/100 \u2014 nothing to regenerate.\n"));
     return;
   }
-  const genSpinner = ora4("Regenerating setup...").start();
+  const genSpinner = ora5("Regenerating setup...").start();
   const genMessages = new SpinnerMessages(genSpinner, GENERATION_MESSAGES, { showElapsedTime: true });
   genMessages.start();
   let generatedSetup = null;
@@ -7768,6 +7954,7 @@ async function regenerateCommand(options) {
     throw new Error("__exit__");
   }
   genSpinner.succeed("Setup regenerated");
+  generatedSetup = await runScoreRefineWithSpinner(generatedSetup, process.cwd(), []);
   const setupFiles = collectSetupFiles(generatedSetup, targetAgent);
   const staged = stageFiles(setupFiles, process.cwd());
   const totalChanges = staged.newFiles + staged.modifiedFiles;
@@ -7804,7 +7991,7 @@ async function regenerateCommand(options) {
     console.log(chalk14.dim("Regeneration cancelled. No files were modified."));
     return;
   }
-  const writeSpinner = ora4("Writing config files...").start();
+  const writeSpinner = ora5("Writing config files...").start();
   try {
     const result = writeSetup(generatedSetup);
     writeSpinner.succeed("Config files written");
@@ -7883,7 +8070,7 @@ async function scoreCommand(options) {
 import fs30 from "fs";
 import path24 from "path";
 import chalk16 from "chalk";
-import ora5 from "ora";
+import ora6 from "ora";
 
 // src/lib/git-diff.ts
 import { execSync as execSync13 } from "child_process";
@@ -8237,7 +8424,7 @@ async function refreshSingleRepo(repoDir, options) {
     log2(quiet, chalk16.dim(`${prefix}No changes since last refresh.`));
     return;
   }
-  const spinner = quiet ? null : ora5(`${prefix}Analyzing changes...`).start();
+  const spinner = quiet ? null : ora6(`${prefix}Analyzing changes...`).start();
   const existingDocs = readExistingConfigs(repoDir);
   const learnedSection = readLearnedSection();
   const fingerprint = await collectFingerprint(repoDir);
@@ -9269,7 +9456,7 @@ import path28 from "path";
 import { fileURLToPath as fileURLToPath2 } from "url";
 import { execSync as execSync14 } from "child_process";
 import chalk20 from "chalk";
-import ora6 from "ora";
+import ora7 from "ora";
 import confirm2 from "@inquirer/confirm";
 var __dirname_vc = path28.dirname(fileURLToPath2(import.meta.url));
 var pkg2 = JSON.parse(
@@ -9343,7 +9530,7 @@ Update available: ${current} -> ${latest}`)
       return;
     }
     const tag = channel === "latest" ? latest : channel;
-    const spinner = ora6("Updating caliber...").start();
+    const spinner = ora7("Updating caliber...").start();
     try {
       execSync14(`npm install -g @rely-ai/caliber@${tag}`, {
         stdio: "pipe",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rely-ai/caliber",
-  "version": "1.20.0-dev.1773691540",
+  "version": "1.20.0-dev.1773694290",
   "description": "Analyze your codebase and generate optimized AI agent configs (CLAUDE.md, .cursorrules, skills) — no API key needed",
   "type": "module",
   "bin": {