@rely-ai/caliber 1.20.0-dev.1773696964 → 1.20.0-dev.1773698265

package/dist/bin.js

@@ -5827,18 +5827,21 @@ function trackLearnSessionAnalyzed(props) {
     had_learnings_available: props.hadLearningsAvailable,
     learnings_available_count: props.learningsAvailableCount,
     new_learnings_produced: props.newLearningsProduced,
-    waste_tokens: props.wasteTokens
+    waste_tokens: props.wasteTokens,
+    waste_seconds: props.wasteSeconds
   });
 }
 function trackLearnROISnapshot(props) {
   trackEvent("learn_roi_snapshot", {
     total_waste_tokens: props.totalWasteTokens,
+    total_waste_seconds: props.totalWasteSeconds,
     total_sessions: props.totalSessions,
     sessions_with_learnings: props.sessionsWithLearnings,
     sessions_without_learnings: props.sessionsWithoutLearnings,
     failure_rate_with_learnings: props.failureRateWithLearnings,
     failure_rate_without_learnings: props.failureRateWithoutLearnings,
     estimated_savings_tokens: props.estimatedSavingsTokens,
+    estimated_savings_seconds: props.estimatedSavingsSeconds,
     learning_count: props.learningCount
   });
 }
@@ -6535,7 +6538,7 @@ function validateSetup(setup, dir, checkExists = existsSync9) {
 }
 function buildFeedbackMessage(issues) {
   const lines = [
-    "Your generated config has these scoring issues. Fix ONLY these \u2014 do not rewrite, restructure, or make cosmetic changes to anything else:\n"
+    "Fix ONLY these scoring issues \u2014 do not rewrite, restructure, or make cosmetic changes:\n"
   ];
   for (let i = 0; i < issues.length; i++) {
     const issue = issues[i];
@@ -6543,7 +6546,6 @@ function buildFeedbackMessage(issues) {
     lines.push(`   Action: ${issue.fixInstruction}
 `);
   }
-  lines.push("Return the complete updated AgentSetup JSON with only these fixes applied.");
   return lines.join("\n");
 }
 function countIssuePoints(issues) {
@@ -6576,12 +6578,20 @@ async function scoreAndRefine(setup, dir, sessionHistory, callbacks) {
       const issueNames = issues.map((i) => i.check).join(", ");
       callbacks.onStatus(`Fixing ${issues.length} scoring issue${issues.length === 1 ? "" : "s"}: ${issueNames}...`);
     }
-    const patched = await applyTargetedFixes(currentSetup, issues, sessionHistory);
-    if (!patched) {
+    const refined = await applyTargetedFixes(currentSetup, issues);
+    if (!refined) {
       if (callbacks?.onStatus) callbacks.onStatus("Refinement failed, keeping current setup");
       return bestSetup;
     }
-    currentSetup = patched;
+    sessionHistory.push({
+      role: "user",
+      content: `Fix scoring issues: ${issues.map((i) => i.check).join(", ")}`
+    });
+    sessionHistory.push({
+      role: "assistant",
+      content: `Applied scoring fixes for: ${issues.map((i) => i.check).join(", ")}`
+    });
+    currentSetup = refined;
   }
   const finalIssues = validateSetup(currentSetup, dir, cachedExists);
   const finalLostPoints = countIssuePoints(finalIssues);
@@ -6590,41 +6600,44 @@ async function scoreAndRefine(setup, dir, sessionHistory, callbacks) {
   }
   return bestSetup;
 }
-async function applyTargetedFixes(setup, issues, sessionHistory) {
+async function applyTargetedFixes(setup, issues) {
   const { claudeMd, agentsMd } = extractConfigContent(setup);
-  const content = claudeMd ?? agentsMd;
-  if (!content) return null;
+  const targets = [];
+  if (claudeMd) targets.push({ key: "claudeMd", label: "CLAUDE.md", content: claudeMd });
+  if (agentsMd) targets.push({ key: "agentsMd", label: "AGENTS.md", content: agentsMd });
+  if (targets.length === 0) return null;
   const feedbackMessage = buildFeedbackMessage(issues);
-  const contentLabel = claudeMd ? "CLAUDE.md" : "AGENTS.md";
+  const contentBlock = targets.map(
+    (t) => `### ${t.label}
+\`\`\`markdown
+${t.content}
+\`\`\``
+  ).join("\n\n");
   const prompt = [
-    `Here is the current ${contentLabel} content:
-`,
-    "```markdown",
-    content,
-    "```\n",
+    "Here are the config files with scoring issues:\n",
+    contentBlock,
+    "\n",
     feedbackMessage,
     `
-Return ONLY the fixed ${contentLabel} content as raw markdown (no JSON, no code fences). Apply the fixes above and nothing else \u2014 do not rewrite, restructure, or make cosmetic changes.`
+Return ONLY the fixed content as a JSON object with keys ${targets.map((t) => `"${t.key}"`).join(", ")}. Each value is the fixed markdown string. No code fences, no explanations.`
   ].join("\n");
   try {
     const raw = await llmCall({
-      system: `You fix scoring issues in AI agent configuration files. Return only the fixed markdown content \u2014 no explanations, no JSON wrappers, no code fences.`,
+      system: "You fix scoring issues in AI agent configuration files. Return only a JSON object with the fixed content \u2014 no explanations, no code fences.",
       prompt,
       maxTokens: 8e3
     });
-    const fixed = stripMarkdownFences(raw).trim();
-    if (!fixed || fixed.length < 50) return null;
-    const patched = JSON.parse(JSON.stringify(setup));
-    if (claudeMd) {
-      patched.claude.claudeMd = fixed;
-    } else {
-      patched.codex.agentsMd = fixed;
+    const cleaned = stripMarkdownFences(raw);
+    const jsonStart = cleaned.indexOf("{");
+    const jsonToParse = jsonStart !== -1 ? cleaned.slice(jsonStart) : cleaned;
+    const fixes = JSON.parse(jsonToParse);
+    const patched = structuredClone(setup);
+    for (const target of targets) {
+      if (typeof fixes[target.key] === "string" && fixes[target.key].length > 50) {
+        const parent = target.key === "claudeMd" ? patched.claude : patched.codex;
+        if (parent) parent[target.key] = fixes[target.key];
+      }
     }
-    sessionHistory.push({ role: "user", content: feedbackMessage });
-    sessionHistory.push({
-      role: "assistant",
-      content: `Applied scoring fixes for: ${issues.map((i) => i.check).join(", ")}`
-    });
     return patched;
   } catch {
     return null;
@@ -9123,20 +9136,28 @@ ${eventsText}`;
 }
 function calculateSessionWaste(events) {
   let totalWasteTokens = 0;
+  let totalWasteSeconds = 0;
   let failureCount = 0;
   let promptCount = 0;
-  for (const event of events) {
+  for (let i = 0; i < events.length; i++) {
+    const event = events[i];
     if (event.hook_event_name === "PostToolUseFailure") {
       const te = event;
       const inputStr = JSON.stringify(te.tool_input);
       const responseStr = typeof te.tool_response === "object" && "_truncated" in te.tool_response ? String(te.tool_response._truncated) : JSON.stringify(te.tool_response);
       totalWasteTokens += estimateTokens(inputStr + responseStr);
       failureCount++;
+      if (i > 0) {
+        const prev = new Date(events[i - 1].timestamp).getTime();
+        const curr = new Date(event.timestamp).getTime();
+        const elapsed = (curr - prev) / 1e3;
+        if (elapsed > 0 && elapsed < 600) totalWasteSeconds += elapsed;
+      }
     } else if (event.hook_event_name === "UserPromptSubmit") {
       promptCount++;
     }
   }
-  return { totalWasteTokens, failureCount, promptCount };
+  return { totalWasteTokens, totalWasteSeconds, failureCount, promptCount };
 }
 
 // src/commands/learn.ts
@@ -9148,11 +9169,13 @@ import fs32 from "fs";
 import path26 from "path";
 var DEFAULT_TOTALS = {
   totalWasteTokens: 0,
+  totalWasteSeconds: 0,
   totalSessionsWithLearnings: 0,
   totalSessionsWithoutLearnings: 0,
   totalFailuresWithLearnings: 0,
   totalFailuresWithoutLearnings: 0,
   estimatedSavingsTokens: 0,
+  estimatedSavingsSeconds: 0,
   firstSessionTimestamp: "",
   lastSessionTimestamp: ""
 };
@@ -9177,11 +9200,13 @@ function writeROIStats(stats) {
 function recalculateTotals(stats) {
   const totals = stats.totals;
   totals.totalWasteTokens = stats.learnings.reduce((sum, l) => sum + l.wasteTokens, 0);
+  totals.totalWasteSeconds = 0;
   totals.totalSessionsWithLearnings = 0;
   totals.totalSessionsWithoutLearnings = 0;
   totals.totalFailuresWithLearnings = 0;
   totals.totalFailuresWithoutLearnings = 0;
   for (const s of stats.sessions) {
+    totals.totalWasteSeconds += s.wasteSeconds || 0;
     if (s.hadLearningsAvailable) {
       totals.totalSessionsWithLearnings++;
       totals.totalFailuresWithLearnings += s.failureCount;
@@ -9191,6 +9216,7 @@ function recalculateTotals(stats) {
     }
   }
   totals.estimatedSavingsTokens = totals.totalWasteTokens * totals.totalSessionsWithLearnings;
+  totals.estimatedSavingsSeconds = totals.totalWasteSeconds * totals.totalSessionsWithLearnings;
   if (stats.sessions.length > 0) {
     totals.firstSessionTimestamp = stats.sessions[0].timestamp;
     totals.lastSessionTimestamp = stats.sessions[stats.sessions.length - 1].timestamp;
@@ -9214,6 +9240,12 @@ function recordSession(summary, learnings) {
   writeROIStats(stats);
   return stats;
 }
+function formatDuration(seconds) {
+  if (seconds < 60) return `${Math.round(seconds)}s`;
+  const mins = Math.floor(seconds / 60);
+  const secs = Math.round(seconds % 60);
+  return secs > 0 ? `${mins}m ${secs}s` : `${mins}m`;
+}
 function formatROISummary(stats) {
   const t = stats.totals;
   const totalSessions = t.totalSessionsWithLearnings + t.totalSessionsWithoutLearnings;
@@ -9222,7 +9254,7 @@ function formatROISummary(stats) {
   lines.push(`  Sessions tracked:              ${totalSessions}`);
   lines.push(`  Sessions with learnings:       ${t.totalSessionsWithLearnings}`);
   if (t.totalSessionsWithoutLearnings > 0) {
-    const rateWithout = t.totalSessionsWithoutLearnings > 0 ? (t.totalFailuresWithoutLearnings / t.totalSessionsWithoutLearnings).toFixed(1) : "0.0";
+    const rateWithout = (t.totalFailuresWithoutLearnings / t.totalSessionsWithoutLearnings).toFixed(1);
     lines.push(`  Failure rate (no learnings):   ${rateWithout}/session`);
   }
   if (t.totalSessionsWithLearnings > 0) {
@@ -9235,6 +9267,11 @@ function formatROISummary(stats) {
   if (t.estimatedSavingsTokens > 0) {
     lines.push(`  Estimated savings:             ~${t.estimatedSavingsTokens.toLocaleString()} tokens`);
   }
+  if (t.estimatedSavingsSeconds > 0) {
+    lines.push(`  Time saved:                    at least ${formatDuration(t.estimatedSavingsSeconds)} (not counting human frustration)`);
+  } else if (t.totalWasteSeconds > 0) {
+    lines.push(`  Time wasted on failures:       ${formatDuration(t.totalWasteSeconds)} (and counting)`);
+  }
   return lines.join("\n");
 }
 
@@ -9363,6 +9400,7 @@ async function learnFinalizeCommand(options) {
       eventCount: events.length,
       failureCount: waste.failureCount,
       promptCount: waste.promptCount,
+      wasteSeconds: Math.round(waste.totalWasteSeconds),
       hadLearningsAvailable: hadLearnings,
       learningsCount: existingLearnedItems,
       newLearningsProduced
@@ -9375,18 +9413,21 @@ async function learnFinalizeCommand(options) {
       hadLearningsAvailable: hadLearnings,
       learningsAvailableCount: existingLearnedItems,
       newLearningsProduced,
-      wasteTokens: waste.totalWasteTokens
+      wasteTokens: waste.totalWasteTokens,
+      wasteSeconds: Math.round(waste.totalWasteSeconds)
     });
     const t = roiStats.totals;
     const totalSessions = t.totalSessionsWithLearnings + t.totalSessionsWithoutLearnings;
     trackLearnROISnapshot({
       totalWasteTokens: t.totalWasteTokens,
+      totalWasteSeconds: t.totalWasteSeconds,
       totalSessions,
       sessionsWithLearnings: t.totalSessionsWithLearnings,
       sessionsWithoutLearnings: t.totalSessionsWithoutLearnings,
       failureRateWithLearnings: t.totalSessionsWithLearnings > 0 ? t.totalFailuresWithLearnings / t.totalSessionsWithLearnings : 0,
       failureRateWithoutLearnings: t.totalSessionsWithoutLearnings > 0 ? t.totalFailuresWithoutLearnings / t.totalSessionsWithoutLearnings : 0,
       estimatedSavingsTokens: t.estimatedSavingsTokens,
+      estimatedSavingsSeconds: t.estimatedSavingsSeconds,
       learningCount: roiStats.learnings.length
     });
     if (t.estimatedSavingsTokens > 0) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rely-ai/caliber",
-  "version": "1.20.0-dev.1773696964",
+  "version": "1.20.0-dev.1773698265",
   "description": "Analyze your codebase and generate optimized AI agent configs (CLAUDE.md, .cursorrules, skills) — no API key needed",
   "type": "module",
   "bin": {