@rely-ai/caliber 1.20.0-dev.1773687255 → 1.20.0-dev.1773690306

package/dist/bin.js

@@ -149,13 +149,14 @@ __export(constants_exports, {
   CALIBER_DIR: () => CALIBER_DIR,
   LEARNING_DIR: () => LEARNING_DIR,
   LEARNING_MAX_EVENTS: () => LEARNING_MAX_EVENTS,
+  LEARNING_ROI_FILE: () => LEARNING_ROI_FILE,
   LEARNING_SESSION_FILE: () => LEARNING_SESSION_FILE,
   LEARNING_STATE_FILE: () => LEARNING_STATE_FILE,
   MANIFEST_FILE: () => MANIFEST_FILE
 });
 import path9 from "path";
 import os2 from "os";
-var AUTH_DIR, CALIBER_DIR, MANIFEST_FILE, BACKUPS_DIR, LEARNING_DIR, LEARNING_SESSION_FILE, LEARNING_STATE_FILE, LEARNING_MAX_EVENTS;
+var AUTH_DIR, CALIBER_DIR, MANIFEST_FILE, BACKUPS_DIR, LEARNING_DIR, LEARNING_SESSION_FILE, LEARNING_STATE_FILE, LEARNING_MAX_EVENTS, LEARNING_ROI_FILE;
 var init_constants = __esm({
   "src/constants.ts"() {
     "use strict";
@@ -167,6 +168,7 @@ var init_constants = __esm({
     LEARNING_SESSION_FILE = "current-session.jsonl";
     LEARNING_STATE_FILE = "state.json";
     LEARNING_MAX_EVENTS = 500;
+    LEARNING_ROI_FILE = "roi-stats.json";
   }
 });
 
@@ -219,8 +221,8 @@ var init_lock = __esm({
 
 // src/cli.ts
 import { Command } from "commander";
-import fs33 from "fs";
-import path26 from "path";
+import fs34 from "fs";
+import path27 from "path";
 import { fileURLToPath } from "url";
 
 // src/commands/init.ts
@@ -2240,15 +2242,15 @@ init_config();
 // src/utils/dependencies.ts
 import { readFileSync } from "fs";
 import { join } from "path";
-function readFileOrNull(path28) {
+function readFileOrNull(path29) {
   try {
-    return readFileSync(path28, "utf-8");
+    return readFileSync(path29, "utf-8");
   } catch {
     return null;
   }
 }
-function readJsonOrNull(path28) {
-  const content = readFileOrNull(path28);
+function readJsonOrNull(path29) {
+  const content = readFileOrNull(path29);
   if (!content) return null;
   try {
     return JSON.parse(content);
@@ -5757,6 +5759,36 @@ function trackSkillsInstalled(count) {
 function trackUndoExecuted() {
   trackEvent("undo_executed");
 }
+function trackLearnSessionAnalyzed(props) {
+  trackEvent("learn_session_analyzed", {
+    event_count: props.eventCount,
+    failure_count: props.failureCount,
+    correction_count: props.correctionCount,
+    had_learnings_available: props.hadLearningsAvailable,
+    learnings_available_count: props.learningsAvailableCount,
+    new_learnings_produced: props.newLearningsProduced,
+    waste_tokens: props.wasteTokens
+  });
+}
+function trackLearnROISnapshot(props) {
+  trackEvent("learn_roi_snapshot", {
+    total_waste_tokens: props.totalWasteTokens,
+    total_sessions: props.totalSessions,
+    sessions_with_learnings: props.sessionsWithLearnings,
+    sessions_without_learnings: props.sessionsWithoutLearnings,
+    failure_rate_with_learnings: props.failureRateWithLearnings,
+    failure_rate_without_learnings: props.failureRateWithoutLearnings,
+    estimated_savings_tokens: props.estimatedSavingsTokens,
+    learning_count: props.learningCount
+  });
+}
+function trackLearnNewLearning(props) {
+  trackEvent("learn_new_learning", {
+    observation_type: props.observationType,
+    waste_tokens: props.wasteTokens,
+    source_event_count: props.sourceEventCount
+  });
+}
 
 // src/commands/recommend.ts
 function detectLocalPlatforms() {
@@ -7084,7 +7116,7 @@ async function refineLoop(currentSetup, _targetAgent, sessionHistory) {
 }
 function summarizeSetup(action, setup) {
   const descriptions = setup.fileDescriptions;
-  const files = descriptions ? Object.entries(descriptions).map(([path28, desc]) => `  ${path28}: ${desc}`).join("\n") : Object.keys(setup).filter((k) => k !== "targetAgent" && k !== "fileDescriptions").join(", ");
+  const files = descriptions ? Object.entries(descriptions).map(([path29, desc]) => `  ${path29}: ${desc}`).join("\n") : Object.keys(setup).filter((k) => k !== "targetAgent" && k !== "fileDescriptions").join(", ");
   return `${action}. Files:
 ${files}`;
 }
@@ -8285,7 +8317,7 @@ async function configCommand() {
 }
 
 // src/commands/learn.ts
-import fs32 from "fs";
+import fs33 from "fs";
 import chalk18 from "chalk";
 
 // src/learner/stdin.ts
@@ -8555,9 +8587,124 @@ ${eventsText}`;
   });
   return parseAnalysisResponse(raw);
 }
+function calculateSessionWaste(events) {
+  let totalWasteTokens = 0;
+  let failureCount = 0;
+  let promptCount = 0;
+  for (const event of events) {
+    if (event.hook_event_name === "PostToolUseFailure") {
+      const te = event;
+      const inputStr = JSON.stringify(te.tool_input);
+      const responseStr = typeof te.tool_response === "object" && "_truncated" in te.tool_response ? String(te.tool_response._truncated) : JSON.stringify(te.tool_response);
+      totalWasteTokens += estimateTokens(inputStr + responseStr);
+      failureCount++;
+    } else if (event.hook_event_name === "UserPromptSubmit") {
+      promptCount++;
+    }
+  }
+  return { totalWasteTokens, failureCount, promptCount };
+}
 
 // src/commands/learn.ts
 init_config();
+
+// src/learner/roi.ts
+init_constants();
+import fs32 from "fs";
+import path26 from "path";
+var DEFAULT_TOTALS = {
+  totalWasteTokens: 0,
+  totalSessionsWithLearnings: 0,
+  totalSessionsWithoutLearnings: 0,
+  totalFailuresWithLearnings: 0,
+  totalFailuresWithoutLearnings: 0,
+  estimatedSavingsTokens: 0,
+  firstSessionTimestamp: "",
+  lastSessionTimestamp: ""
+};
+function roiFilePath() {
+  return path26.join(LEARNING_DIR, LEARNING_ROI_FILE);
+}
+function readROIStats() {
+  const filePath = roiFilePath();
+  if (!fs32.existsSync(filePath)) {
+    return { learnings: [], sessions: [], totals: { ...DEFAULT_TOTALS } };
+  }
+  try {
+    return JSON.parse(fs32.readFileSync(filePath, "utf-8"));
+  } catch {
+    return { learnings: [], sessions: [], totals: { ...DEFAULT_TOTALS } };
+  }
+}
+function writeROIStats(stats) {
+  ensureLearningDir();
+  fs32.writeFileSync(roiFilePath(), JSON.stringify(stats, null, 2));
+}
+function recalculateTotals(stats) {
+  const totals = stats.totals;
+  totals.totalWasteTokens = stats.learnings.reduce((sum, l) => sum + l.wasteTokens, 0);
+  totals.totalSessionsWithLearnings = 0;
+  totals.totalSessionsWithoutLearnings = 0;
+  totals.totalFailuresWithLearnings = 0;
+  totals.totalFailuresWithoutLearnings = 0;
+  for (const s of stats.sessions) {
+    if (s.hadLearningsAvailable) {
+      totals.totalSessionsWithLearnings++;
+      totals.totalFailuresWithLearnings += s.failureCount;
+    } else {
+      totals.totalSessionsWithoutLearnings++;
+      totals.totalFailuresWithoutLearnings += s.failureCount;
+    }
+  }
+  totals.estimatedSavingsTokens = totals.totalWasteTokens * totals.totalSessionsWithLearnings;
+  if (stats.sessions.length > 0) {
+    totals.firstSessionTimestamp = stats.sessions[0].timestamp;
+    totals.lastSessionTimestamp = stats.sessions[stats.sessions.length - 1].timestamp;
+  }
+}
+var MAX_SESSIONS = 500;
+var MAX_LEARNINGS = 1e3;
+function recordSession(summary, learnings) {
+  const stats = readROIStats();
+  stats.sessions.push(summary);
+  if (learnings?.length) {
+    stats.learnings.push(...learnings);
+  }
+  if (stats.sessions.length > MAX_SESSIONS) {
+    stats.sessions = stats.sessions.slice(-MAX_SESSIONS);
+  }
+  if (stats.learnings.length > MAX_LEARNINGS) {
+    stats.learnings = stats.learnings.slice(-MAX_LEARNINGS);
+  }
+  recalculateTotals(stats);
+  writeROIStats(stats);
+  return stats;
+}
+function formatROISummary(stats) {
+  const t = stats.totals;
+  const totalSessions = t.totalSessionsWithLearnings + t.totalSessionsWithoutLearnings;
+  if (totalSessions === 0) return "";
+  const lines = ["ROI Summary"];
+  lines.push(`  Sessions tracked:              ${totalSessions}`);
+  lines.push(`  Sessions with learnings:       ${t.totalSessionsWithLearnings}`);
+  if (t.totalSessionsWithoutLearnings > 0) {
+    const rateWithout = t.totalSessionsWithoutLearnings > 0 ? (t.totalFailuresWithoutLearnings / t.totalSessionsWithoutLearnings).toFixed(1) : "0.0";
+    lines.push(`  Failure rate (no learnings):   ${rateWithout}/session`);
+  }
+  if (t.totalSessionsWithLearnings > 0) {
+    const rateWith = (t.totalFailuresWithLearnings / t.totalSessionsWithLearnings).toFixed(1);
+    lines.push(`  Failure rate (with learnings): ${rateWith}/session`);
+  }
+  if (t.totalWasteTokens > 0) {
+    lines.push(`  Total waste captured:          ${t.totalWasteTokens.toLocaleString()} tokens`);
+  }
+  if (t.estimatedSavingsTokens > 0) {
+    lines.push(`  Estimated savings:             ~${t.estimatedSavingsTokens.toLocaleString()} tokens`);
+  }
+  return lines.join("\n");
+}
+
+// src/commands/learn.ts
 var MIN_EVENTS_FOR_ANALYSIS = 25;
 async function learnObserveCommand(options) {
   try {
@@ -8636,18 +8783,82 @@ async function learnFinalizeCommand(options) {
       existingSkills
     );
     analyzed = true;
+    const waste = calculateSessionWaste(events);
+    const existingLearnedItems = existingLearnedSection ? existingLearnedSection.split("\n").filter((l) => l.startsWith("- ")).length : 0;
+    const hadLearnings = existingLearnedItems > 0;
+    let newLearningsProduced = 0;
+    let roiLearningEntries = [];
     if (response.claudeMdLearnedSection || response.skills?.length) {
       const result = writeLearnedContent({
         claudeMdLearnedSection: response.claudeMdLearnedSection,
         skills: response.skills
       });
+      newLearningsProduced = result.newItemCount;
       if (result.newItemCount > 0) {
-        console.log(chalk18.dim(`caliber: learned ${result.newItemCount} new pattern${result.newItemCount === 1 ? "" : "s"}`));
+        const wasteLabel = waste.totalWasteTokens > 0 ? ` (~${waste.totalWasteTokens.toLocaleString()} wasted tokens captured)` : "";
+        console.log(chalk18.dim(`caliber: learned ${result.newItemCount} new pattern${result.newItemCount === 1 ? "" : "s"}${wasteLabel}`));
         for (const item of result.newItems) {
           console.log(chalk18.dim(`  + ${item.replace(/^- /, "").slice(0, 80)}`));
         }
+        const wastePerLearning = Math.round(waste.totalWasteTokens / result.newItemCount);
+        const TYPE_RE = /^\*\*\[([^\]]+)\]\*\*/;
+        const learningEntries = result.newItems.map((item) => {
+          const clean = item.replace(/^- /, "");
+          const typeMatch = clean.match(TYPE_RE);
+          return {
+            timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+            observationType: typeMatch ? typeMatch[1] : "unknown",
+            summary: clean.replace(TYPE_RE, "").trim().slice(0, 80),
+            wasteTokens: wastePerLearning,
+            sourceEventCount: events.length
+          };
+        });
+        for (const entry of learningEntries) {
+          trackLearnNewLearning({
+            observationType: entry.observationType,
+            wasteTokens: entry.wasteTokens,
+            sourceEventCount: entry.sourceEventCount
+          });
+        }
+        roiLearningEntries = learningEntries;
       }
     }
+    const sessionSummary = {
+      timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+      sessionId: readState2().sessionId || "unknown",
+      eventCount: events.length,
+      failureCount: waste.failureCount,
+      promptCount: waste.promptCount,
+      hadLearningsAvailable: hadLearnings,
+      learningsCount: existingLearnedItems,
+      newLearningsProduced
+    };
+    const roiStats = recordSession(sessionSummary, roiLearningEntries);
+    trackLearnSessionAnalyzed({
+      eventCount: events.length,
+      failureCount: waste.failureCount,
+      correctionCount: waste.promptCount,
+      hadLearningsAvailable: hadLearnings,
+      learningsAvailableCount: existingLearnedItems,
+      newLearningsProduced,
+      wasteTokens: waste.totalWasteTokens
+    });
+    const t = roiStats.totals;
+    const totalSessions = t.totalSessionsWithLearnings + t.totalSessionsWithoutLearnings;
+    trackLearnROISnapshot({
+      totalWasteTokens: t.totalWasteTokens,
+      totalSessions,
+      sessionsWithLearnings: t.totalSessionsWithLearnings,
+      sessionsWithoutLearnings: t.totalSessionsWithoutLearnings,
+      failureRateWithLearnings: t.totalSessionsWithLearnings > 0 ? t.totalFailuresWithLearnings / t.totalSessionsWithLearnings : 0,
+      failureRateWithoutLearnings: t.totalSessionsWithoutLearnings > 0 ? t.totalFailuresWithoutLearnings / t.totalSessionsWithoutLearnings : 0,
+      estimatedSavingsTokens: t.estimatedSavingsTokens,
+      learningCount: roiStats.learnings.length
+    });
+    if (t.estimatedSavingsTokens > 0) {
+      const totalLearnings = existingLearnedItems + newLearningsProduced;
+      console.log(chalk18.dim(`caliber: ${totalLearnings} learnings active \u2014 est. ~${t.estimatedSavingsTokens.toLocaleString()} tokens saved across ${t.totalSessionsWithLearnings} sessions`));
+    }
   } catch (err) {
     if (options?.force) {
       console.error(chalk18.red("caliber: finalize failed \u2014"), err instanceof Error ? err.message : err);
@@ -8662,7 +8873,7 @@ async function learnFinalizeCommand(options) {
 }
 async function learnInstallCommand() {
   let anyInstalled = false;
-  if (fs32.existsSync(".claude")) {
+  if (fs33.existsSync(".claude")) {
     const r = installLearningHooks();
     if (r.installed) {
       console.log(chalk18.green("\u2713") + " Claude Code learning hooks installed");
@@ -8671,7 +8882,7 @@ async function learnInstallCommand() {
       console.log(chalk18.dim("  Claude Code hooks already installed"));
     }
   }
-  if (fs32.existsSync(".cursor")) {
+  if (fs33.existsSync(".cursor")) {
     const r = installCursorLearningHooks();
     if (r.installed) {
       console.log(chalk18.green("\u2713") + " Cursor learning hooks installed");
@@ -8680,7 +8891,7 @@ async function learnInstallCommand() {
       console.log(chalk18.dim("  Cursor hooks already installed"));
     }
   }
-  if (!fs32.existsSync(".claude") && !fs32.existsSync(".cursor")) {
+  if (!fs33.existsSync(".claude") && !fs33.existsSync(".cursor")) {
     console.log(chalk18.yellow("No .claude/ or .cursor/ directory found."));
     console.log(chalk18.dim("  Run `caliber init` first, or create the directory manually."));
     return;
@@ -8740,12 +8951,21 @@ async function learnStatusCommand() {
     console.log(`
 Learned items in CALIBER_LEARNINGS.md: ${chalk18.cyan(String(lineCount))}`);
   }
+  const roiStats = readROIStats();
+  const roiSummary = formatROISummary(roiStats);
+  if (roiSummary) {
+    console.log();
+    console.log(chalk18.bold(roiSummary.split("\n")[0]));
+    for (const line of roiSummary.split("\n").slice(1)) {
+      console.log(line);
+    }
+  }
 }
 
 // src/cli.ts
-var __dirname = path26.dirname(fileURLToPath(import.meta.url));
+var __dirname = path27.dirname(fileURLToPath(import.meta.url));
 var pkg = JSON.parse(
-  fs33.readFileSync(path26.resolve(__dirname, "..", "package.json"), "utf-8")
+  fs34.readFileSync(path27.resolve(__dirname, "..", "package.json"), "utf-8")
 );
 var program = new Command();
 var displayVersion = process.env.CALIBER_LOCAL ? `${pkg.version}-local` : pkg.version;
@@ -8819,16 +9039,16 @@ learn.command("remove").description("Remove learning hooks from .claude/settings
 learn.command("status").description("Show learning system status").action(tracked("learn:status", learnStatusCommand));
 
 // src/utils/version-check.ts
-import fs34 from "fs";
-import path27 from "path";
+import fs35 from "fs";
+import path28 from "path";
 import { fileURLToPath as fileURLToPath2 } from "url";
 import { execSync as execSync14 } from "child_process";
 import chalk19 from "chalk";
 import ora6 from "ora";
 import confirm2 from "@inquirer/confirm";
-var __dirname_vc = path27.dirname(fileURLToPath2(import.meta.url));
+var __dirname_vc = path28.dirname(fileURLToPath2(import.meta.url));
 var pkg2 = JSON.parse(
-  fs34.readFileSync(path27.resolve(__dirname_vc, "..", "package.json"), "utf-8")
+  fs35.readFileSync(path28.resolve(__dirname_vc, "..", "package.json"), "utf-8")
 );
 function getChannel(version) {
   const match = version.match(/-(dev|next)\./);
@@ -8853,8 +9073,8 @@ function isNewer(registry, current) {
 function getInstalledVersion() {
   try {
     const globalRoot = execSync14("npm root -g", { encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"] }).trim();
-    const pkgPath = path27.join(globalRoot, "@rely-ai", "caliber", "package.json");
-    return JSON.parse(fs34.readFileSync(pkgPath, "utf-8")).version;
+    const pkgPath = path28.join(globalRoot, "@rely-ai", "caliber", "package.json");
+    return JSON.parse(fs35.readFileSync(pkgPath, "utf-8")).version;
   } catch {
     return null;
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rely-ai/caliber",
-  "version": "1.20.0-dev.1773687255",
+  "version": "1.20.0-dev.1773690306",
   "description": "Analyze your codebase and generate optimized AI agent configs (CLAUDE.md, .cursorrules, skills) — no API key needed",
   "type": "module",
   "bin": {