agenr 0.9.15 → 0.9.16

package/CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## 0.9.16 - 2026-02-27
+
+### Added
+- Added progress logging throughout the consolidate pipeline. Pairwise
+  similarity scan, rules phases, cluster processing, LLM dedup checks, and
+  LLM merge calls now report progress so users can see the system is working.
+  Phase-level progress logs are always shown (not gated behind `--verbose`).
+- Added live cluster progress updates with ETA during consolidation phases so
+  long-running Phase 1, Phase 2, and Phase 3 work shows continuous terminal
+  activity.
+
 ## 0.9.14 - 2026-02-27
 
 ### Fixed

package/dist/cli-main.js

@@ -3741,6 +3741,13 @@ function collapsePreview(text4, maxLength = 80) {
   }
   return `${collapsed.slice(0, maxLength - 3)}...`;
 }
+function formatSeconds(ms) {
+  const seconds = Math.max(0, Math.round(ms / 1e3));
+  return `${seconds}s`;
+}
+function formatCount(value) {
+  return value.toLocaleString("en-US");
+}
 function forgettingScore(entry, now) {
   const ageDays2 = parseDaysBetween(now, entry.created_at);
   const recallCount = entry.recall_count ?? 0;
@@ -3932,6 +3939,13 @@ async function mergeNearExactDuplicates(db, options) {
   if (entries.length < 2) {
     return 0;
   }
+  const totalComparisons = entries.length * (entries.length - 1) / 2;
+  options.onLog(
+    `[merge] Computing pairwise similarity for ${entries.length} entries (${formatCount(totalComparisons)} comparisons)...`
+  );
+  const scanStartedAt = Date.now();
+  let checkedPairs = 0;
+  let nextProgressPercent = 10;
   const entryById = new Map(entries.map((entry) => [entry.id, entry]));
   const unionFind = new UnionFind();
   for (const entry of entries) {
@@ -3941,6 +3955,7 @@ async function mergeNearExactDuplicates(db, options) {
     const entry = entries[i];
     for (let j = i + 1; j < entries.length; j += 1) {
       const candidate = entries[j];
+      checkedPairs += 1;
       const similarity = cosineSim(entry.embedding, candidate.embedding);
       if (similarity <= MERGE_SIMILARITY_THRESHOLD) {
         continue;
@@ -3956,6 +3971,14 @@ async function mergeNearExactDuplicates(db, options) {
       }
       unionFind.union(entry.id, candidate.id);
     }
+    while (nextProgressPercent <= 100 && i + 1 >= Math.ceil(entries.length * nextProgressPercent / 100)) {
+      const elapsedMs = Date.now() - scanStartedAt;
+      const remainingPairs = Math.max(totalComparisons - checkedPairs, 0);
+      const etaMs = checkedPairs > 0 ? elapsedMs / checkedPairs * remainingPairs : 0;
+      const etaSuffix = checkedPairs > 0 ? ` ~${formatSeconds(etaMs)} remaining` : "";
+      options.onLog(`[merge] ...${nextProgressPercent}% (${formatCount(checkedPairs)} pairs checked)${etaSuffix}`);
+      nextProgressPercent += 10;
+    }
   }
   const groups = /* @__PURE__ */ new Map();
   for (const entry of entries) {
@@ -3964,6 +3987,9 @@ async function mergeNearExactDuplicates(db, options) {
     current.push(entry);
     groups.set(root, current);
   }
+  const groupedCount = Array.from(groups.values()).filter((group) => group.length >= 2).length;
+  const scanSeconds = ((Date.now() - scanStartedAt) / 1e3).toFixed(1);
+  options.onLog(`[merge] Similarity scan complete: ${groupedCount} groups found in ${scanSeconds}s`);
   let mergedCount = 0;
   for (const rawGroup of groups.values()) {
     if (rawGroup.length < 2) {
@@ -4126,17 +4152,44 @@ async function consolidateRules(db, dbPath, options = {}) {
   let expiredCount = 0;
   let mergedCount = 0;
   let orphanedRelationsCleaned = 0;
+  const mergePassCount = 1;
+  const runRulePasses = async () => {
+    for (let pass = 1; pass <= mergePassCount; pass += 1) {
+      onLog("[rules] Pruning expired entries...");
+      expiredCount += await expireDecayedEntries(db, now, {
+        dryRun,
+        verbose,
+        onLog,
+        platform,
+        project,
+        excludeProject
+      });
+      onLog(`[rules] Pass ${pass}: merging near-exact duplicates...`);
+      mergedCount += await mergeNearExactDuplicates(db, {
+        dryRun,
+        verbose,
+        onLog,
+        platform,
+        project,
+        excludeProject
+      });
+    }
+  };
   if (dryRun) {
-    expiredCount = await expireDecayedEntries(db, now, { dryRun, verbose, onLog, platform, project, excludeProject });
-    mergedCount = await mergeNearExactDuplicates(db, { dryRun, verbose, onLog, platform, project, excludeProject });
-    orphanedRelationsCleaned = skipOrphanCleanup ? 0 : await cleanOrphanedRelations(db, true);
+    await runRulePasses();
+    if (!skipOrphanCleanup) {
+      onLog("[rules] Cleaning orphaned relations...");
+      orphanedRelationsCleaned = await cleanOrphanedRelations(db, true);
+    }
   } else {
     await db.execute("BEGIN");
     try {
       await ensureExpiredSentinel(db);
-      expiredCount = await expireDecayedEntries(db, now, { dryRun, verbose, onLog, platform, project, excludeProject });
-      mergedCount = await mergeNearExactDuplicates(db, { dryRun, verbose, onLog, platform, project, excludeProject });
-      orphanedRelationsCleaned = skipOrphanCleanup ? 0 : await cleanOrphanedRelations(db, false);
+      await runRulePasses();
+      if (!skipOrphanCleanup) {
+        onLog("[rules] Cleaning orphaned relations...");
+        orphanedRelationsCleaned = await cleanOrphanedRelations(db, false);
+      }
       await db.execute("COMMIT");
     } catch (error) {
       try {
@@ -4421,6 +4474,7 @@ async function buildClusters(db, options = {}) {
   const entryById = new Map(candidates.map((entry) => [entry.id, entry]));
   const unionFind = new UnionFind();
   const looseUnionPairs = /* @__PURE__ */ new Set();
+  const llmDedupQueue = [];
   let llmDedupCalls = 0;
   let llmDedupMatches = 0;
   for (const entry of candidates) {
@@ -4450,14 +4504,21 @@ async function buildClusters(db, options = {}) {
       if (!llmClient) {
         continue;
       }
-      llmDedupCalls += 1;
-      const isSame = await llmDedupCheck(llmClient, entry, candidate);
-      if (isSame) {
-        llmDedupMatches += 1;
-        looseUnionPairs.add(key);
-        unionFind.union(entry.id, candidate.id);
-      }
+      llmDedupQueue.push({ entry, candidate, key });
+    }
+  }
+  for (const pair of llmDedupQueue) {
+    if (!llmClient) {
+      break;
+    }
+    llmDedupCalls += 1;
+    const isSame = await llmDedupCheck(llmClient, pair.entry, pair.candidate);
+    if (isSame) {
+      llmDedupMatches += 1;
+      looseUnionPairs.add(pair.key);
+      unionFind.union(pair.entry.id, pair.candidate.id);
     }
+    onLog(`[dedup] Checked ${llmDedupCalls}/${llmDedupQueue.length} pairs (${llmDedupMatches} matched)`);
   }
   const groups = /* @__PURE__ */ new Map();
   for (const entry of candidates) {
@@ -7916,6 +7977,10 @@ async function mergeCluster(db, cluster, llmClient, apiKey, options = {}) {
       );
     }
   }
+  const subjectPreview = truncateContent(cluster.entries[0]?.subject ?? "", 60);
+  onLog(
+    `[merge-llm] Merging cluster of ${cluster.entries.length} entries (subject: "${subjectPreview || "unknown"}")`
+  );
   let mergeResult = null;
   try {
     const response = await runSimpleStream({
@@ -8376,6 +8441,41 @@ async function runFinalization(db, dryRun, onWarn, deps) {
 async function processPhaseClusters(params, deps) {
   const stats = defaultClusterStats();
   stats.clustersFound = params.clusters.length;
+  const onLog = params.options.onLog ?? (() => void 0);
+  const showLiveProgress = process.stderr.isTTY && params.options.verbose !== true;
+  const liveLineWidth = 120;
+  const phaseStartedAt = Date.now();
+  const formatEta = (ms) => {
+    const seconds = Math.max(0, Math.round(ms / 1e3));
+    if (seconds < 60) {
+      return `${seconds}s`;
+    }
+    const minutes = Math.floor(seconds / 60);
+    const remainingSeconds = seconds % 60;
+    return remainingSeconds === 0 ? `${minutes}m` : `${minutes}m ${remainingSeconds}s`;
+  };
+  const updateLiveProgress = (clusterIndex, totalClusters) => {
+    if (!showLiveProgress) {
+      return;
+    }
+    const completedClusters = clusterIndex - 1;
+    let etaSuffix = "";
+    if (completedClusters >= 2) {
+      const elapsedMs = Date.now() - phaseStartedAt;
+      const etaMs = (totalClusters - completedClusters) * (elapsedMs / completedClusters);
+      etaSuffix = ` ~${formatEta(etaMs)} remaining`;
+    }
+    process.stderr.write(
+      `\rPhase ${params.phase}: Processing cluster ${clusterIndex}/${totalClusters} (${params.type}) ...${etaSuffix}`
+    );
+  };
+  const clearLiveProgress = () => {
+    if (!showLiveProgress) {
+      return;
+    }
+    process.stderr.write("\r");
+    process.stderr.write(`${" ".repeat(liveLineWidth)}\r`);
+  };
   const pending = [];
   for (let i = 0; i < params.clusters.length; i += 1) {
     const cluster = params.clusters[i];
@@ -8395,10 +8495,14 @@ async function processPhaseClusters(params, deps) {
       params.context.batchReached = true;
       break;
     }
+    clearLiveProgress();
+    const clusterNumber = stats.clustersProcessed + 1;
+    onLog(`[phase ${params.phase}] Processing cluster ${clusterNumber}/${pending.length}...`);
+    updateLiveProgress(clusterNumber, pending.length);
     const outcome = await deps.mergeClusterFn(params.db, item.cluster, params.llmClient, params.embeddingApiKey, {
       dryRun: params.options.dryRun,
       verbose: params.options.verbose,
-      onLog: params.options.verbose ? params.options.onLog : void 0
+      onLog
     });
     stats.clustersProcessed += 1;
     stats.llmCalls += 1;
@@ -8425,6 +8529,7 @@ async function processPhaseClusters(params, deps) {
       break;
     }
   }
+  clearLiveProgress();
   return stats;
 }
 async function runConsolidationOrchestrator(db, dbPath, llmClient, embeddingApiKey, options = {}, deps = {}) {
@@ -8577,7 +8682,7 @@ async function runConsolidationOrchestrator(db, dbPath, llmClient, embeddingApiK
       skipBackup: projectIndex > 0,
       backupPath: projectIndex > 0 ? sharedBackupPath : void 0,
       skipOrphanCleanup: projectIndex > 0,
-      onLog: options.verbose ? onLog : void 0
+      onLog
     });
     if (projectIndex === 0) {
       sharedBackupPath = phase0Stats.backupPath;
@@ -8636,7 +8741,7 @@ async function runConsolidationOrchestrator(db, dbPath, llmClient, embeddingApiK
           looseThreshold: options.looseThreshold,
           idempotencyDays: options.idempotencyDays,
           verbose: options.verbose,
-          onLog: options.verbose ? onLog : void 0,
+          onLog,
           onStats: (stats) => {
             phase1ClusterStats = stats;
           }
@@ -8671,7 +8776,7 @@ async function runConsolidationOrchestrator(db, dbPath, llmClient, embeddingApiK
           looseThreshold: options.looseThreshold,
           idempotencyDays: options.idempotencyDays,
           verbose: options.verbose,
-          onLog: options.verbose ? onLog : void 0,
+          onLog,
           onStats: (stats) => {
             phase2ClusterStats = stats;
           }
@@ -8834,7 +8939,7 @@ async function runConsolidationOrchestrator(db, dbPath, llmClient, embeddingApiK
           looseThreshold: options.looseThreshold,
           idempotencyDays: 0,
           verbose: options.verbose,
-          onLog: options.verbose ? onLog : void 0,
+          onLog,
           onStats: (stats) => {
             phase3ClusterStats = stats;
           }
@@ -15865,7 +15970,7 @@ function parsePositiveInt2(value, fallback, label) {
 function formatMetric(value) {
   return value.toFixed(2);
 }
-function formatCount(value) {
+function formatCount2(value) {
   const rounded = Math.round(value);
   if (Math.abs(value - rounded) < 1e-9) {
     return String(rounded);
@@ -16294,7 +16399,7 @@ function renderSummary(result, rangesBySession) {
   for (const session of result.sessions) {
     const range = rangesBySession.get(session.session);
     const avgEntryCount = mean(session.runs.map((run) => run.total_entries));
-    const countLabel = range !== void 0 ? `${formatCount(avgEntryCount)}/${range.min}-${range.max}` : `${formatCount(avgEntryCount)}/-`;
+    const countLabel = range !== void 0 ? `${formatCount2(avgEntryCount)}/${range.min}-${range.max}` : `${formatCount2(avgEntryCount)}/-`;
     const passCount = session.runs.filter((run) => run.pass).length;
     lines.push(
       `${session.session.padEnd(sessionWidth)}  ${formatMetric(session.mean_recall).padEnd(6)}  ${formatMetric(session.mean_partial_recall).padEnd(8)}  ${formatMetric(session.mean_precision).padEnd(9)}  ${formatMetric(session.mean_composite).padEnd(9)}  ${countLabel.padEnd(9)}  ${passCount}/${result.runs}`

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agenr",
-  "version": "0.9.15",
+  "version": "0.9.16",
   "openclaw": {
     "extensions": [
       "dist/openclaw-plugin/index.js"