agenr 0.9.15 → 0.9.17

package/CHANGELOG.md

@@ -1,5 +1,23 @@
 # Changelog
 
+## 0.9.17 - 2026-02-27
+
+### Changed
+- Optimized LLM dedup in consolidate clustering: batch up to 10 pairs per API
+  call with 5 concurrent batches. Reduces a 2400-pair dedup queue from ~60min
+  (sequential, 1 call per pair) to ~2min.
+
+## 0.9.16 - 2026-02-27
+
+### Added
+- Added progress logging throughout the consolidate pipeline. Pairwise
+  similarity scan, rules phases, cluster processing, LLM dedup checks, and
+  LLM merge calls now report progress so users can see the system is working.
+  Phase-level progress logs are always shown (not gated behind `--verbose`).
+- Added live cluster progress updates with ETA during consolidation phases so
+  long-running Phase 1, Phase 2, and Phase 3 work shows continuous terminal
+  activity.
+
 ## 0.9.14 - 2026-02-27
 
 ### Fixed

package/dist/cli-main.js

@@ -3741,6 +3741,13 @@ function collapsePreview(text4, maxLength = 80) {
   }
   return `${collapsed.slice(0, maxLength - 3)}...`;
 }
+function formatSeconds(ms) {
+  const seconds = Math.max(0, Math.round(ms / 1e3));
+  return `${seconds}s`;
+}
+function formatCount(value) {
+  return value.toLocaleString("en-US");
+}
 function forgettingScore(entry, now) {
   const ageDays2 = parseDaysBetween(now, entry.created_at);
   const recallCount = entry.recall_count ?? 0;
@@ -3932,6 +3939,13 @@ async function mergeNearExactDuplicates(db, options) {
   if (entries.length < 2) {
     return 0;
   }
+  const totalComparisons = entries.length * (entries.length - 1) / 2;
+  options.onLog(
+    `[merge] Computing pairwise similarity for ${entries.length} entries (${formatCount(totalComparisons)} comparisons)...`
+  );
+  const scanStartedAt = Date.now();
+  let checkedPairs = 0;
+  let nextProgressPercent = 10;
   const entryById = new Map(entries.map((entry) => [entry.id, entry]));
   const unionFind = new UnionFind();
   for (const entry of entries) {
@@ -3941,6 +3955,7 @@ async function mergeNearExactDuplicates(db, options) {
     const entry = entries[i];
     for (let j = i + 1; j < entries.length; j += 1) {
       const candidate = entries[j];
+      checkedPairs += 1;
       const similarity = cosineSim(entry.embedding, candidate.embedding);
       if (similarity <= MERGE_SIMILARITY_THRESHOLD) {
         continue;
@@ -3956,6 +3971,14 @@ async function mergeNearExactDuplicates(db, options) {
       }
       unionFind.union(entry.id, candidate.id);
     }
+    while (nextProgressPercent <= 100 && i + 1 >= Math.ceil(entries.length * nextProgressPercent / 100)) {
+      const elapsedMs = Date.now() - scanStartedAt;
+      const remainingPairs = Math.max(totalComparisons - checkedPairs, 0);
+      const etaMs = checkedPairs > 0 ? elapsedMs / checkedPairs * remainingPairs : 0;
+      const etaSuffix = checkedPairs > 0 ? ` ~${formatSeconds(etaMs)} remaining` : "";
+      options.onLog(`[merge] ...${nextProgressPercent}% (${formatCount(checkedPairs)} pairs checked)${etaSuffix}`);
+      nextProgressPercent += 10;
+    }
   }
   const groups = /* @__PURE__ */ new Map();
   for (const entry of entries) {
@@ -3964,6 +3987,9 @@ async function mergeNearExactDuplicates(db, options) {
     current.push(entry);
     groups.set(root, current);
   }
+  const groupedCount = Array.from(groups.values()).filter((group) => group.length >= 2).length;
+  const scanSeconds = ((Date.now() - scanStartedAt) / 1e3).toFixed(1);
+  options.onLog(`[merge] Similarity scan complete: ${groupedCount} groups found in ${scanSeconds}s`);
   let mergedCount = 0;
   for (const rawGroup of groups.values()) {
     if (rawGroup.length < 2) {
@@ -4126,17 +4152,44 @@ async function consolidateRules(db, dbPath, options = {}) {
   let expiredCount = 0;
   let mergedCount = 0;
   let orphanedRelationsCleaned = 0;
+  const mergePassCount = 1;
+  const runRulePasses = async () => {
+    for (let pass = 1; pass <= mergePassCount; pass += 1) {
+      onLog("[rules] Pruning expired entries...");
+      expiredCount += await expireDecayedEntries(db, now, {
+        dryRun,
+        verbose,
+        onLog,
+        platform,
+        project,
+        excludeProject
+      });
+      onLog(`[rules] Pass ${pass}: merging near-exact duplicates...`);
+      mergedCount += await mergeNearExactDuplicates(db, {
+        dryRun,
+        verbose,
+        onLog,
+        platform,
+        project,
+        excludeProject
+      });
+    }
+  };
   if (dryRun) {
-    expiredCount = await expireDecayedEntries(db, now, { dryRun, verbose, onLog, platform, project, excludeProject });
-    mergedCount = await mergeNearExactDuplicates(db, { dryRun, verbose, onLog, platform, project, excludeProject });
-    orphanedRelationsCleaned = skipOrphanCleanup ? 0 : await cleanOrphanedRelations(db, true);
+    await runRulePasses();
+    if (!skipOrphanCleanup) {
+      onLog("[rules] Cleaning orphaned relations...");
+      orphanedRelationsCleaned = await cleanOrphanedRelations(db, true);
+    }
   } else {
     await db.execute("BEGIN");
     try {
       await ensureExpiredSentinel(db);
-      expiredCount = await expireDecayedEntries(db, now, { dryRun, verbose, onLog, platform, project, excludeProject });
-      mergedCount = await mergeNearExactDuplicates(db, { dryRun, verbose, onLog, platform, project, excludeProject });
-      orphanedRelationsCleaned = skipOrphanCleanup ? 0 : await cleanOrphanedRelations(db, false);
+      await runRulePasses();
+      if (!skipOrphanCleanup) {
+        onLog("[rules] Cleaning orphaned relations...");
+        orphanedRelationsCleaned = await cleanOrphanedRelations(db, false);
+      }
       await db.execute("COMMIT");
     } catch (error) {
       try {
@@ -4201,15 +4254,26 @@ var DEFAULT_IDEMPOTENCY_DAYS = 7;
 var DEFAULT_NEIGHBOR_LIMIT = 20;
 var MAX_ACTIVE_EMBEDDED_ENTRIES2 = 2e4;
 var MILLISECONDS_PER_DAY = 1e3 * 60 * 60 * 24;
-var LLM_DEDUP_TOOL_NAME = "dedup_check";
+var LLM_DEDUP_BATCH_TOOL_NAME = "batch_dedup_check";
+var LLM_DEDUP_BATCH_SIZE = 10;
+var LLM_DEDUP_CONCURRENCY = 5;
 var LLM_DEDUP_TOOL_SCHEMA = Type.Object({
   same: Type.Boolean(),
   reason: Type.String()
 });
-var LLM_DEDUP_TOOL = {
-  name: LLM_DEDUP_TOOL_NAME,
-  description: "Decide whether two knowledge entries express the same knowledge.",
-  parameters: LLM_DEDUP_TOOL_SCHEMA
+var LLM_DEDUP_BATCH_TOOL_SCHEMA = Type.Object({
+  results: Type.Array(
+    Type.Object({
+      pair: Type.Number(),
+      same: Type.Boolean(),
+      reason: Type.Optional(Type.String())
+    })
+  )
+});
+var LLM_DEDUP_BATCH_TOOL = {
+  name: LLM_DEDUP_BATCH_TOOL_NAME,
+  description: "Return dedup results for all pairs",
+  parameters: LLM_DEDUP_BATCH_TOOL_SCHEMA
 };
 function toNumber5(value) {
   if (typeof value === "number") {
@@ -4250,71 +4314,99 @@ function normalizeSubject2(value) {
 function pairKey(a, b) {
   return a < b ? `${a}|${b}` : `${b}|${a}`;
 }
-function buildLlmDedupContext(entryA, entryB) {
+function buildLlmDedupBatchContext(pairs) {
   const systemPrompt = [
     "You are a deduplication assistant for knowledge entries.",
-    "Decide if two entries express the same knowledge in different wording.",
-    "Return your answer by calling dedup_check."
-  ].join("\n");
-  const userPrompt = [
-    "Are these two knowledge entries expressing the same fact or genuinely distinct?",
-    `Entry A: ${entryA.content}`,
-    `Entry B: ${entryB.content}`,
-    'Set "same" to true only when both entries represent the same knowledge.'
+    "For each numbered pair, decide if they express the same knowledge.",
+    "Call batch_dedup_check once with your results."
   ].join("\n");
+  const pairBlocks = pairs.map(
+    (pair, index) => [
+      `Pair ${index + 1}:`,
+      `  Entry A: ${pair.entry.content}`,
+      `  Entry B: ${pair.candidate.content}`
+    ].join("\n")
+  ).join("\n\n");
   return {
     systemPrompt,
     messages: [
       {
         role: "user",
-        content: userPrompt,
+        content: pairBlocks,
         timestamp: Date.now()
       }
     ],
-    tools: [LLM_DEDUP_TOOL]
+    tools: [LLM_DEDUP_BATCH_TOOL]
   };
 }
-function extractLlmDedupFromToolCall(message) {
+function extractLlmDedupBatchFromToolCall(message) {
   for (const block of message.content) {
-    if (block.type !== "toolCall" || block.name !== LLM_DEDUP_TOOL_NAME) {
+    if (block.type !== "toolCall" || block.name !== LLM_DEDUP_BATCH_TOOL_NAME) {
       continue;
     }
     const args = block.arguments;
-    if (!args || typeof args.same !== "boolean") {
+    if (!args || !Array.isArray(args.results)) {
       continue;
     }
-    return {
-      same: args.same,
-      reason: typeof args.reason === "string" ? args.reason : ""
-    };
+    const parsed = args.results.filter(
+      (result) => typeof result === "object" && result !== null && typeof result.pair === "number" && typeof result.same === "boolean"
+    ).map((result) => ({
+      pair: result.pair,
+      same: result.same,
+      reason: typeof result.reason === "string" ? result.reason : void 0
+    }));
+    return parsed;
   }
   return null;
 }
-async function llmDedupCheck(llmClient, entryA, entryB) {
+async function llmDedupCheckBatch(llmClient, pairs) {
+  if (pairs.length === 0) {
+    return [];
+  }
+  const fallback = Array.from({ length: pairs.length }, () => false);
   try {
-    const timeoutMs = 15e3;
+    const timeoutMs = 3e4;
+    let timer;
     const response = await Promise.race([
       runSimpleStream({
         model: llmClient.resolvedModel.model,
-        context: buildLlmDedupContext(entryA, entryB),
+        context: buildLlmDedupBatchContext(pairs),
         options: {
           apiKey: llmClient.credentials.apiKey
         },
         verbose: false
       }),
-      new Promise(
-        (_, reject) => setTimeout(() => reject(new Error("llmDedupCheck timed out")), timeoutMs)
-      )
-    ]);
+      new Promise((_, reject) => {
+        timer = setTimeout(() => reject(new Error("llmDedupCheckBatch timed out")), timeoutMs);
+      })
+    ]).finally(() => clearTimeout(timer));
     if (response.stopReason === "error" || response.errorMessage) {
-      return false;
+      return fallback;
+    }
+    const parsed = extractLlmDedupBatchFromToolCall(response);
+    if (!parsed) {
+      return fallback;
     }
-    const parsed = extractLlmDedupFromToolCall(response);
-    return parsed?.same === true;
+    const results = [...fallback];
+    for (const result of parsed) {
+      const pairIndex = Math.trunc(result.pair) - 1;
+      if (pairIndex < 0 || pairIndex >= results.length) {
+        continue;
+      }
+      results[pairIndex] = result.same;
+    }
+    return results;
   } catch {
-    return false;
+    return fallback;
   }
 }
+function chunkPairs(items, size) {
+  const chunks = [];
+  for (let i = 0; i < items.length; i += size) {
+    chunks.push(items.slice(i, i + size));
+  }
+  return chunks;
+}
 function parseDaysSince(value, now) {
   const parsed = new Date(value);
   if (Number.isNaN(parsed.getTime())) {
@@ -4421,7 +4513,9 @@ async function buildClusters(db, options = {}) {
   const entryById = new Map(candidates.map((entry) => [entry.id, entry]));
   const unionFind = new UnionFind();
   const looseUnionPairs = /* @__PURE__ */ new Set();
+  const llmDedupQueue = [];
   let llmDedupCalls = 0;
+  let llmDedupCheckedPairs = 0;
   let llmDedupMatches = 0;
   for (const entry of candidates) {
     unionFind.add(entry.id);
@@ -4450,13 +4544,47 @@ async function buildClusters(db, options = {}) {
       if (!llmClient) {
         continue;
       }
-      llmDedupCalls += 1;
-      const isSame = await llmDedupCheck(llmClient, entry, candidate);
-      if (isSame) {
-        llmDedupMatches += 1;
-        looseUnionPairs.add(key);
-        unionFind.union(entry.id, candidate.id);
+      llmDedupQueue.push({ entry, candidate, key });
+    }
+  }
+  if (llmClient && llmDedupQueue.length > 0) {
+    const startedAt = Date.now();
+    const batches = chunkPairs(llmDedupQueue, LLM_DEDUP_BATCH_SIZE);
+    for (let i = 0; i < batches.length; i += LLM_DEDUP_CONCURRENCY) {
+      const batchGroup = batches.slice(i, i + LLM_DEDUP_CONCURRENCY);
+      const settled = await Promise.allSettled(
+        batchGroup.map(
+          (batch) => llmDedupCheckBatch(
+            llmClient,
+            batch.map((pair) => ({
+              entry: pair.entry,
+              candidate: pair.candidate
+            }))
+          )
+        )
+      );
+      llmDedupCalls += batchGroup.length;
+      for (let batchIndex = 0; batchIndex < batchGroup.length; batchIndex += 1) {
+        const batch = batchGroup[batchIndex];
+        const result2 = settled[batchIndex];
+        const matches = result2 && result2.status === "fulfilled" ? result2.value : Array.from({ length: batch.length }, () => false);
+        llmDedupCheckedPairs += batch.length;
+        for (let pairIndex = 0; pairIndex < batch.length; pairIndex += 1) {
+          const pair = batch[pairIndex];
+          if (matches[pairIndex] !== true) {
+            continue;
+          }
+          llmDedupMatches += 1;
+          looseUnionPairs.add(pair.key);
+          unionFind.union(pair.entry.id, pair.candidate.id);
+        }
       }
+      const elapsedSeconds = Math.max(1, Math.floor((Date.now() - startedAt) / 1e3));
+      const remainingPairs = Math.max(0, llmDedupQueue.length - llmDedupCheckedPairs);
+      const estimatedRemainingSeconds = Math.round(elapsedSeconds / llmDedupCheckedPairs * remainingPairs);
+      onLog(
+        `[dedup] Checked ${llmDedupCheckedPairs}/${llmDedupQueue.length} pairs (${llmDedupMatches} matched) ~${estimatedRemainingSeconds}s remaining`
+      );
     }
   }
   const groups = /* @__PURE__ */ new Map();
@@ -7916,6 +8044,10 @@ async function mergeCluster(db, cluster, llmClient, apiKey, options = {}) {
       );
     }
   }
+  const subjectPreview = truncateContent(cluster.entries[0]?.subject ?? "", 60);
+  onLog(
+    `[merge-llm] Merging cluster of ${cluster.entries.length} entries (subject: "${subjectPreview || "unknown"}")`
+  );
   let mergeResult = null;
   try {
     const response = await runSimpleStream({
@@ -8376,6 +8508,41 @@ async function runFinalization(db, dryRun, onWarn, deps) {
 async function processPhaseClusters(params, deps) {
   const stats = defaultClusterStats();
   stats.clustersFound = params.clusters.length;
+  const onLog = params.options.onLog ?? (() => void 0);
+  const showLiveProgress = process.stderr.isTTY && params.options.verbose !== true;
+  const liveLineWidth = 120;
+  const phaseStartedAt = Date.now();
+  const formatEta = (ms) => {
+    const seconds = Math.max(0, Math.round(ms / 1e3));
+    if (seconds < 60) {
+      return `${seconds}s`;
+    }
+    const minutes = Math.floor(seconds / 60);
+    const remainingSeconds = seconds % 60;
+    return remainingSeconds === 0 ? `${minutes}m` : `${minutes}m ${remainingSeconds}s`;
+  };
+  const updateLiveProgress = (clusterIndex, totalClusters) => {
+    if (!showLiveProgress) {
+      return;
+    }
+    const completedClusters = clusterIndex - 1;
+    let etaSuffix = "";
+    if (completedClusters >= 2) {
+      const elapsedMs = Date.now() - phaseStartedAt;
+      const etaMs = (totalClusters - completedClusters) * (elapsedMs / completedClusters);
+      etaSuffix = ` ~${formatEta(etaMs)} remaining`;
+    }
+    process.stderr.write(
+      `\rPhase ${params.phase}: Processing cluster ${clusterIndex}/${totalClusters} (${params.type}) ...${etaSuffix}`
+    );
+  };
+  const clearLiveProgress = () => {
+    if (!showLiveProgress) {
+      return;
+    }
+    process.stderr.write("\r");
+    process.stderr.write(`${" ".repeat(liveLineWidth)}\r`);
+  };
   const pending = [];
   for (let i = 0; i < params.clusters.length; i += 1) {
     const cluster = params.clusters[i];
@@ -8395,10 +8562,14 @@ async function processPhaseClusters(params, deps) {
       params.context.batchReached = true;
       break;
     }
+    clearLiveProgress();
+    const clusterNumber = stats.clustersProcessed + 1;
+    onLog(`[phase ${params.phase}] Processing cluster ${clusterNumber}/${pending.length}...`);
+    updateLiveProgress(clusterNumber, pending.length);
     const outcome = await deps.mergeClusterFn(params.db, item.cluster, params.llmClient, params.embeddingApiKey, {
       dryRun: params.options.dryRun,
       verbose: params.options.verbose,
-      onLog: params.options.verbose ? params.options.onLog : void 0
+      onLog
     });
     stats.clustersProcessed += 1;
     stats.llmCalls += 1;
@@ -8425,6 +8596,7 @@ async function processPhaseClusters(params, deps) {
       break;
     }
   }
+  clearLiveProgress();
   return stats;
 }
 async function runConsolidationOrchestrator(db, dbPath, llmClient, embeddingApiKey, options = {}, deps = {}) {
@@ -8577,7 +8749,7 @@ async function runConsolidationOrchestrator(db, dbPath, llmClient, embeddingApiK
       skipBackup: projectIndex > 0,
       backupPath: projectIndex > 0 ? sharedBackupPath : void 0,
       skipOrphanCleanup: projectIndex > 0,
-      onLog: options.verbose ? onLog : void 0
+      onLog
     });
     if (projectIndex === 0) {
       sharedBackupPath = phase0Stats.backupPath;
@@ -8636,7 +8808,7 @@ async function runConsolidationOrchestrator(db, dbPath, llmClient, embeddingApiK
           looseThreshold: options.looseThreshold,
           idempotencyDays: options.idempotencyDays,
           verbose: options.verbose,
-          onLog: options.verbose ? onLog : void 0,
+          onLog,
           onStats: (stats) => {
             phase1ClusterStats = stats;
           }
@@ -8671,7 +8843,7 @@ async function runConsolidationOrchestrator(db, dbPath, llmClient, embeddingApiK
           looseThreshold: options.looseThreshold,
           idempotencyDays: options.idempotencyDays,
           verbose: options.verbose,
-          onLog: options.verbose ? onLog : void 0,
+          onLog,
           onStats: (stats) => {
             phase2ClusterStats = stats;
           }
@@ -8834,7 +9006,7 @@ async function runConsolidationOrchestrator(db, dbPath, llmClient, embeddingApiK
           looseThreshold: options.looseThreshold,
           idempotencyDays: 0,
           verbose: options.verbose,
-          onLog: options.verbose ? onLog : void 0,
+          onLog,
           onStats: (stats) => {
             phase3ClusterStats = stats;
           }
@@ -15865,7 +16037,7 @@ function parsePositiveInt2(value, fallback, label) {
 function formatMetric(value) {
   return value.toFixed(2);
 }
-function formatCount(value) {
+function formatCount2(value) {
   const rounded = Math.round(value);
   if (Math.abs(value - rounded) < 1e-9) {
     return String(rounded);
@@ -16294,7 +16466,7 @@ function renderSummary(result, rangesBySession) {
   for (const session of result.sessions) {
     const range = rangesBySession.get(session.session);
     const avgEntryCount = mean(session.runs.map((run) => run.total_entries));
-    const countLabel = range !== void 0 ? `${formatCount(avgEntryCount)}/${range.min}-${range.max}` : `${formatCount(avgEntryCount)}/-`;
+    const countLabel = range !== void 0 ? `${formatCount2(avgEntryCount)}/${range.min}-${range.max}` : `${formatCount2(avgEntryCount)}/-`;
     const passCount = session.runs.filter((run) => run.pass).length;
     lines.push(
       `${session.session.padEnd(sessionWidth)}  ${formatMetric(session.mean_recall).padEnd(6)}  ${formatMetric(session.mean_partial_recall).padEnd(8)}  ${formatMetric(session.mean_precision).padEnd(9)}  ${formatMetric(session.mean_composite).padEnd(9)}  ${countLabel.padEnd(9)}  ${passCount}/${result.runs}`

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agenr",
-  "version": "0.9.15",
+  "version": "0.9.17",
   "openclaw": {
     "extensions": [
       "dist/openclaw-plugin/index.js"