agenr 0.9.16 → 0.9.17

package/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## 0.9.17 - 2026-02-27
+
+### Changed
+- Optimized LLM dedup in consolidate clustering: batch up to 10 pairs per API
+  call with 5 concurrent batches. Reduces a 2400-pair dedup queue from ~60min
+  (sequential, 1 call per pair) to ~2min.
+
 ## 0.9.16 - 2026-02-27
 
 ### Added

package/dist/cli-main.js

@@ -4254,15 +4254,26 @@ var DEFAULT_IDEMPOTENCY_DAYS = 7;
 var DEFAULT_NEIGHBOR_LIMIT = 20;
 var MAX_ACTIVE_EMBEDDED_ENTRIES2 = 2e4;
 var MILLISECONDS_PER_DAY = 1e3 * 60 * 60 * 24;
-var LLM_DEDUP_TOOL_NAME = "dedup_check";
+var LLM_DEDUP_BATCH_TOOL_NAME = "batch_dedup_check";
+var LLM_DEDUP_BATCH_SIZE = 10;
+var LLM_DEDUP_CONCURRENCY = 5;
 var LLM_DEDUP_TOOL_SCHEMA = Type.Object({
   same: Type.Boolean(),
   reason: Type.String()
 });
-var LLM_DEDUP_TOOL = {
-  name: LLM_DEDUP_TOOL_NAME,
-  description: "Decide whether two knowledge entries express the same knowledge.",
-  parameters: LLM_DEDUP_TOOL_SCHEMA
+var LLM_DEDUP_BATCH_TOOL_SCHEMA = Type.Object({
+  results: Type.Array(
+    Type.Object({
+      pair: Type.Number(),
+      same: Type.Boolean(),
+      reason: Type.Optional(Type.String())
+    })
+  )
+});
+var LLM_DEDUP_BATCH_TOOL = {
+  name: LLM_DEDUP_BATCH_TOOL_NAME,
+  description: "Return dedup results for all pairs",
+  parameters: LLM_DEDUP_BATCH_TOOL_SCHEMA
 };
 function toNumber5(value) {
   if (typeof value === "number") {
@@ -4303,70 +4314,98 @@ function normalizeSubject2(value) {
 function pairKey(a, b) {
   return a < b ? `${a}|${b}` : `${b}|${a}`;
 }
-function buildLlmDedupContext(entryA, entryB) {
+function buildLlmDedupBatchContext(pairs) {
   const systemPrompt = [
     "You are a deduplication assistant for knowledge entries.",
-    "Decide if two entries express the same knowledge in different wording.",
-    "Return your answer by calling dedup_check."
-  ].join("\n");
-  const userPrompt = [
-    "Are these two knowledge entries expressing the same fact or genuinely distinct?",
-    `Entry A: ${entryA.content}`,
-    `Entry B: ${entryB.content}`,
-    'Set "same" to true only when both entries represent the same knowledge.'
+    "For each numbered pair, decide if they express the same knowledge.",
+    "Call batch_dedup_check once with your results."
   ].join("\n");
+  const pairBlocks = pairs.map(
+    (pair, index) => [
+      `Pair ${index + 1}:`,
+      `  Entry A: ${pair.entry.content}`,
+      `  Entry B: ${pair.candidate.content}`
+    ].join("\n")
+  ).join("\n\n");
   return {
     systemPrompt,
     messages: [
       {
         role: "user",
-        content: userPrompt,
+        content: pairBlocks,
         timestamp: Date.now()
       }
     ],
-    tools: [LLM_DEDUP_TOOL]
+    tools: [LLM_DEDUP_BATCH_TOOL]
   };
 }
-function extractLlmDedupFromToolCall(message) {
+function extractLlmDedupBatchFromToolCall(message) {
   for (const block of message.content) {
-    if (block.type !== "toolCall" || block.name !== LLM_DEDUP_TOOL_NAME) {
+    if (block.type !== "toolCall" || block.name !== LLM_DEDUP_BATCH_TOOL_NAME) {
       continue;
     }
     const args = block.arguments;
-    if (!args || typeof args.same !== "boolean") {
+    if (!args || !Array.isArray(args.results)) {
       continue;
     }
-    return {
-      same: args.same,
-      reason: typeof args.reason === "string" ? args.reason : ""
-    };
+    const parsed = args.results.filter(
+      (result) => typeof result === "object" && result !== null && typeof result.pair === "number" && typeof result.same === "boolean"
+    ).map((result) => ({
+      pair: result.pair,
+      same: result.same,
+      reason: typeof result.reason === "string" ? result.reason : void 0
+    }));
+    return parsed;
   }
   return null;
 }
-async function llmDedupCheck(llmClient, entryA, entryB) {
+async function llmDedupCheckBatch(llmClient, pairs) {
+  if (pairs.length === 0) {
+    return [];
+  }
+  const fallback = Array.from({ length: pairs.length }, () => false);
   try {
-    const timeoutMs = 15e3;
+    const timeoutMs = 3e4;
+    let timer;
     const response = await Promise.race([
       runSimpleStream({
         model: llmClient.resolvedModel.model,
-        context: buildLlmDedupContext(entryA, entryB),
+        context: buildLlmDedupBatchContext(pairs),
         options: {
           apiKey: llmClient.credentials.apiKey
         },
         verbose: false
       }),
-      new Promise(
-        (_, reject) => setTimeout(() => reject(new Error("llmDedupCheck timed out")), timeoutMs)
-      )
-    ]);
+      new Promise((_, reject) => {
+        timer = setTimeout(() => reject(new Error("llmDedupCheckBatch timed out")), timeoutMs);
+      })
+    ]).finally(() => clearTimeout(timer));
     if (response.stopReason === "error" || response.errorMessage) {
-      return false;
+      return fallback;
+    }
+    const parsed = extractLlmDedupBatchFromToolCall(response);
+    if (!parsed) {
+      return fallback;
+    }
+    const results = [...fallback];
+    for (const result of parsed) {
+      const pairIndex = Math.trunc(result.pair) - 1;
+      if (pairIndex < 0 || pairIndex >= results.length) {
+        continue;
+      }
+      results[pairIndex] = result.same;
     }
-    const parsed = extractLlmDedupFromToolCall(response);
-    return parsed?.same === true;
+    return results;
   } catch {
-    return false;
+    return fallback;
+  }
+}
+function chunkPairs(items, size) {
+  const chunks = [];
+  for (let i = 0; i < items.length; i += size) {
+    chunks.push(items.slice(i, i + size));
   }
+  return chunks;
 }
 function parseDaysSince(value, now) {
   const parsed = new Date(value);
@@ -4476,6 +4515,7 @@ async function buildClusters(db, options = {}) {
   const looseUnionPairs = /* @__PURE__ */ new Set();
   const llmDedupQueue = [];
   let llmDedupCalls = 0;
+  let llmDedupCheckedPairs = 0;
   let llmDedupMatches = 0;
   for (const entry of candidates) {
     unionFind.add(entry.id);
@@ -4507,18 +4547,45 @@ async function buildClusters(db, options = {}) {
       llmDedupQueue.push({ entry, candidate, key });
     }
   }
-  for (const pair of llmDedupQueue) {
-    if (!llmClient) {
-      break;
-    }
-    llmDedupCalls += 1;
-    const isSame = await llmDedupCheck(llmClient, pair.entry, pair.candidate);
-    if (isSame) {
-      llmDedupMatches += 1;
-      looseUnionPairs.add(pair.key);
-      unionFind.union(pair.entry.id, pair.candidate.id);
+  if (llmClient && llmDedupQueue.length > 0) {
+    const startedAt = Date.now();
+    const batches = chunkPairs(llmDedupQueue, LLM_DEDUP_BATCH_SIZE);
+    for (let i = 0; i < batches.length; i += LLM_DEDUP_CONCURRENCY) {
+      const batchGroup = batches.slice(i, i + LLM_DEDUP_CONCURRENCY);
+      const settled = await Promise.allSettled(
+        batchGroup.map(
+          (batch) => llmDedupCheckBatch(
+            llmClient,
+            batch.map((pair) => ({
+              entry: pair.entry,
+              candidate: pair.candidate
+            }))
+          )
+        )
+      );
+      llmDedupCalls += batchGroup.length;
+      for (let batchIndex = 0; batchIndex < batchGroup.length; batchIndex += 1) {
+        const batch = batchGroup[batchIndex];
+        const result2 = settled[batchIndex];
+        const matches = result2 && result2.status === "fulfilled" ? result2.value : Array.from({ length: batch.length }, () => false);
+        llmDedupCheckedPairs += batch.length;
+        for (let pairIndex = 0; pairIndex < batch.length; pairIndex += 1) {
+          const pair = batch[pairIndex];
+          if (matches[pairIndex] !== true) {
+            continue;
+          }
+          llmDedupMatches += 1;
+          looseUnionPairs.add(pair.key);
+          unionFind.union(pair.entry.id, pair.candidate.id);
+        }
+      }
+      const elapsedSeconds = Math.max(1, Math.floor((Date.now() - startedAt) / 1e3));
+      const remainingPairs = Math.max(0, llmDedupQueue.length - llmDedupCheckedPairs);
+      const estimatedRemainingSeconds = Math.round(elapsedSeconds / llmDedupCheckedPairs * remainingPairs);
+      onLog(
+        `[dedup] Checked ${llmDedupCheckedPairs}/${llmDedupQueue.length} pairs (${llmDedupMatches} matched) ~${estimatedRemainingSeconds}s remaining`
+      );
     }
-    onLog(`[dedup] Checked ${llmDedupCalls}/${llmDedupQueue.length} pairs (${llmDedupMatches} matched)`);
   }
   const groups = /* @__PURE__ */ new Map();
   for (const entry of candidates) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agenr",
-  "version": "0.9.16",
+  "version": "0.9.17",
   "openclaw": {
     "extensions": [
       "dist/openclaw-plugin/index.js"