deeper-cli 1.2.4 → 1.3.0

package/dist/cli/index.js

@@ -1131,44 +1131,100 @@ function uid() {
 function ensureDir() {
   if (!existsSync5(MEM_DIR)) mkdirSync4(MEM_DIR, { recursive: true });
   if (!existsSync5(STATS_FILE)) {
-    writeFileSync3(STATS_FILE, JSON.stringify({ totalEntries: 0, byType: {}, totalTokens: 0, lastCleanup: Date.now() }), "utf-8");
+    writeFileSync3(STATS_FILE, JSON.stringify({ totalEntries: 0, byType: {}, totalTokens: 0, lastCleanup: Date.now(), lastConsolidate: Date.now() }, null, 2), "utf-8");
   }
 }
 function loadStats() {
   try {
     return JSON.parse(readFileSync4(STATS_FILE, "utf-8"));
   } catch {
-    return { totalEntries: 0, byType: {}, totalTokens: 0, lastCleanup: 0 };
+    return { totalEntries: 0, byType: {}, totalTokens: 0, lastCleanup: 0, lastConsolidate: 0 };
   }
 }
 function saveStats(s) {
   writeFileSync3(STATS_FILE, JSON.stringify(s, null, 2), "utf-8");
 }
-var MEM_DIR, STATS_FILE, MAX_WORKING_MEM, MAX_TOTAL_MEM, CLEANUP_THRESHOLD, currentSessionId, XMemory, xmemory;
+function tokenize(text) {
+  const lower = text.toLowerCase();
+  const words = lower.split(/[\s\p{P}\p{S}]+/).filter((w) => w.length >= 2);
+  const bigrams = [];
+  for (let i = 0; i < words.length - 1; i++) bigrams.push(words[i] + "_" + words[i + 1]);
+  return [.../* @__PURE__ */ new Set([...words, ...bigrams])];
+}
+function similarity(a, b2) {
+  if (a === b2) return 1;
+  if (!a || !b2) return 0;
+  const ta = tokenize(a);
+  const tb = tokenize(b2);
+  if (ta.length === 0 || tb.length === 0) return 0;
+  const sa = new Set(ta);
+  const sb = new Set(tb);
+  let intersection = 0;
+  for (const t of sa) if (sb.has(t)) intersection++;
+  const union = Math.max(sa.size, sb.size);
+  return union > 0 ? intersection / union : 0;
+}
+var MEM_DIR, STATS_FILE, MAX_WORKING_MEM, MAX_TOTAL_MEM, CLEANUP_THRESHOLD, DEDUP_SIMILARITY_THRESHOLD, CONSOLIDATE_THRESHOLD, SAVE_DEBOUNCE_MS, currentSessionId, TFIDFIndexer, XMemory, xmemory;
 var init_xmemory = __esm({
   "src/memory/xmemory.ts"() {
     "use strict";
     init_constants();
     MEM_DIR = join4(DEEPER_HOME, "xmemory");
     STATS_FILE = join4(MEM_DIR, "stats.json");
-    MAX_WORKING_MEM = 50;
-    MAX_TOTAL_MEM = 2e3;
-    CLEANUP_THRESHOLD = 1500;
+    MAX_WORKING_MEM = 100;
+    MAX_TOTAL_MEM = 3e3;
+    CLEANUP_THRESHOLD = 2e3;
+    DEDUP_SIMILARITY_THRESHOLD = 0.85;
+    CONSOLIDATE_THRESHOLD = 5;
+    SAVE_DEBOUNCE_MS = 3e3;
     currentSessionId = "";
+    TFIDFIndexer = class {
+      docFreq = /* @__PURE__ */ new Map();
+      numDocs = 0;
+      build(entries) {
+        this.docFreq.clear();
+        this.numDocs = 0;
+        for (const entry of entries) {
+          this.numDocs++;
+          const tokens = new Set(tokenize(entry.content));
+          for (const t of tokens) this.docFreq.set(t, (this.docFreq.get(t) || 0) + 1);
+        }
+      }
+      idf(term) {
+        const df = this.docFreq.get(term) || 0;
+        if (df === 0 || df >= this.numDocs) return 0;
+        return Math.log((this.numDocs + 0.5) / (df + 0.5)) + 1;
+      }
+      tfidfScore(content, queryTerms) {
+        const tokens = tokenize(content);
+        const tfMap = /* @__PURE__ */ new Map();
+        for (const t of tokens) tfMap.set(t, (tfMap.get(t) || 0) + 1);
+        const maxTf = Math.max(...tfMap.values(), 1);
+        let score = 0;
+        for (const qt of queryTerms) {
+          const tf = (tfMap.get(qt) || 0) / maxTf * (1 + Math.log(tokens.length));
+          score += tf * this.idf(qt);
+        }
+        return score;
+      }
+    };
     XMemory = class {
       working = [];
       index = /* @__PURE__ */ new Map();
       dirty = false;
+      saveTimer = null;
+      tfidf = new TFIDFIndexer();
       constructor() {
         ensureDir();
       }
-      // ============ 写入 ============
       store(type, content, tags = [], importance = 5, accuracy = 7, source = "agent", references = []) {
+        const deduped = this.deduplicate(type, content, importance);
+        if (deduped !== null) return deduped;
         const id = uid();
         const entry = {
           id,
           type,
-          content: content.slice(0, 2e3),
+          content: content.slice(0, 4e3),
           tags,
           importance: Math.min(10, Math.max(0, importance)),
           accuracy: Math.min(10, Math.max(0, accuracy)),
@@ -1181,12 +1237,11 @@ var init_xmemory = __esm({
         };
         if (type === "working") {
           this.working.push(entry);
-          if (this.working.length > MAX_WORKING_MEM) {
-            this.working.shift();
-          }
+          this.evictWorking();
         }
         this.index.set(id, entry);
         this.dirty = true;
+        this.scheduleSave();
         if (this.index.size > CLEANUP_THRESHOLD) {
           this.autoCleanup();
         }
@@ -1204,31 +1259,41 @@ var init_xmemory = __esm({
       storeWorking(content, tags = []) {
         return this.store("working", content, tags, 3, 5, "agent");
       }
-      // ============ 检索 ============
       recall(query, limit = 5, minImportance = 0) {
-        const keywords = query.toLowerCase().split(/[\s,，。]+/).filter((w) => w.length > 1);
+        const keywords = tokenize(query).filter((w) => w.length >= 2);
         if (keywords.length === 0) return [];
-        const scored = [];
+        this.tfidf.build(this.index.values());
         const now = Date.now();
+        const scored = [];
         for (const entry of this.index.values()) {
           if (entry.importance < minImportance) continue;
-          let score = 0;
-          const c2 = (entry.content || "").toLowerCase();
+          const c2 = entry.content.toLowerCase();
           const t = entry.tags.join(" ").toLowerCase();
+          let keywordScore = 0;
+          let exactMatchBonus = 0;
           for (const kw of keywords) {
-            if (c2.includes(kw)) score += 3;
-            if (t.includes(kw)) score += 2;
-            if (entry.type === "procedural" && c2.includes(kw)) score += 1;
-          }
-          const age = (now - entry.accessedAt) / (1e3 * 60 * 60);
-          score += entry.importance * 0.5;
-          score -= Math.min(age / 24, 5);
-          if (score > 0) scored.push({ entry, score });
+            if (c2.includes(kw)) {
+              keywordScore += 3;
+              exactMatchBonus += 1;
+            }
+            if (t.includes(kw)) keywordScore += 2;
+            if (entry.type === "procedural" && c2.includes(kw)) keywordScore += 1;
+            if (entry.type === "semantic" && c2.includes(kw)) keywordScore += 1.5;
+          }
+          const tfidfScore = this.tfidf.tfidfScore(entry.content, keywords);
+          const recencyBoost = Math.max(0, 3 - (now - entry.accessedAt) / (1e3 * 60 * 30));
+          const frequencyBoost = Math.min(entry.accessCount * 0.3, 4);
+          const importanceWeight = entry.importance * 0.6;
+          const ageHours = (now - entry.createdAt) / (1e3 * 60 * 60);
+          const ageDecay = Math.min(ageHours / 48, 6);
+          const totalScore = keywordScore * 1 + tfidfScore * 2 + importanceWeight + recencyBoost + frequencyBoost - ageDecay + (exactMatchBonus >= 3 ? 3 : 0);
+          if (totalScore > 0) scored.push({ entry, score: totalScore });
         }
         scored.sort((a, b2) => b2.score - a.score);
         return scored.slice(0, limit).map((s) => {
           s.entry.accessedAt = now;
           s.entry.accessCount++;
+          s.entry.importance = Math.min(10, s.entry.importance + 0.1);
           return s.entry;
         });
       }
@@ -1237,18 +1302,21 @@ var init_xmemory = __esm({
         for (const entry of this.index.values()) {
           if (entry.type === type) res.push(entry);
         }
-        res.sort((a, b2) => b2.createdAt - a.createdAt);
+        res.sort((a, b2) => b2.importance * 2 + b2.accessCount - (a.importance * 2 + a.accessCount));
         return res.slice(0, limit);
       }
       getWorking() {
-        return this.working;
+        return this.working.sort(
+          (a, b2) => b2.importance * 2 + b2.accessCount - (a.importance * 2 + a.accessCount)
+        );
       }
-      getWorkingContext(maxTokens = 2e3) {
+      getWorkingContext(maxTokens = 4e3) {
         if (this.working.length === 0) return "";
-        const lines = this.working.map((e) => `[\u5DE5\u4F5C\u8BB0\u5FC6] ${e.content.slice(0, 300)}`);
+        const sorted = this.getWorking();
+        const lines = sorted.map((e) => `[${e.source}] ${e.content.slice(0, 500)}`);
         let result = "";
         for (const line of lines) {
-          if ((result + line).length > maxTokens * 4) break;
+          if ((result + line).length > maxTokens * 3) break;
           result += line + "\n";
         }
         return result;
@@ -1257,19 +1325,34 @@ var init_xmemory = __esm({
         const recalled = this.recall(task, limit, 3);
         const proc = recalled.filter((r2) => r2.type === "procedural" || r2.type === "semantic");
         if (proc.length === 0) return "";
-        return "[\u8BB0\u5FC6\u63D0\u793A]\n" + proc.map((p) => `- ${p.content.slice(0, 250)}`).join("\n");
+        return "[\u8BB0\u5FC6\u63D0\u793A]\n" + proc.map((p) => `- [${p.type === "procedural" ? "\u6280\u80FD" : "\u77E5\u8BC6"}] ${p.content.slice(0, 350)}`).join("\n");
       }
       getSessionSummary() {
         const entries = [...this.index.values()].filter((e) => e.sessionId === currentSessionId);
         if (entries.length === 0) return "";
-        const key = entries.filter((e) => e.importance >= 5).sort((a, b2) => b2.importance - a.importance).slice(0, 8);
-        return `[XMemory\xB7\u672C\u4F1A\u8BDD]
-` + key.map((e) => {
-          const t = e.type === "semantic" ? "\u77E5\u8BC6" : e.type === "procedural" ? "\u6280\u80FD" : e.type === "episodic" ? "\u7ECF\u5386" : "\u5DE5\u4F5C";
-          return `[${t}] ${e.content.slice(0, 200)}`;
-        }).join("\n");
-      }
-      // ============ 持久化 ============
+        const byType = { semantic: [], procedural: [], episodic: [], working: [] };
+        for (const e of entries) {
+          if (byType[e.type]) byType[e.type].push(e);
+        }
+        const parts = [];
+        const keySemantic = byType.semantic.sort((a, b2) => b2.importance - a.importance).slice(0, 5);
+        if (keySemantic.length > 0) {
+          parts.push(`[\u77E5\u8BC6\u8BB0\u5FC6\xB7${keySemantic.length}\u6761]
+` + keySemantic.map((e) => `\u2022 ${e.content.slice(0, 250)}`).join("\n"));
+        }
+        const keyProcedural = byType.procedural.sort((a, b2) => b2.importance - a.importance).slice(0, 5);
+        if (keyProcedural.length > 0) {
+          parts.push(`[\u6280\u80FD\u8BB0\u5FC6\xB7${keyProcedural.length}\u6761]
+` + keyProcedural.map((e) => `\u2022 ${e.content.slice(0, 250)}`).join("\n"));
+        }
+        const keyEpisodic = byType.episodic.sort((a, b2) => b2.importance - a.importance).slice(0, 3);
+        if (keyEpisodic.length > 0) {
+          parts.push(`[\u7ECF\u5386\u8BB0\u5FC6\xB7${keyEpisodic.length}\u6761]
+` + keyEpisodic.map((e) => `\u2192 ${e.content.slice(0, 150)}`).join("\n"));
+        }
+        return `[XMemory\xB7\u672C\u4F1A\u8BDD ${entries.length}\u6761]
+${parts.join("\n")}`;
+      }
       async save() {
         if (!this.dirty) return;
         ensureDir();
@@ -1280,7 +1363,7 @@ var init_xmemory = __esm({
         }
         for (const [type, entries] of Object.entries(byType)) {
           const file = join4(MEM_DIR, `${type}.json`);
-          writeFileSync3(file, JSON.stringify(entries.slice(-500)), "utf-8");
+          writeFileSync3(file, JSON.stringify(entries.slice(-800)), "utf-8");
         }
         const stats = loadStats();
         stats.totalEntries = this.index.size;
@@ -1312,16 +1395,112 @@ var init_xmemory = __esm({
         if (this.working.length > MAX_WORKING_MEM) {
           this.working = this.working.slice(-MAX_WORKING_MEM);
         }
+        this.consolidateIfNeeded();
+      }
+      scheduleSave() {
+        if (this.saveTimer) return;
+        this.saveTimer = setTimeout(() => {
+          this.saveTimer = null;
+          this.save().catch(() => {
+          });
+        }, SAVE_DEBOUNCE_MS);
+      }
+      deduplicate(type, content, importance) {
+        const threshold = type === "working" ? 0.95 : type === "episodic" ? 0.85 : DEDUP_SIMILARITY_THRESHOLD;
+        const shortContent = content.slice(0, 500);
+        for (const entry of this.index.values()) {
+          if (entry.type !== type) continue;
+          const sim = similarity(shortContent, entry.content.slice(0, 500));
+          if (sim >= threshold) {
+            entry.accessedAt = Date.now();
+            entry.accessCount++;
+            entry.importance = Math.min(10, Math.max(entry.importance, importance));
+            if (content.length > entry.content.length && content.length <= 4e3) {
+              entry.content = content;
+              this.dirty = true;
+            }
+            return entry.id;
+          }
+        }
+        return null;
+      }
+      evictWorking() {
+        while (this.working.length > MAX_WORKING_MEM) {
+          let worstIdx = 0;
+          let worstScore = Infinity;
+          for (let i = 0; i < this.working.length; i++) {
+            const e = this.working[i];
+            const score = e.importance + e.accessCount * 0.3 - (Date.now() - e.accessedAt) / (1e3 * 60 * 15);
+            if (score < worstScore) {
+              worstScore = score;
+              worstIdx = i;
+            }
+          }
+          this.working.splice(worstIdx, 1);
+        }
+      }
+      consolidateIfNeeded() {
+        const stats = loadStats();
+        const now = Date.now();
+        if (now - stats.lastConsolidate < 24 * 60 * 60 * 1e3) return;
+        const candidates = /* @__PURE__ */ new Map();
+        for (const entry of this.index.values()) {
+          if (entry.type === "episodic" && entry.accessCount >= CONSOLIDATE_THRESHOLD) {
+            const key = entry.tags.slice(0, 3).sort().join("|") || "_default_";
+            if (!candidates.has(key)) candidates.set(key, []);
+            candidates.get(key).push(entry);
+          }
+        }
+        let consolidated = 0;
+        for (const [, group] of candidates) {
+          if (group.length < 2) continue;
+          group.sort((a, b2) => b2.accessCount - a.accessCount);
+          const best = group[0];
+          const allContent = group.map((g2) => g2.content).join("\n");
+          const summary = `[ consolidated from ${group.length} memories ]
+${allContent.slice(0, 2e3)}`;
+          const newType = best.accessCount >= 10 ? "procedural" : "semantic";
+          const existing = this.findSimilar(newType, best.content);
+          if (existing) {
+            existing.content = summary.slice(0, 4e3);
+            existing.accessCount += Math.floor(group.reduce((s, g2) => s + g2.accessCount, 0) / group.length);
+            existing.importance = Math.min(10, existing.importance + 1);
+          } else {
+            this.store(newType, summary, best.tags, Math.min(10, best.importance + 2), 9, "system");
+            consolidated++;
+          }
+          for (const old of group) {
+            if (old.id !== best.id) this.index.delete(old.id);
+          }
+        }
+        if (consolidated > 0 || candidates.size > 0) {
+          stats.lastConsolidate = now;
+          saveStats(stats);
+          this.dirty = true;
+        }
+      }
+      findSimilar(type, content) {
+        const threshold = 0.75;
+        let best = null;
+        let bestSim = 0;
+        for (const entry of this.index.values()) {
+          if (entry.type !== type) continue;
+          const sim = similarity(content.slice(0, 300), entry.content.slice(0, 300));
+          if (sim > bestSim) {
+            bestSim = sim;
+            best = entry;
+          }
+        }
+        return bestSim >= threshold ? best : null;
       }
-      // ============ 维护 ============
       autoCleanup() {
         const entries = [...this.index.values()];
         entries.sort((a, b2) => {
-          const scoreA = a.importance * 2 + a.accessCount - (Date.now() - a.accessedAt) / (1e3 * 60 * 60 * 24);
-          const scoreB = b2.importance * 2 + b2.accessCount - (Date.now() - b2.accessedAt) / (1e3 * 60 * 60 * 24);
+          const scoreA = a.importance * 2 + a.accessCount + (a.type === "semantic" ? 3 : a.type === "procedural" ? 2 : 0) - (Date.now() - a.accessedAt) / (1e3 * 60 * 60 * 24);
+          const scoreB = b2.importance * 2 + b2.accessCount + (b2.type === "semantic" ? 3 : b2.type === "procedural" ? 2 : 0) - (Date.now() - b2.accessedAt) / (1e3 * 60 * 60 * 24);
           return scoreA - scoreB;
         });
-        const toKeep = entries.slice(-500);
+        const toKeep = entries.slice(-800);
         this.index.clear();
         this.working = [];
         for (const entry of toKeep) {
@@ -3843,6 +4022,11 @@ var init_DeepSeekClient = __esm({
             const rc = m.reasoning_content || m.thinking;
             if (rc) msg.reasoning_content = rc;
             return msg;
+          }).map((msg) => {
+            if (msg.role === "tool" && !msg.name) {
+              msg.name = "tool";
+            }
+            return msg;
           }),
           temperature: config.temperature,
           max_tokens: config.maxTokens,
@@ -12450,7 +12634,10 @@ ${globalRules}` });
     }
     r2.push(e);
   }
-  return r2;
+  return r2.map((m) => {
+    if (m.role === "tool" && !m.name) m.name = "tool";
+    return m;
+  });
 }
 function trimHistory(h, max) {
   while (h.length > max) h.shift();