minimal-agent 0.11.0 → 0.12.0

package/src/memory/episodicMemory.js

@@ -1,21 +1,33 @@
 import { createHash, randomUUID } from 'node:crypto';
-import { open, readFile, readdir, stat } from 'node:fs/promises';
+import { open, readFile, readdir, stat, unlink } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
 import { agentStateDir } from '../context/agentHome.js';
 import { containsForbiddenSecret, sha256 } from './codec.js';
-import { appendPrivateLine, atomicWriteFile, ensurePrivateDir, pathExists, } from './fileStore.js';
+import { acquireFileLock, appendPrivateLine, atomicWriteFile, ensurePrivateDir, pathExists, } from './fileStore.js';
 import { memoryTerms } from './retrieval.js';
 import { BM25_DOCUMENT_THRESHOLD, Bm25Index, recordSearchIndexBuild, recordSearchIndexCacheHit, recordSearchIndexFallback, recordSearchStrategy, } from './searchIndex.js';
 const MAX_EPISODE_CHARS = 8_000;
 const MAX_SEARCH_FILES = 200;
 const MAX_SEARCH_LINES = 20_000;
 const MAX_SNIPPET_CHARS = 500;
+/**
+ * 已解析 episode 常驻内存的上限。
+ *
+ * ⚠️ `episodeFileCache` 缓存的是**整个文件解析出的全部 episode**，而日志本身是只增不减的
+ * （每条 user/assistant 消息 ≤8000 字符）。没有上限时，Desktop 这种长驻进程搜一次历史就把
+ * 几十 MB 对话原文永久钉在堆上。按 episode 条数做 LRU，逐出最久未用的文件。
+ */
+const MAX_CACHED_EPISODES = 20_000;
+/** 两次保留清理之间的最小间隔——清理是尽力而为的后台动作，不该每条消息都扫盘。 */
+const RETENTION_SWEEP_INTERVAL_MS = 5 * 60_000;
 const episodeFileCache = new Map();
 const episodeIndexCache = new Map();
 const episodeDiagnostics = {
     fullFileReads: 0,
     incrementalFileReads: 0,
     unchangedFileHits: 0,
+    cacheEvictions: 0,
+    retentionSweeps: 0,
 };
 export function episodeSearchDiagnostics() {
     return { ...episodeDiagnostics };
@@ -24,8 +36,35 @@ export function resetEpisodeSearchDiagnostics() {
     episodeDiagnostics.fullFileReads = 0;
     episodeDiagnostics.incrementalFileReads = 0;
     episodeDiagnostics.unchangedFileHits = 0;
+    episodeDiagnostics.cacheEvictions = 0;
+    episodeDiagnostics.retentionSweeps = 0;
     episodeFileCache.clear();
     episodeIndexCache.clear();
+    cacheClock = 0;
+    retentionSweepStates.clear();
+}
+let cacheClock = 0;
+const retentionSweepStates = new Map();
+const MAX_RETENTION_SWEEP_STATES = 128;
+function cachedEpisodeCount() {
+    let total = 0;
+    for (const entry of episodeFileCache.values())
+        total += entry.episodes.length;
+    return total;
+}
+/** 超出 MAX_CACHED_EPISODES 就按最久未用逐出整份文件缓存，直到回到上限之内。 */
+function evictEpisodeCache() {
+    if (cachedEpisodeCount() <= MAX_CACHED_EPISODES)
+        return;
+    const byAge = [...episodeFileCache.entries()].sort((a, b) => a[1].usedAt - b[1].usedAt);
+    for (const [path] of byAge) {
+        if (cachedEpisodeCount() <= MAX_CACHED_EPISODES)
+            break;
+        episodeFileCache.delete(path);
+        episodeDiagnostics.cacheEvictions += 1;
+    }
+    // 文件缓存变了，基于它建的 BM25 语料签名也就失效了。
+    episodeIndexCache.clear();
 }
 function episodeRoot() {
     return join(agentStateDir(), 'episodes');
@@ -43,6 +82,9 @@ function episodePath(workspaceId, sessionId) {
 function backfillMarkerPath(workspaceId) {
     return join(episodeRoot(), `.backfill-${validPart(workspaceId)}.json`);
 }
+function workspaceJournalLockPath(workspaceId) {
+    return join(episodeRoot(), `.workspace-${validPart(workspaceId)}.journal`);
+}
 export function sanitizeEpisodeContent(content) {
     const normalized = content
         .replace(/data:[^;\s]+;base64,[A-Za-z0-9+/=]{80,}/g, '[binary omitted]')
@@ -74,6 +116,219 @@ export async function appendConversationEpisode(input) {
     await appendPrivateLine(path, JSON.stringify(episode));
     return episode;
 }
+/** 删除某工作区（缺省为全部）的会话原文日志。供 Desktop「清空记忆」一并调用。 */
+export async function clearConversationEpisodes(workspaceId) {
+    const release = workspaceId
+        ? await acquireFileLock(workspaceJournalLockPath(workspaceId))
+        : null;
+    try {
+        return await clearConversationEpisodesUnlocked(workspaceId);
+    }
+    finally {
+        await release?.();
+    }
+}
+async function clearConversationEpisodesUnlocked(workspaceId) {
+    const files = await episodeFiles(workspaceId).catch(() => []);
+    let removed = 0;
+    for (const path of files) {
+        const result = await mutateEpisodeFile(path, () => ({ kind: 'delete' })).catch(() => null);
+        if (result?.changed)
+            removed += 1;
+    }
+    // ⚠️ backfill 标记必须**留下（没有就补写）**，绝不能删。
+    //    backfillLegacyHistory 会从 `sessions/*.json` 一次性导入旧会话；标记一旦消失，
+    //    下一次 SearchConversations 就会把刚刚清掉的对话原文整批重新导回来 —— 用户点了
+    //    "永久清空"，转头又能搜到，等于清空是假的。
+    if (workspaceId) {
+        await atomicWriteFile(backfillMarkerPath(workspaceId), `${JSON.stringify({ version: 1, workspaceId, sources: 0, completedAt: Date.now(), reason: 'cleared-by-user' }, null, 2)}\n`).catch(() => undefined);
+    }
+    episodeIndexCache.clear();
+    return removed;
+}
+function sameEpisodeFile(left, right) {
+    return left.ino === right.ino && left.size === right.size && left.mtimeMs === right.mtimeMs;
+}
+async function readStableEpisodeFile(path) {
+    for (let attempt = 0; attempt < 2; attempt += 1) {
+        const before = await stat(path).catch((error) => {
+            if (error.code === 'ENOENT')
+                return null;
+            throw error;
+        });
+        if (!before)
+            return null;
+        const raw = await readFile(path, 'utf8');
+        const after = await stat(path).catch(() => null);
+        if (after
+            && before.ino === after.ino
+            && before.size === after.size
+            && before.mtimeMs === after.mtimeMs
+            && Buffer.byteLength(raw, 'utf8') === after.size) {
+            return { raw, ino: after.ino, size: after.size, mtimeMs: after.mtimeMs };
+        }
+    }
+    return null;
+}
+/**
+ * 与 appendPrivateLine 共用同一把 `<episode>.lock`。大文件的解析在锁外完成，
+ * 获锁后只验证文件代际并提交；如果这期间有追加，就重读重算，绝不覆盖新行。
+ */
+async function mutateEpisodeFile(path, plan) {
+    for (let attempt = 0; attempt < 3; attempt += 1) {
+        const source = await readStableEpisodeFile(path);
+        if (!source)
+            return { changed: false, bytes: 0 };
+        const mutation = plan(source.raw);
+        if (mutation.kind === 'unchanged')
+            return { changed: false, bytes: source.size };
+        const release = await acquireFileLock(path);
+        try {
+            const live = await stat(path).catch(() => null);
+            if (!live || !sameEpisodeFile(source, live))
+                continue;
+            if (mutation.kind === 'delete')
+                await unlink(path);
+            else
+                await atomicWriteFile(path, mutation.content);
+            episodeFileCache.delete(path);
+            episodeIndexCache.clear();
+            return {
+                changed: true,
+                bytes: mutation.kind === 'write' ? Buffer.byteLength(mutation.content, 'utf8') : 0,
+            };
+        }
+        finally {
+            await release();
+        }
+    }
+    // 文件持续忙时跳过本轮维护；下次保留扫描会再处理。
+    const current = await stat(path).catch(() => null);
+    return { changed: false, bytes: current?.size ?? 0 };
+}
+/** 列出某工作区（或全部）的会话原文日志文件。 */
+async function episodeFiles(workspaceId) {
+    const prefix = workspaceId ? `workspace-${validPart(workspaceId)}--` : 'workspace-';
+    try {
+        return (await readdir(episodeRoot(), { withFileTypes: true }))
+            .filter((entry) => entry.isFile() && entry.name.endsWith('.jsonl') && entry.name.startsWith(prefix))
+            .map((entry) => join(episodeRoot(), entry.name));
+    }
+    catch (error) {
+        if (error.code === 'ENOENT')
+            return [];
+        throw error;
+    }
+}
+/** 按龄清理一个文件；返回清理后的字节数（文件被清空则删除并返回 0）。 */
+async function pruneFileByAge(path, cutoff) {
+    const result = await mutateEpisodeFile(path, (raw) => {
+        const lines = raw.split('\n').filter((line) => line.trim().length > 0);
+        const kept = lines.filter((line) => {
+            const episode = parseEpisode(line);
+            // 解析不出来的行按"保留"处理：清理是维护动作，不该顺手吞掉看不懂的数据。
+            return episode ? episode.createdAt >= cutoff : true;
+        });
+        if (kept.length === lines.length)
+            return { kind: 'unchanged' };
+        if (kept.length === 0)
+            return { kind: 'delete' };
+        return { kind: 'write', content: `${kept.join('\n')}\n` };
+    });
+    return result.bytes;
+}
+/**
+ * 会话原文保留策略：先按龄清理，仍超体积上限就从**最旧的文件**整份删起。
+ *
+ * 全程 best-effort：清理失败绝不能影响正在进行的对话（这是维护动作，不是功能）。
+ * 与 records 的删除语义严格区分——records 只有用户能删，这里删的是为检索而留的对话副本。
+ */
+export async function applyEpisodeRetention(workspaceId, retention, now = Date.now()) {
+    const files = await episodeFiles(workspaceId).catch(() => []);
+    if (!files.length)
+        return;
+    episodeDiagnostics.retentionSweeps += 1;
+    const cutoff = now - retention.episodeDays * 86_400_000;
+    const sized = [];
+    for (const path of files) {
+        const bytes = await pruneFileByAge(path, cutoff).catch(() => -1);
+        if (bytes <= 0)
+            continue;
+        const info = await stat(path).catch(() => null);
+        sized.push({ path, bytes, mtimeMs: info?.mtimeMs ?? 0 });
+    }
+    const limit = retention.episodeMaxMegabytes * 1024 * 1024;
+    let total = sized.reduce((sum, item) => sum + item.bytes, 0);
+    if (total <= limit)
+        return;
+    // 从最旧的文件开始按**条目**裁剪，而不是整份删除：
+    // 一个工作区常常只有一个长期会话文件，整份删会让全部历史在某次追加后突然消失（悬崖），
+    // 而按条目裁剪的结果是"最旧的对话先被丢掉"，与按龄清理的语义一致。
+    for (const item of sized.sort((a, b) => a.mtimeMs - b.mtimeMs)) {
+        if (total <= limit)
+            break;
+        const keepBytes = Math.max(0, item.bytes - (total - limit));
+        const remaining = await trimFileToNewestBytes(item.path, keepBytes).catch(() => item.bytes);
+        total -= item.bytes - remaining;
+    }
+    episodeIndexCache.clear();
+}
+/** 只保留文件末尾（最新）能放进 keepBytes 的整行；keepBytes 为 0 时删除文件。返回剩余字节数。 */
+async function trimFileToNewestBytes(path, keepBytes) {
+    const result = await mutateEpisodeFile(path, (raw) => {
+        if (keepBytes <= 0)
+            return { kind: 'delete' };
+        const lines = raw.split('\n').filter((line) => line.trim().length > 0);
+        const kept = [];
+        let bytes = 0;
+        for (let index = lines.length - 1; index >= 0; index -= 1) {
+            const size = Buffer.byteLength(`${lines[index]}\n`, 'utf8');
+            if (bytes + size > keepBytes)
+                break;
+            kept.unshift(lines[index]);
+            bytes += size;
+        }
+        if (!kept.length)
+            return { kind: 'delete' };
+        if (kept.length === lines.length)
+            return { kind: 'unchanged' };
+        return { kind: 'write', content: `${kept.join('\n')}\n` };
+    });
+    return result.bytes;
+}
+/**
+ * 是否该跑一次保留清理（同一进程内每 RETENTION_SWEEP_INTERVAL_MS 最多一次）。
+ *
+ * 本模块**不自己去读设置**：那需要 `import('./memoryService.js')`，而 memoryService 正好
+ * 静态 import 本模块，绕一圈会变成动态 import 的循环依赖——dev 下能跑，但 `bun --compile`
+ * 的绿色版打包期解析动态 import 的行为另算，不值得赌（见 GREEN-BUILD 相关教训）。
+ * 由知道设置的那一层（turnJournal）来问「该扫了吗」，本模块只保留纯粹的存储职责。
+ */
+export function shouldSweepRetention(workspaceId, now = Date.now()) {
+    const current = retentionSweepStates.get(workspaceId);
+    if (current?.inFlight || (current && now - current.lastStartedAt < RETENTION_SWEEP_INTERVAL_MS)) {
+        if (current)
+            current.lastUsedAt = now;
+        return false;
+    }
+    retentionSweepStates.set(workspaceId, { lastStartedAt: now, lastUsedAt: now, inFlight: true });
+    if (retentionSweepStates.size > MAX_RETENTION_SWEEP_STATES) {
+        const victims = [...retentionSweepStates.entries()]
+            .filter(([, state]) => !state.inFlight)
+            .sort((a, b) => a[1].lastUsedAt - b[1].lastUsedAt)
+            .slice(0, retentionSweepStates.size - MAX_RETENTION_SWEEP_STATES);
+        for (const [key] of victims)
+            retentionSweepStates.delete(key);
+    }
+    return true;
+}
+export function finishRetentionSweep(workspaceId) {
+    const state = retentionSweepStates.get(workspaceId);
+    if (state) {
+        state.inFlight = false;
+        state.lastUsedAt = Date.now();
+    }
+}
 function parseEpisode(line) {
     try {
         const value = JSON.parse(line);
@@ -120,10 +375,12 @@ async function readEpisodeBytes(path, offset, length) {
 async function loadEpisodeFile(path) {
     const info = await stat(path);
     const cached = episodeFileCache.get(path);
+    cacheClock += 1;
     if (cached
         && cached.ino === info.ino
         && cached.size === info.size
         && cached.mtimeMs === info.mtimeMs) {
+        cached.usedAt = cacheClock;
         episodeDiagnostics.unchangedFileHits += 1;
         return cached;
     }
@@ -136,9 +393,11 @@ async function loadEpisodeFile(path) {
             mtimeMs: info.mtimeMs,
             trailing: parsed.trailing,
             episodes: [...cached.episodes, ...parsed.episodes],
+            usedAt: cacheClock,
         };
         episodeFileCache.set(path, next);
         episodeDiagnostics.incrementalFileReads += 1;
+        evictEpisodeCache();
         return next;
     }
     const raw = await readFile(path, 'utf8');
@@ -149,9 +408,11 @@ async function loadEpisodeFile(path) {
         mtimeMs: info.mtimeMs,
         trailing: parsed.trailing,
         episodes: parsed.episodes,
+        usedAt: cacheClock,
     };
     episodeFileCache.set(path, next);
     episodeDiagnostics.fullFileReads += 1;
+    evictEpisodeCache();
     return next;
 }
 function persistedMessageText(message) {
@@ -201,6 +462,15 @@ async function workspaceArchiveDirs(workspaceId) {
     }
 }
 async function backfillLegacyHistory(workspaceId) {
+    const release = await acquireFileLock(workspaceJournalLockPath(workspaceId));
+    try {
+        await backfillLegacyHistoryUnlocked(workspaceId);
+    }
+    finally {
+        await release();
+    }
+}
+async function backfillLegacyHistoryUnlocked(workspaceId) {
     const marker = backfillMarkerPath(workspaceId);
     if (await pathExists(marker))
         return;
@@ -409,4 +679,6 @@ function conversationResult(episode, score) {
 export const episodicMemoryService = {
     append: appendConversationEpisode,
     search: searchConversationEpisodes,
+    clear: clearConversationEpisodes,
+    applyRetention: applyEpisodeRetention,
 };