@downcity/agent 1.1.7 → 1.1.10

package/src/service/builtins/memory/runtime/Search.ts

@@ -2,26 +2,219 @@
  * Memory Search 运行时。
  *
  * 关键点（中文）
- * - 收敛检索默认参数与容错返回。
- * - 检索前确保索引同步（dirty 时自动补齐）。
+ * - 直接扫描 Markdown 文件，不依赖额外索引库。
+ * - 统一收敛检索、分块、打分与状态统计逻辑。
  */
 
+import fs from "node:fs/promises";
 import type { AgentContext } from "@/agent/AgentContextTypes.js";
 import type {
   MemorySearchPayload,
   MemorySearchResponse,
+  MemorySearchResultItem,
+  MemorySourceStat,
+  MemoryStatusResponse,
 } from "@/service/builtins/memory/types/Memory.js";
 import {
+  listMemorySourceFiles,
   MEMORY_DEFAULTS,
-  ensureMemoryIndexed,
   type MemoryRuntimeState,
 } from "./Store.js";
 
+const SNIPPET_MAX_CHARS = 700;
+const CHUNK_MAX_CHARS = 1600;
+const CHUNK_OVERLAP_CHARS = 240;
+
 function clampNumber(value: number, min: number, max: number): number {
   if (!Number.isFinite(value)) return min;
   return Math.max(min, Math.min(max, value));
 }
 
+function normalizeText(input: string): string {
+  return String(input || "").replace(/\r\n/g, "\n");
+}
+
+function truncateText(value: string, maxChars: number): string {
+  if (value.length <= maxChars) {
+    return value;
+  }
+  return value.slice(0, maxChars);
+}
+
+function tokenizeQuery(raw: string): string[] {
+  return String(raw || "")
+    .toLowerCase()
+    .replace(/[^\p{L}\p{N}_-]+/gu, " ")
+    .split(/\s+/)
+    .map((item) => item.trim())
+    .filter(Boolean)
+    .slice(0, 12);
+}
+
+function countTokenOccurrences(text: string, token: string): number {
+  if (!token) return 0;
+  let hits = 0;
+  let start = 0;
+  while (start < text.length) {
+    const index = text.indexOf(token, start);
+    if (index < 0) break;
+    hits += 1;
+    start = index + token.length;
+  }
+  return hits;
+}
+
+function buildSnippetScore(text: string, tokens: string[]): number {
+  if (tokens.length === 0) return 0;
+  const normalized = normalizeText(text).toLowerCase();
+  let matchedTokens = 0;
+  let totalHits = 0;
+  for (const token of tokens) {
+    const hits = countTokenOccurrences(normalized, token);
+    if (hits > 0) {
+      matchedTokens += 1;
+      totalHits += Math.min(hits, 4);
+    }
+  }
+  if (matchedTokens === 0) {
+    return 0;
+  }
+  const coverageScore = matchedTokens / tokens.length;
+  const densityScore = Math.min(totalHits, tokens.length * 3) / (tokens.length * 3);
+  return Number((coverageScore * 0.75 + densityScore * 0.25).toFixed(4));
+}
+
+function chunkMarkdown(content: string): Array<{
+  startLine: number;
+  endLine: number;
+  text: string;
+}> {
+  const lines = normalizeText(content).split("\n");
+  if (lines.length === 0) {
+    return [];
+  }
+  const out: Array<{
+    startLine: number;
+    endLine: number;
+    text: string;
+  }> = [];
+  let bucket: Array<{ line: string; lineNo: number }> = [];
+  let chars = 0;
+
+  const flush = () => {
+    if (bucket.length === 0) return;
+    const startLine = bucket[0]?.lineNo ?? 1;
+    const endLine = bucket[bucket.length - 1]?.lineNo ?? startLine;
+    const text = bucket.map((item) => item.line).join("\n").trim();
+    if (!text) return;
+    out.push({
+      startLine,
+      endLine,
+      text,
+    });
+  };
+
+  const carryOverlap = () => {
+    if (bucket.length === 0 || CHUNK_OVERLAP_CHARS <= 0) {
+      bucket = [];
+      chars = 0;
+      return;
+    }
+    let acc = 0;
+    const next: Array<{ line: string; lineNo: number }> = [];
+    for (let i = bucket.length - 1; i >= 0; i -= 1) {
+      const row = bucket[i];
+      if (!row) continue;
+      acc += row.line.length + 1;
+      next.unshift(row);
+      if (acc >= CHUNK_OVERLAP_CHARS) break;
+    }
+    bucket = next;
+    chars = bucket.reduce((sum, item) => sum + item.line.length + 1, 0);
+  };
+
+  for (let i = 0; i < lines.length; i += 1) {
+    const line = lines[i] ?? "";
+    const rowSize = line.length + 1;
+    if (bucket.length > 0 && chars + rowSize > CHUNK_MAX_CHARS) {
+      flush();
+      carryOverlap();
+    }
+    bucket.push({ line, lineNo: i + 1 });
+    chars += rowSize;
+  }
+  flush();
+  return out;
+}
+
+async function readMemoryChunks(context: AgentContext): Promise<Array<{
+  path: string;
+  source: "longterm" | "daily" | "working";
+  startLine: number;
+  endLine: number;
+  text: string;
+}>> {
+  const files = await listMemorySourceFiles(context.rootPath);
+  const out: Array<{
+    path: string;
+    source: "longterm" | "daily" | "working";
+    startLine: number;
+    endLine: number;
+    text: string;
+  }> = [];
+  for (const file of files) {
+    const content = String(await fs.readFile(file.absPath, "utf-8"));
+    for (const chunk of chunkMarkdown(content)) {
+      out.push({
+        path: file.relPath,
+        source: file.source,
+        startLine: chunk.startLine,
+        endLine: chunk.endLine,
+        text: chunk.text,
+      });
+    }
+  }
+  return out;
+}
+
+/**
+ * 收集当前 memory Markdown 状态。
+ */
+export async function collectMemoryStatus(
+  context: AgentContext,
+  state: MemoryRuntimeState,
+): Promise<MemoryStatusResponse> {
+  const files = await listMemorySourceFiles(context.rootPath);
+  const sourceCounts: MemorySourceStat[] = [
+    { source: "longterm", files: 0, chunks: 0 },
+    { source: "daily", files: 0, chunks: 0 },
+    { source: "working", files: 0, chunks: 0 },
+  ];
+
+  let totalChunks = 0;
+  for (const file of files) {
+    const bucket = sourceCounts.find((item) => item.source === file.source);
+    if (bucket) {
+      bucket.files += 1;
+    }
+    const content = String(await fs.readFile(file.absPath, "utf-8"));
+    const chunks = chunkMarkdown(content).length;
+    totalChunks += chunks;
+    if (bucket) {
+      bucket.chunks += chunks;
+    }
+  }
+
+  return {
+    enabled: state.enabled,
+    backend: "builtin",
+    mode: "scan",
+    files: files.length,
+    chunks: totalChunks,
+    sourceCounts,
+  };
+}
+
 /**
  * 执行检索。
  */
@@ -33,7 +226,7 @@ export async function searchMemory(
   if (!state.enabled) {
     return {
       backend: "builtin",
-      mode: "fts",
+      mode: "scan",
       results: [],
       disabled: true,
       error: "memory service disabled",
@@ -45,7 +238,16 @@ export async function searchMemory(
   if (!query) {
     return {
       backend: "builtin",
-      mode: "fts",
+      mode: "scan",
+      results: [],
+    };
+  }
+
+  const tokens = tokenizeQuery(query);
+  if (tokens.length === 0) {
+    return {
+      backend: "builtin",
+      mode: "scan",
       results: [],
     };
   }
@@ -64,27 +266,65 @@ export async function searchMemory(
   );
 
   try {
-    await ensureMemoryIndexed(context, state, { reason: "search" });
-    const results = state.indexer.search({
-      query,
-      maxResults,
-      minScore,
-      maxInjectedChars: MEMORY_DEFAULTS.maxInjectedChars,
-    });
+    const results = (await readMemoryChunks(context))
+      .map((chunk) => {
+        const score = buildSnippetScore(chunk.text, tokens);
+        const citation =
+          chunk.startLine === chunk.endLine
+            ? `${chunk.path}#L${chunk.startLine}`
+            : `${chunk.path}#L${chunk.startLine}-L${chunk.endLine}`;
+        return {
+          path: chunk.path,
+          source: chunk.source,
+          startLine: chunk.startLine,
+          endLine: chunk.endLine,
+          score,
+          snippet: truncateText(chunk.text, SNIPPET_MAX_CHARS),
+          citation,
+        } satisfies MemorySearchResultItem;
+      })
+      .filter((item) => item.score >= minScore)
+      .sort((left, right) => {
+        if (right.score !== left.score) {
+          return right.score - left.score;
+        }
+        if (left.path !== right.path) {
+          return left.path.localeCompare(right.path);
+        }
+        return left.startLine - right.startLine;
+      })
+      .slice(0, maxResults * 3);
+
+    let remain = Math.max(0, MEMORY_DEFAULTS.maxInjectedChars);
+    const clamped: MemorySearchResultItem[] = [];
+    for (const item of results) {
+      if (remain <= 0 || clamped.length >= maxResults) break;
+      if (item.snippet.length <= remain) {
+        clamped.push(item);
+        remain -= item.snippet.length;
+        continue;
+      }
+      clamped.push({
+        ...item,
+        snippet: item.snippet.slice(0, remain),
+      });
+      break;
+    }
+
     return {
       backend: "builtin",
-      mode: "fts",
-      results,
+      mode: "scan",
+      results: clamped,
     };
   } catch (error) {
     const message = String(error);
     return {
       backend: "builtin",
-      mode: "fts",
+      mode: "scan",
       results: [],
       disabled: true,
       error: message,
-      action: "Run `city memory index --force` to rebuild local memory index.",
+      action: "Check the Markdown memory files under `.downcity/memory/`.",
     };
   }
 }

package/src/service/builtins/memory/runtime/Store.ts

@@ -2,32 +2,25 @@
  * Memory Store（文件与运行态管理）。
  *
  * 关键点（中文）
- * - 管理 Memory service 的运行时状态（dirty/sync/watcher）。
- * - 统一管理 memory 源文件枚举与路径白名单。
- * - 不承载检索算法，检索在 Search/Indexer 模块。
+ * - 管理 Memory service 的最小运行时状态。
+ * - 统一管理 memory 源文件枚举。
+ * - 不承载检索算法，检索在 Search 模块。
  * - 新版本不再使用 module-global state，状态归属 MemoryService 实例。
  */
 
-import fs from "node:fs";
+import type { Dirent } from "node:fs";
 import fsp from "node:fs/promises";
 import path from "node:path";
-import type { FSWatcher } from "node:fs";
 import type { AgentContext } from "@/agent/AgentContextTypes.js";
 import type {
   MemoryDefaults,
   MemorySourceType,
 } from "@/service/builtins/memory/types/Memory.js";
-import {
-  MemoryIndexer,
-  type MemoryIndexSyncResult,
-} from "./Indexer.js";
 
 export const MEMORY_DEFAULTS: MemoryDefaults = {
   maxResults: 6,
   minScore: 0.35,
   maxInjectedChars: 4000,
-  watchDebounceMs: 1500,
-  intervalMinutes: 5,
 };
 
 export type MemorySourceFile = {
@@ -54,38 +47,6 @@ export type MemoryRuntimeState = {
    * Memory 是否启用。
    */
   enabled: boolean;
-  /**
-   * 索引器实例。
-   */
-  indexer: MemoryIndexer;
-  /**
-   * 当前是否 dirty。
-   */
-  dirty: boolean;
-  /**
-   * 最近一次同步时间戳。
-   */
-  lastSyncAt?: number;
-  /**
-   * 最近一次同步错误。
-   */
-  lastError?: string;
-  /**
-   * 当前同步中的 Promise（用于并发合并）。
-   */
-  syncing: Promise<void> | null;
-  /**
-   * watcher 列表。
-   */
-  watchers: FSWatcher[];
-  /**
-   * debounce timer。
-   */
-  watchDebounceTimer: NodeJS.Timeout | null;
-  /**
-   * interval timer。
-   */
-  intervalTimer: NodeJS.Timeout | null;
 };
 
 function normalizeRelPath(rootPath: string, absPath: string): string {
@@ -107,7 +68,7 @@ async function pathExists(absPath: string): Promise<boolean> {
 
 async function listMarkdownFilesRecursively(dirPath: string): Promise<string[]> {
   const out: string[] = [];
-  let entries: fs.Dirent[] = [];
+  let entries: Dirent[] = [];
   try {
     entries = await fsp.readdir(dirPath, { withFileTypes: true });
   } catch {
@@ -156,7 +117,7 @@ export async function listMemorySourceFiles(
   }
 
   const sessionRootDir = path.join(rootPath, ".downcity", "session");
-  let sessions: fs.Dirent[] = [];
+  let sessions: Dirent[] = [];
   try {
     sessions = await fsp.readdir(sessionRootDir, { withFileTypes: true });
   } catch {
@@ -206,64 +167,15 @@ export function createMemoryRuntimeState(
   return {
     rootPath: context.rootPath,
     enabled: isMemoryEnabled(context),
-    indexer: new MemoryIndexer(context.rootPath),
-    dirty: true,
-    syncing: null,
-    watchers: [],
-    watchDebounceTimer: null,
-    intervalTimer: null,
   };
 }
 
 /**
- * 标记 dirty 并触发 debounce 同步。
- */
-export function markMemoryDirty(
-  context: AgentContext,
-  state: MemoryRuntimeState,
-  reason: string,
-): void {
-  state.dirty = true;
-  if (state.watchDebounceTimer) {
-    clearTimeout(state.watchDebounceTimer);
-  }
-  state.watchDebounceTimer = setTimeout(() => {
-    state.watchDebounceTimer = null;
-    void ensureMemoryIndexed(context, state, { reason: `watch:${reason}` });
-  }, MEMORY_DEFAULTS.watchDebounceMs);
-  if (typeof state.watchDebounceTimer.unref === "function") {
-    state.watchDebounceTimer.unref();
-  }
-}
-
-function registerWatcher(
-  context: AgentContext,
-  state: MemoryRuntimeState,
-  watchPath: string,
-): void {
-  try {
-    const watcher = fs.watch(
-      watchPath,
-      { recursive: true },
-      (_event, fileName) => {
-        const name = String(fileName || "").toLowerCase();
-        if (!name.endsWith(".md")) {
-          return;
-        }
-        markMemoryDirty(context, state, name || "unknown");
-      },
-    );
-    state.watchers.push(watcher);
-  } catch (error) {
-    context.logger.warn("[memory] watcher init skipped", {
-      watchPath,
-      error: String(error),
-    });
-  }
-}
-
-/**
- * 启动 memory 运行时（watcher + interval）。
+ * 启动 memory 运行时。
+ *
+ * 关键点（中文）
+ * - Markdown-only 方案下不再维护后台索引同步。
+ * - 这里只负责根据配置刷新 enabled 状态。
  */
 export async function startMemoryRuntime(
   context: AgentContext,
@@ -274,97 +186,13 @@ export async function startMemoryRuntime(
     context.logger.info("[memory] disabled by config");
     return;
   }
-
-  if (state.watchers.length === 0) {
-    registerWatcher(context, state, path.join(context.rootPath, ".downcity"));
-  }
-  if (!state.intervalTimer) {
-    state.intervalTimer = setInterval(() => {
-      void ensureMemoryIndexed(context, state, { reason: "interval" });
-    }, MEMORY_DEFAULTS.intervalMinutes * 60 * 1000);
-    if (typeof state.intervalTimer.unref === "function") {
-      state.intervalTimer.unref();
-    }
-  }
-
-  await ensureMemoryIndexed(context, state, { reason: "startup" });
 }
 
 /**
  * 停止 memory 运行时。
  */
 export async function stopMemoryRuntime(
-  state: MemoryRuntimeState,
+  _state: MemoryRuntimeState,
 ): Promise<void> {
-  if (state.watchDebounceTimer) {
-    clearTimeout(state.watchDebounceTimer);
-    state.watchDebounceTimer = null;
-  }
-  if (state.intervalTimer) {
-    clearInterval(state.intervalTimer);
-    state.intervalTimer = null;
-  }
-  for (const watcher of state.watchers) {
-    try {
-      watcher.close();
-    } catch {
-      // ignore
-    }
-  }
-  state.watchers = [];
-
-  if (state.syncing) {
-    try {
-      await state.syncing;
-    } catch {
-      // ignore
-    }
-  }
-  state.indexer.close();
-}
-
-/**
- * 确保索引已同步。
- */
-export async function ensureMemoryIndexed(
-  context: AgentContext,
-  state: MemoryRuntimeState,
-  params?: { force?: boolean; reason?: string },
-): Promise<MemoryIndexSyncResult | null> {
-  if (!state.enabled) {
-    return null;
-  }
-  const force = params?.force === true;
-  if (!force && !state.dirty) {
-    return null;
-  }
-  if (state.syncing) {
-    await state.syncing;
-    return null;
-  }
-  let syncResult: MemoryIndexSyncResult | null = null;
-  state.syncing = (async () => {
-    try {
-      const files = await listMemorySourceFiles(context.rootPath);
-      syncResult = await state.indexer.sync(files, { force });
-      state.dirty = false;
-      state.lastError = undefined;
-      state.lastSyncAt = Date.now();
-      context.logger.info("[memory] index synced", {
-        reason: params?.reason || "manual",
-        files: files.length,
-      });
-    } catch (error) {
-      state.lastError = String(error);
-      context.logger.error("[memory] index sync failed", {
-        reason: params?.reason || "manual",
-        error: state.lastError,
-      });
-      throw error;
-    } finally {
-      state.syncing = null;
-    }
-  })();
-  await state.syncing;
-  return syncResult;
+  void _state;
 }

package/src/service/builtins/memory/runtime/Writer.ts

@@ -17,7 +17,6 @@ import type {
   MemoryStoreResponse,
 } from "@/service/builtins/memory/types/Memory.js";
 import type { MemoryRuntimeState } from "./Store.js";
-import { markMemoryDirty } from "./Store.js";
 
 function nowIso(): string {
   return new Date().toISOString();
@@ -88,6 +87,7 @@ export async function storeMemory(
   state: MemoryRuntimeState,
   payload: MemoryStorePayload,
 ): Promise<MemoryStoreResponse> {
+  void state;
   const target: MemorySourceType = payload.target ?? "daily";
   const content = String(payload.content || "").trim();
   if (!content) {
@@ -104,7 +104,6 @@ export async function storeMemory(
   }
   const entry = formatEntry(content);
   await fs.appendFile(resolved.absPath, `\n${entry}`, "utf-8");
-  markMemoryDirty(context, state, `store:${resolved.relPath}`);
   return {
     path: resolved.relPath,
     target,